diff --git a/daemon-rs/rustfmt.toml b/daemon-rs/rustfmt.toml new file mode 100644 index 00000000..03c2967a --- /dev/null +++ b/daemon-rs/rustfmt.toml @@ -0,0 +1,8 @@ +edition = "2021" +max_width = 160 +use_small_heuristics = "Max" +fn_params_layout = "Compressed" +struct_lit_width = 100 +struct_variant_width = 100 +array_width = 120 +chain_width = 120 diff --git a/daemon-rs/scripts/add_pub_crate.py b/daemon-rs/scripts/add_pub_crate.py deleted file mode 100644 index 471cffdb..00000000 --- a/daemon-rs/scripts/add_pub_crate.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -"""Add pub(crate) to CLI module entry points.""" - -from pathlib import Path -import re - -CLI = Path(__file__).resolve().parents[1] / "src" / "cli" - -PRIVATE_PREFIXES = { - "common": ( - "resolve_client_target_inputs", - "parse_truthy_flag", - "mask_secret_for_logs", - "read_auth_token_from_path", - "resolve_boot_auth_header", - ), - "cleanup": ( - "rotate_backups", - "cleanup_expired_rows", - "rotate_log_file", - "collect_backup_cleanup_files", - "format_cleanup_bytes", - "path_size_bytes", - "run_backup_cleanup", - "run_log_cleanup", - "run_bridge_backup_cleanup", - "run_stale_pid_cleanup", - ), - "status": ( - "status_", - "compact_status_", - "print_status_", - "probe_status_", - ), - "usage": ("top_level_command_",), - "admin": ("print_budget_status_human",), - "sync": ( - "validate_sync_cli_options", - "parse_import_cli_args", - "import_payload_from_file", - "validate_import_", - "validate_rfc3339", - "resolve_sync_since", - "read_sync_cursor", - "write_sync_cursor", - "ensure_sync_site", - "sanitize_sync", - "sync_watch_state", - "is_sync_changeset", - "collect_sync_watch", - "load_sync_seen", - "write_sync_seen", - "acquire_sync_lock", - "export_snapshot", - "export_changeset", - "write_atomic_text", - "writable_parent", - "sync_parent_dir", - "run_sync_export_cli", - "run_sync_import_cli", - "run_sync_watch_cli", - ), - "daemon": ( - "daemon_lock_", - "try_acquire_", - "acquire_runtime", - "daemon_owner_", - "spawn_parent_", - "should_watch_", - "is_control_center", - "parse_env_u64_nonnegative", - "app_managed_", - "startup_delay", - "startup_schedule", - "background_db_lock", - "acquire_background_db_lock", - "process_pid", - "process_looks_like", - "detect_other_cortex", - "spawned_owner_", - "validate_spawned_owner", - "app_init_required", - "local_spawn_allowed", - "control_center_lock", - "is_lock_contention", - "control_center_is_active", - "ensure_service_ready", - "plugin_owner_tag", - "normalized_path_for_guard", - "path_is_under_root", - "ensure_local_plugin_spawn", - "read_auth_token", - "backfill_batch", - "collect_unembedded", - "count_unembedded", - "build_embeddings_async", - "request_boot_payload", - ), -} - - -def should_privatize(module: str, name: str) -> bool: - return any(name.startswith(p) for p in PRIVATE_PREFIXES.get(module, ())) - - -def process_file(path: Path) -> None: - module = path.stem - out_lines = [] - for line in path.read_text().splitlines(): - if line.startswith("pub(crate)") or line.startswith("pub "): - out_lines.append(line) - continue - m = re.match(r"^(async )?fn ([a-zA-Z0-9_]+)", line) - if m and not should_privatize(module, m.group(2)): - if line.startswith("async fn "): - line = "pub(crate) " + line - elif line.startswith("fn "): - line = "pub(crate) " + line - out_lines.append(line) - path.write_text("\n".join(out_lines) + "\n") - - -def main() -> None: - for path in sorted(CLI.glob("*.rs")): - if path.name in ("mod.rs", "tests.rs"): - continue - process_file(path) - - -if __name__ == "__main__": - main() diff --git a/daemon-rs/scripts/fix_branch_modules.py b/daemon-rs/scripts/fix_branch_modules.py deleted file mode 100644 index 39af24e9..00000000 --- a/daemon-rs/scripts/fix_branch_modules.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -"""Fix compile issues from parallel branch module splits.""" -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - - -def replace_in(path: str, old: str, new: str) -> None: - p = ROOT / path - if not p.exists(): - return - text = p.read_text() - if old in text: - p.write_text(text.replace(old, new)) - - -def main() -> None: - replace_in( - "src/db/connection.rs", - " pub(crate) fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {", - " fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {", - ) - replace_in( - "src/handlers/mutate/types.rs", - "use rusqlite::{params, Connection};\nuse serde_json::{json, Value};", - "use rusqlite::{params, Connection};\nuse serde::Deserialize;\nuse serde_json::{json, Value};", - ) - replace_in( - "src/handlers/mutate/types.rs", - " pub(crate) fn default() -> Self {", - " fn default() -> Self {", - ) - replace_in( - "src/handlers/store/types.rs", - " pub(crate) fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {", - " fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {", - ) - replace_in( - "src/handlers/store/types.rs", - " pub(crate) fn from(value: String) -> Self {", - " fn from(value: String) -> Self {", - ) - - -if __name__ == "__main__": - main() diff --git a/daemon-rs/scripts/fix_remaining_splits.py b/daemon-rs/scripts/fix_remaining_splits.py deleted file mode 100644 index 15c09fb0..00000000 --- a/daemon-rs/scripts/fix_remaining_splits.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -"""Post-split fixes for remaining module refactor.""" -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - - -def strip_duplicate_import_block(path: Path, marker: str = "use super::*;\n") -> None: - text = path.read_text(encoding="utf-8") - idx = text.find(marker) - if idx == -1: - return - after = text[idx + len(marker) :] - if not after.startswith("use "): - return - # Drop duplicated import block until blank line before const/fn - lines = after.splitlines(keepends=True) - cut = 0 - for i, line in enumerate(lines): - if line.startswith("use ") or line.strip() == "": - cut = i + 1 - continue - break - path.write_text(text[: idx + len(marker)] + "".join(lines[cut:]), encoding="utf-8") - - -def strip_trailing_orphan(path: Path) -> None: - text = path.read_text(encoding="utf-8").rstrip() + "\n" - while True: - stripped = text.rstrip("\n") - if stripped.endswith("#[cfg(test)]"): - text = stripped[: -len("#[cfg(test)]")].rstrip() + "\n" - continue - if stripped.endswith("/// 5. Return prompt with compilation metadata and savings"): - # orphan doc lines from mis-split compile docs - lines = stripped.splitlines() - while lines and (lines[-1].startswith("///") or lines[-1].strip() == ""): - lines.pop() - text = "\n".join(lines) + "\n" - continue - break - path.write_text(text, encoding="utf-8") - - -def fix_cli_daemon_imports(path: Path) -> None: - text = path.read_text(encoding="utf-8") - text = text.replace("use super::boot::", "use crate::cli::boot::") - text = text.replace("use super::cleanup::", "use crate::cli::cleanup::") - text = text.replace("use super::common::", "use crate::cli::common::") - path.write_text(text, encoding="utf-8") - - -def main() -> None: - for name in ("session.rs", "run.rs"): - strip_duplicate_import_block(ROOT / "src/mcp_proxy" / name) - - for name in ("packing.rs", "compile.rs"): - strip_trailing_orphan(ROOT / "src/compiler" / name) - - strip_trailing_orphan(ROOT / "src/mcp_proxy/run.rs") - strip_trailing_orphan(ROOT / "src/server/runtime.rs") - - compile = ROOT / "src/compiler/compile.rs" - doc = ( - "/// Compile the boot prompt for an agent within a token budget.\n" - "///\n" - "/// Prompt Compiler Pipeline (v3 -- score-adaptive context packing):\n" - "/// 1. Gather all context items with priority scores\n" - "/// 2. Sort by utility (priority / token_cost) -- best bang-per-token first\n" - "/// 3. Pack within budget using score-adaptive truncation when score variance exists\n" - "/// 4. Record admitted vs rejected for observability\n" - "/// 5. Return prompt with compilation metadata and savings\n" - ) - text = compile.read_text(encoding="utf-8") - if doc.strip() not in text: - text = text.replace("use super::*;\n", f"use super::*;\n{doc}", 1) - compile.write_text(text, encoding="utf-8") - - (ROOT / "src/cli/daemon/mod.rs").write_text( - """// SPDX-License-Identifier: MIT -mod startup; -mod run; -mod backfill; - -pub(crate) use startup::*; -pub(crate) use run::*; -pub(crate) use backfill::*; -""", - encoding="utf-8", - ) - - (ROOT / "src/mcp_proxy/mod.rs").write_text( - """// SPDX-License-Identifier: MIT -mod session; -mod run; - -#[cfg(test)] -mod tests; - -pub(crate) use session::*; -pub(crate) use run::*; - -pub use run::run; -""", - encoding="utf-8", - ) - - for name in ("startup.rs", "run.rs", "backfill.rs"): - fix_cli_daemon_imports(ROOT / "src/cli/daemon" / name) - - print("post-split fixes applied") - - -if __name__ == "__main__": - main() diff --git a/daemon-rs/scripts/write_main_rs.py b/daemon-rs/scripts/write_main_rs.py deleted file mode 100644 index be0c578b..00000000 --- a/daemon-rs/scripts/write_main_rs.py +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path - -MAIN = Path(__file__).resolve().parents[1] / "src" / "main.rs" - -MAIN.write_text("""// SPDX-License-Identifier: MIT - -/// Default TCP port the Cortex daemon binds to when no `--port` flag or -/// `CORTEX_PORT` env var is set. -pub const DEFAULT_CORTEX_PORT: u16 = 7437; - -mod admin; -mod aging; -mod api_types; -mod auth; -mod budgets; -mod cli; -mod co_occurrence; -mod compaction; -mod compiler; -mod conflict; -mod crystallize; -mod daemon_lifecycle; -mod db; -mod embeddings; -mod eval; -mod export_data; -mod focus; -mod handlers; -mod hook_boot; -mod indexer; -mod mcp_proxy; -mod prompt_inject; -mod rate_limit; -mod rerank; -mod server; -mod service; -mod setup; -mod state; -#[cfg(test)] -mod test_env; -mod tls; -mod transport; -mod workspace; - -use chrono::Utc; -use std::io::Write as _; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; - -use cli::{ - apply_path_env, cli_capabilities_payload, cli_capabilities_summary, cli_robot_docs_guide, - cli_service_usage, ensure_daemon, ensure_remote_target_has_api_key, - is_disallowed_startup_binary_path, parse_flag_usize, parse_flag_value, print_usage_and_exit, - resolve_client_target, run_admin_cli, run_backup_cli, run_boot_cli, run_cleanup_cli, - run_doctor_cli, run_embeddings_cli, run_embeddings_drain_cli, run_eval_cli, run_export_cli, - run_import_cli, run_recrystallize_cli, run_reindex_cli, run_restore_cli, run_status_cli, - run_sync_cli, run_team_cli, run_user_cli, unknown_cli_command_message, - unknown_robot_docs_subcommand_message, validate_cli_options_or_exit, -}; - -pub(crate) use cli::run_daemon; - -pub(crate) fn install_daemon_panic_hook(paths: &auth::CortexPaths) { - static INSTALLED: AtomicBool = AtomicBool::new(false); - if INSTALLED.swap(true, Ordering::SeqCst) { - return; - } - let panic_log_path = paths.home.join("panic.log"); - let previous = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - let payload = info.payload(); - let message = if let Some(s) = payload.downcast_ref::<&str>() { - (*s).to_string() - } else if let Some(s) = payload.downcast_ref::() { - s.clone() - } else { - "".to_string() - }; - let location = info - .location() - .map(|loc| format!("{}:{}:{}", loc.file(), loc.line(), loc.column())) - .unwrap_or_else(|| "".to_string()); - let backtrace = std::backtrace::Backtrace::force_capture(); - let entry = format!( - "[{ts}] PANIC at {location}: {message}\\n{backtrace}\\n", - ts = Utc::now().to_rfc3339(), - ); - eprintln!("[cortex] {entry}"); - if let Some(parent) = panic_log_path.parent() { - let _ = std::fs::create_dir_all(parent); - } - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&panic_log_path) - { - let _ = file.write_all(entry.as_bytes()); - } - previous(info); - })); -} - -#[tokio::main] -async fn main() { - let args: Vec = std::env::args().collect(); - let mode = args.get(1).map(|s| s.as_str()).unwrap_or(""); - let paths = auth::CortexPaths::resolve_from_args(&args); - if let Ok(current_exe) = std::env::current_exe() { - if is_disallowed_startup_binary_path(¤t_exe) { - eprintln!( - "[cortex] Refusing to run from disallowed runtime path: {}", - current_exe.display() - ); - std::process::exit(1); - } - } - - match mode { - "" | "--help" | "-h" | "help" => print_usage_and_exit(0), - "--version" | "-V" | "version" => println!("cortex {}", env!("CARGO_PKG_VERSION")), - "capabilities" => { - validate_cli_options_or_exit(&args[2..], &[], &["--json"]); - if args.iter().any(|arg| arg == "--json") { - println!("{}", serde_json::to_string_pretty(&cli_capabilities_payload()).unwrap()); - } else { - println!("{}", cli_capabilities_summary()); - } - } - "status" => { - validate_cli_options_or_exit(&args[2..], &[], &["--json"]); - let exit_code = run_status_cli(&paths, args.iter().any(|arg| arg == "--json")).await; - if exit_code != 0 { - std::process::exit(exit_code); - } - } - "robot-docs" => { - let subcmd = args.get(2).map(String::as_str).unwrap_or("guide"); - match subcmd { - "" | "guide" | "help" | "--help" | "-h" => println!("{}", cli_robot_docs_guide()), - other => { - eprintln!("{}", unknown_robot_docs_subcommand_message(other)); - std::process::exit(1); - } - } - } - "serve" => { - validate_cli_options_or_exit(&args[2..], &[], &[]); - #[cfg(unix)] - async fn sigterm_future() { - use tokio::signal::unix::{signal, SignalKind}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(sigterm) => sigterm, - Err(err) => { - eprintln!("[cortex] Warning: failed to register SIGTERM handler: {err}"); - std::future::pending::<()>().await; - return; - } - }; - sigterm.recv().await; - } - #[cfg(not(unix))] - async fn sigterm_future() { - std::future::pending::<()>().await; - } - run_daemon(paths.clone(), async { - tokio::select! { - _ = tokio::signal::ctrl_c() => eprintln!("[cortex] Received Ctrl+C, shutting down..."), - _ = sigterm_future() => eprintln!("[cortex] Received SIGTERM, shutting down..."), - } - }).await; - } - "mcp" => { - let remaining = &args[2..]; - validate_cli_options_or_exit(remaining, &["--agent", "--url", "--api-key"], &[]); - let agent = parse_flag_value(remaining, "--agent"); - let (base_url, api_key, local_owner_mode) = resolve_client_target(remaining, &paths); - if let Err(e) = ensure_remote_target_has_api_key(&base_url, api_key.as_deref(), &paths) { - eprintln!("[cortex-mcp] {e}"); - std::process::exit(1); - } - if local_owner_mode { - apply_path_env(&paths); - if let Err(e) = ensure_daemon(&paths, agent.as_deref(), false, false).await { - eprintln!("[cortex-mcp] {e}"); - std::process::exit(1); - } - } - if let Err(e) = mcp_proxy::run(&base_url, api_key.as_deref(), agent.as_deref()).await { - eprintln!("[cortex-mcp] {e}"); - std::process::exit(1); - } - } - "paths" => { - validate_cli_options_or_exit(&args[2..], &[], &["--json"]); - if args.iter().any(|a| a == "--json") { - println!("{}", paths.to_json()); - } else { - eprintln!("Usage: cortex paths --json"); - std::process::exit(1); - } - } - "boot" => { - if let Err(e) = run_boot_cli(&paths, &args[2..]).await { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - "plugin" => match args.get(2).map(|s| s.as_str()).unwrap_or("") { - "ensure-daemon" => { - validate_cli_options_or_exit(&args[3..], &["--agent"], &[]); - let agent = parse_flag_value(&args[3..], "--agent"); - apply_path_env(&paths); - if let Err(e) = ensure_daemon(&paths, agent.as_deref(), true, true).await { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - "mcp" => { - let remaining = &args[3..]; - validate_cli_options_or_exit(remaining, &["--agent", "--url", "--api-key"], &[]); - let (base_url, api_key, local_owner_mode) = resolve_client_target(remaining, &paths); - let agent = parse_flag_value(remaining, "--agent"); - if let Err(e) = ensure_remote_target_has_api_key(&base_url, api_key.as_deref(), &paths) { - eprintln!("[cortex-plugin] {e}"); - std::process::exit(1); - } - if local_owner_mode { - apply_path_env(&paths); - if let Err(e) = ensure_daemon(&paths, agent.as_deref(), false, true).await { - eprintln!("[cortex-plugin] {e}"); - std::process::exit(1); - } - } - if let Err(e) = mcp_proxy::run(&base_url, api_key.as_deref(), agent.as_deref()).await { - eprintln!("[cortex-plugin] {e}"); - std::process::exit(1); - } - } - _ => { - eprintln!("Usage: cortex plugin "); - std::process::exit(1); - } - }, - "hook-boot" => { - let agent = args - .get(2) - .and_then(|a| if a == "--agent" { args.get(3).map(|s| s.as_str()) } else { Some(a.as_str()) }) - .unwrap_or("claude-opus"); - hook_boot::run_boot(agent).await; - } - "hook-status" => hook_boot::run_status().await, - "service" => { - let subcmd = args.get(2).cloned().unwrap_or_default(); - if matches!(subcmd.as_str(), "help" | "--help" | "-h") { - println!("{}", cli_service_usage()); - return; - } - let code = match tokio::task::spawn_blocking(move || match subcmd.as_str() { - "install" => u8::from(service::install()), - "uninstall" => u8::from(service::uninstall()), - "start" => u8::from(service::start()), - "stop" => u8::from(service::stop()), - "status" => u8::from(service::status()), - "ensure" => u8::from(service::ensure()), - _ => { eprintln!("{}", cli_service_usage()); 1 } - }).await { - Ok(code) => code, - Err(err) => { eprintln!("[cortex] Service command task failed: {err}"); 1 } - }; - if code != 0 { std::process::exit(code as i32); } - } - "service-run" => service::dispatch_service(), - "prompt-inject" => prompt_inject::run(&args[2..]).await, - "setup" => { - let remaining: Vec = args[2..].to_vec(); - if remaining.iter().any(|a| a == "--team") { - validate_cli_options_or_exit(&remaining, &["--owner", "--display-name"], &["--team", "--dry-run"]); - setup::run_setup_team(&remaining, remaining.iter().any(|a| a == "--dry-run")).await; - } else { - if remaining.iter().any(|a| a == "--dry-run") { - eprintln!("--dry-run requires --team"); - std::process::exit(1); - } - validate_cli_options_or_exit(&remaining, &[], &[]); - setup::run_setup().await; - } - } - "migrate" => { - let remaining: Vec = args[2..].to_vec(); - validate_cli_options_or_exit(&remaining, &["--owner", "--display-name"], &["--dry-run"]); - setup::run_setup_team(&remaining, remaining.iter().any(|a| a == "--dry-run")).await; - } - "export" => run_export_cli(&paths, &args[2..]), - "import" => run_import_cli(&paths, &args[2..]), - "sync" => run_sync_cli(&paths, &args[2..]), - "eval" => run_eval_cli(&paths, &args[2..]), - "doctor" => { - validate_cli_options_or_exit(&args[2..], &[], &[]); - run_doctor_cli(&paths); - } - "reindex" => { - validate_cli_options_or_exit(&args[2..], &[], &["--json"]); - run_reindex_cli(&paths, args.iter().any(|a| a == "--json")); - } - "re-embed" | "reembed" => { - let mut remaining: Vec = args[2..].to_vec(); - if !remaining.iter().any(|arg| arg == "--until-exhausted") { - remaining.push("--until-exhausted".to_string()); - } - run_embeddings_drain_cli(&paths, &remaining).await; - } - "recrystallize" => run_recrystallize_cli(&paths, args.iter().any(|a| a == "--json")).await, - "cleanup" => { - validate_cli_options_or_exit(&args[2..], &["--max-passes"], &["--dry-run", "--events"]); - let max_event_passes = match parse_flag_usize(&args[2..], "--max-passes") { - Ok(Some(value)) => value.clamp(1, 12), - Ok(None) => 3, - Err(err) => { eprintln!("Error: {err}"); std::process::exit(1); } - }; - run_cleanup_cli(&paths, args.iter().any(|a| a == "--dry-run"), args.iter().any(|a| a == "--events"), max_event_passes); - } - "embeddings" => run_embeddings_cli(&paths, &args[2..]).await, - "backup" => { - validate_cli_options_or_exit(&args[2..], &[], &[]); - run_backup_cli(&paths); - } - "restore" => run_restore_cli(&paths, &args), - "user" => run_user_cli(&paths, &args).await, - "team" => run_team_cli(&paths, &args).await, - "admin" => run_admin_cli(&paths, &args).await, - other => { - eprintln!("{}", unknown_cli_command_message(other)); - std::process::exit(1); - } - } -} -""") - -print(f"Wrote {MAIN} ({len(MAIN.read_text().splitlines())} lines)") diff --git a/daemon-rs/src/admin.rs b/daemon-rs/src/admin.rs deleted file mode 100644 index 4f9dddb0..00000000 --- a/daemon-rs/src/admin.rs +++ /dev/null @@ -1,371 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Administrative operations that do not belong on the hot-path MCP/HTTP -//! surface. Kept deliberately thin: each function takes a live `&Connection` -//! and is synchronous. Callers handle process-level concerns (CLI output, -//! JSON formatting, SSE emission). - -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Serialize; - -/// Marker written to `status` for memories and decisions that were part of a -/// rolled-back session. Existing recall paths already filter by -/// `status = 'active'`, so flipping this value transparently hides the rows -/// from recall without touching the hot path. -pub const ROLLED_BACK_STATUS: &str = "rolled_back"; - -/// Counts returned by a dry-run or applied rollback so the CLI + SSE event -/// payload share a single shape. -#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)] -pub struct RollbackStats { - /// Session ID the rollback targeted (echoes the CLI input). - pub session_id: String, - /// Agent identifier (e.g. `claude-code`) resolved from the `sessions` row. - /// Empty when no matching session row exists. - pub agent: String, - /// ISO timestamp `sessions.started_at`. Empty when session not found. - pub session_started_at: String, - /// How many `memories` rows were (or would be) flipped to `rolled_back`. - pub memories_affected: i64, - /// How many `decisions` rows were (or would be) flipped to `rolled_back`. - pub decisions_affected: i64, - /// True when `--apply` was passed; false for dry-run. - pub applied: bool, - /// True when the same rollback was previously applied, in which case - /// the current call is a no-op. Helps callers reason about idempotency. - pub already_rolled_back: bool, -} - -/// Roll back a session by id. Resolves `agent` + `started_at` from the -/// `sessions` table, then soft-deletes every memory and decision written by -/// that agent since session start. -/// -/// **Dry-run by default.** Pass `apply=true` to actually write `status = 'rolled_back'`. -/// -/// **Idempotent.** Re-running after a successful rollback returns counts -/// equal to zero and sets `already_rolled_back=true` when the session's -/// prior memories are all already flipped. -/// -/// Errors on DB failure. Returns a stats struct even when the session id -/// is unknown (counts all zero, `agent`/`session_started_at` empty) — CLI -/// surface decides whether that is an error exit. -pub fn rollback_session_by_id( - conn: &Connection, - session_id: &str, - apply: bool, -) -> rusqlite::Result { - let mut stats = RollbackStats { - session_id: session_id.to_string(), - ..Default::default() - }; - - // Look up the active session row. The `sessions` table is keyed by agent; - // session_id is a column. A session id that rotates across agents would - // match multiple rows — we take the most recent by `started_at` to be - // deterministic. - let session_row: Option<(String, String)> = conn - .query_row( - "SELECT agent, started_at FROM sessions - WHERE session_id = ?1 - ORDER BY started_at DESC - LIMIT 1", - params![session_id], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) - .optional()?; - - let (agent, started_at) = match session_row { - Some(row) => row, - None => return Ok(stats), - }; - stats.agent = agent.clone(); - stats.session_started_at = started_at.clone(); - - // Count candidate rows created by this agent from session start onward. - // Active ones are the rollback target; already-rolled-back ones tell us - // whether this is a repeat invocation. - let active_memories: i64 = conn.query_row( - "SELECT COUNT(*) FROM memories - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = 'active'", - params![agent, started_at], - |r| r.get(0), - )?; - let active_decisions: i64 = conn.query_row( - "SELECT COUNT(*) FROM decisions - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = 'active'", - params![agent, started_at], - |r| r.get(0), - )?; - let prior_rolled_memories: i64 = conn.query_row( - "SELECT COUNT(*) FROM memories - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = ?3", - params![agent, started_at, ROLLED_BACK_STATUS], - |r| r.get(0), - )?; - let prior_rolled_decisions: i64 = conn.query_row( - "SELECT COUNT(*) FROM decisions - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = ?3", - params![agent, started_at, ROLLED_BACK_STATUS], - |r| r.get(0), - )?; - - stats.memories_affected = active_memories; - stats.decisions_affected = active_decisions; - stats.already_rolled_back = active_memories == 0 - && active_decisions == 0 - && (prior_rolled_memories > 0 || prior_rolled_decisions > 0); - - if !apply { - return Ok(stats); - } - - // Apply inside a single transaction. Both updates are idempotent under - // the `status = 'active'` guard, so a rerun after a partial failure is - // safe. - let tx = conn.unchecked_transaction()?; - let updated_memories = tx.execute( - "UPDATE memories - SET status = ?3, - updated_at = datetime('now') - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = 'active'", - params![agent, started_at, ROLLED_BACK_STATUS], - )? as i64; - let updated_decisions = tx.execute( - "UPDATE decisions - SET status = ?3, - updated_at = datetime('now') - WHERE source_agent = ?1 - AND created_at >= ?2 - AND status = 'active'", - params![agent, started_at, ROLLED_BACK_STATUS], - )? as i64; - tx.commit()?; - - // Report the actually-updated counts (should match the pre-count). - stats.memories_affected = updated_memories; - stats.decisions_affected = updated_decisions; - stats.applied = true; - Ok(stats) -} - -#[cfg(test)] -mod tests { - use super::*; - use rusqlite::Connection; - - fn setup() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - // Minimal schema mirroring the two tables + session row the function needs. - conn.execute_batch( - r#" - CREATE TABLE memories ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - text TEXT, - source TEXT, - source_agent TEXT, - status TEXT DEFAULT 'active', - created_at TEXT DEFAULT (datetime('now')), - updated_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE decisions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - decision TEXT, - source_agent TEXT, - status TEXT DEFAULT 'active', - created_at TEXT DEFAULT (datetime('now')), - updated_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE sessions ( - agent TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - started_at TEXT NOT NULL, - last_heartbeat TEXT NOT NULL, - expires_at TEXT NOT NULL - ); - "#, - ) - .unwrap(); - conn - } - - fn seed_session(conn: &Connection, agent: &str, session_id: &str, started_at: &str) { - conn.execute( - "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, ?3, ?3)", - params![agent, session_id, started_at], - ) - .unwrap(); - } - - fn seed_memory(conn: &Connection, agent: &str, text: &str, created_at: &str, status: &str) { - conn.execute( - "INSERT INTO memories(text, source_agent, status, created_at) - VALUES (?1, ?2, ?3, ?4)", - params![text, agent, status, created_at], - ) - .unwrap(); - } - - fn seed_decision(conn: &Connection, agent: &str, decision: &str, created_at: &str) { - conn.execute( - "INSERT INTO decisions(decision, source_agent, status, created_at) - VALUES (?1, ?2, 'active', ?3)", - params![decision, agent, created_at], - ) - .unwrap(); - } - - #[test] - fn unknown_session_returns_zero_stats() { - let conn = setup(); - let stats = rollback_session_by_id(&conn, "nonexistent", false).unwrap(); - assert_eq!(stats.memories_affected, 0); - assert_eq!(stats.decisions_affected, 0); - assert_eq!(stats.agent, ""); - assert!(!stats.applied); - assert!(!stats.already_rolled_back); - } - - #[test] - fn dry_run_counts_without_writing() { - let conn = setup(); - seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); - seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); - seed_memory(&conn, "claude", "m2", "2026-04-24T00:10:00Z", "active"); - seed_decision(&conn, "claude", "d1", "2026-04-24T00:06:00Z"); - - let stats = rollback_session_by_id(&conn, "sess-1", false).unwrap(); - assert_eq!(stats.agent, "claude"); - assert_eq!(stats.memories_affected, 2); - assert_eq!(stats.decisions_affected, 1); - assert!(!stats.applied); - - // Confirm nothing was actually written - let active: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(active, 2); - } - - #[test] - fn apply_flips_statuses_and_excludes_older_rows() { - let conn = setup(); - seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); - // Older memory predating the session — must not be touched. - seed_memory(&conn, "claude", "old", "2026-04-23T23:59:00Z", "active"); - // Another agent's memory in the same time window — must not be touched. - seed_memory( - &conn, - "codex", - "other-agent", - "2026-04-24T00:05:00Z", - "active", - ); - // In-session memories + one already-rolled-back row to confirm we only - // touch active rows. - seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); - seed_memory(&conn, "claude", "m2", "2026-04-24T00:10:00Z", "active"); - seed_memory( - &conn, - "claude", - "pre-rolled", - "2026-04-24T00:07:00Z", - ROLLED_BACK_STATUS, - ); - seed_decision(&conn, "claude", "d1", "2026-04-24T00:06:00Z"); - - let stats = rollback_session_by_id(&conn, "sess-1", true).unwrap(); - assert_eq!( - stats.memories_affected, 2, - "only the 2 active in-session memories" - ); - assert_eq!(stats.decisions_affected, 1); - assert!(stats.applied); - - // Pre-existing rows preserved - let old_status: String = conn - .query_row("SELECT status FROM memories WHERE text = 'old'", [], |r| { - r.get(0) - }) - .unwrap(); - assert_eq!(old_status, "active"); - let other_status: String = conn - .query_row( - "SELECT status FROM memories WHERE text = 'other-agent'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(other_status, "active"); - - // Our two in-session memories flipped - let rolled: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE source_agent = 'claude' AND status = ?1", - params![ROLLED_BACK_STATUS], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(rolled, 3); // pre-rolled + 2 freshly rolled - } - - #[test] - fn idempotent_second_apply() { - let conn = setup(); - seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); - seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); - - // First apply rolls back - let first = rollback_session_by_id(&conn, "sess-1", true).unwrap(); - assert_eq!(first.memories_affected, 1); - assert!(first.applied); - - // Second apply is a no-op — zero rows left to flip. - let second = rollback_session_by_id(&conn, "sess-1", true).unwrap(); - assert_eq!(second.memories_affected, 0); - assert_eq!(second.decisions_affected, 0); - assert!(second.applied); - assert!(second.already_rolled_back); - } - - #[test] - fn multi_row_session_chooses_most_recent() { - // Two sessions with the same id but different agents (edge case — - // we take the most-recent started_at). This is odd in practice - // but guards against cross-agent collisions. - let conn = setup(); - conn.execute( - "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) - VALUES ('old-agent', 'sess-1', '2026-04-24T00:00:00Z', - '2026-04-24T00:00:00Z', '2026-04-25T00:00:00Z')", - [], - ) - .unwrap(); - conn.execute( - "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) - VALUES ('new-agent', 'sess-1', '2026-04-24T01:00:00Z', - '2026-04-24T01:00:00Z', '2026-04-25T01:00:00Z')", - [], - ) - .unwrap(); - seed_memory(&conn, "new-agent", "m1", "2026-04-24T01:30:00Z", "active"); - seed_memory(&conn, "old-agent", "o1", "2026-04-24T00:30:00Z", "active"); - - let stats = rollback_session_by_id(&conn, "sess-1", false).unwrap(); - assert_eq!(stats.agent, "new-agent"); - assert_eq!(stats.memories_affected, 1); - } -} diff --git a/daemon-rs/src/admin/tests/mod.rs b/daemon-rs/src/admin/tests/mod.rs new file mode 100644 index 00000000..f57ea2b3 --- /dev/null +++ b/daemon-rs/src/admin/tests/mod.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use rusqlite::Connection; +fn setup() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + r#" + CREATE TABLE memories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + text TEXT, + source TEXT, + source_agent TEXT, + status TEXT DEFAULT 'active', + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')) + ); + CREATE TABLE decisions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + decision TEXT, + source_agent TEXT, + status TEXT DEFAULT 'active', + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')) + ); + CREATE TABLE sessions ( + agent TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + started_at TEXT NOT NULL, + last_heartbeat TEXT NOT NULL, + expires_at TEXT NOT NULL + ); + "#, + ) + .unwrap(); + conn +} +fn seed_session(conn: &Connection, agent: &str, session_id: &str, started_at: &str) { + conn.execute( + "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) + VALUES (?1, ?2, ?3, ?3, ?3)", + params![agent, session_id, started_at], + ) + .unwrap(); +} +fn seed_memory(conn: &Connection, agent: &str, text: &str, created_at: &str, status: &str) { + conn.execute( + "INSERT INTO memories(text, source_agent, status, created_at) + VALUES (?1, ?2, ?3, ?4)", + params![text, agent, status, created_at], + ) + .unwrap(); +} +fn seed_decision(conn: &Connection, agent: &str, decision: &str, created_at: &str) { + conn.execute( + "INSERT INTO decisions(decision, source_agent, status, created_at) + VALUES (?1, ?2, 'active', ?3)", + params![decision, agent, created_at], + ) + .unwrap(); +} +#[test] +fn unknown_session_returns_zero_stats() { + let conn = setup(); + let stats = rollback_session_by_id(&conn, "nonexistent", false).unwrap(); + assert_eq!(stats.memories_affected, 0); + assert_eq!(stats.decisions_affected, 0); + assert_eq!(stats.agent, ""); + assert!(!stats.applied); + assert!(!stats.already_rolled_back); +} +#[test] +fn dry_run_counts_without_writing() { + let conn = setup(); + seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); + seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); + seed_memory(&conn, "claude", "m2", "2026-04-24T00:10:00Z", "active"); + seed_decision(&conn, "claude", "d1", "2026-04-24T00:06:00Z"); + let stats = rollback_session_by_id(&conn, "sess-1", false).unwrap(); + assert_eq!(stats.agent, "claude"); + assert_eq!(stats.memories_affected, 2); + assert_eq!(stats.decisions_affected, 1); + assert!(!stats.applied); + let active: i64 = conn.query_row("SELECT COUNT(*) FROM memories WHERE status = 'active'", [], |r| r.get(0)).unwrap(); + assert_eq!(active, 2); +} +#[test] +fn apply_flips_statuses_and_excludes_older_rows() { + let conn = setup(); + seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); + seed_memory(&conn, "claude", "old", "2026-04-23T23:59:00Z", "active"); + seed_memory(&conn, "codex", "other-agent", "2026-04-24T00:05:00Z", "active"); + seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); + seed_memory(&conn, "claude", "m2", "2026-04-24T00:10:00Z", "active"); + seed_memory(&conn, "claude", "pre-rolled", "2026-04-24T00:07:00Z", ROLLED_BACK_STATUS); + seed_decision(&conn, "claude", "d1", "2026-04-24T00:06:00Z"); + let stats = rollback_session_by_id(&conn, "sess-1", true).unwrap(); + assert_eq!(stats.memories_affected, 2, "only the 2 active in-session memories"); + assert_eq!(stats.decisions_affected, 1); + assert!(stats.applied); + let old_status: String = conn.query_row("SELECT status FROM memories WHERE text = 'old'", [], |r| r.get(0)).unwrap(); + assert_eq!(old_status, "active"); + let other_status: String = conn.query_row("SELECT status FROM memories WHERE text = 'other-agent'", [], |r| r.get(0)).unwrap(); + assert_eq!(other_status, "active"); + let rolled: i64 = conn + .query_row("SELECT COUNT(*) FROM memories WHERE source_agent = 'claude' AND status = ?1", params![ROLLED_BACK_STATUS], |r| r.get(0)) + .unwrap(); + assert_eq!(rolled, 3); // pre-rolled + 2 freshly rolled +} +#[test] +fn idempotent_second_apply() { + let conn = setup(); + seed_session(&conn, "claude", "sess-1", "2026-04-24T00:00:00Z"); + seed_memory(&conn, "claude", "m1", "2026-04-24T00:05:00Z", "active"); + let first = rollback_session_by_id(&conn, "sess-1", true).unwrap(); + assert_eq!(first.memories_affected, 1); + assert!(first.applied); + let second = rollback_session_by_id(&conn, "sess-1", true).unwrap(); + assert_eq!(second.memories_affected, 0); + assert_eq!(second.decisions_affected, 0); + assert!(second.applied); + assert!(second.already_rolled_back); +} +#[test] +fn multi_row_session_chooses_most_recent() { + let conn = setup(); + conn.execute( + "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) + VALUES ('old-agent', 'sess-1', '2026-04-24T00:00:00Z', + '2026-04-24T00:00:00Z', '2026-04-25T00:00:00Z')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO sessions(agent, session_id, started_at, last_heartbeat, expires_at) + VALUES ('new-agent', 'sess-1', '2026-04-24T01:00:00Z', + '2026-04-24T01:00:00Z', '2026-04-25T01:00:00Z')", + [], + ) + .unwrap(); + seed_memory(&conn, "new-agent", "m1", "2026-04-24T01:30:00Z", "active"); + seed_memory(&conn, "old-agent", "o1", "2026-04-24T00:30:00Z", "active"); + let stats = rollback_session_by_id(&conn, "sess-1", false).unwrap(); + assert_eq!(stats.agent, "new-agent"); + assert_eq!(stats.memories_affected, 1); +} diff --git a/daemon-rs/src/aging.rs b/daemon-rs/src/aging.rs index 6f781973..c16271c4 100644 --- a/daemon-rs/src/aging.rs +++ b/daemon-rs/src/aging.rs @@ -1,66 +1,29 @@ -// SPDX-License-Identifier: MIT -//! Progressive Memory Aging — background worker that compresses old memories. -//! -//! Tiers: -//! fresh (0-3 days) — full text, no compression -//! recent (3-14 days) — compressed to key points (1-2 sentences) -//! old (14-60 days) — reduced to a single sentence -//! ancient (60+ days) — archived (status = 'archived'), only explicit search -//! -//! Pinned entries (pinned = 1) are immune to aging. -//! Compression uses extractive summarization (first sentence + key phrases) -//! to avoid depending on an LLM for the background job. - use crate::handlers::feedback; use rusqlite::{params, Connection}; - -/// Age tier boundaries in days. const FRESH_DAYS: i64 = 3; const RECENT_DAYS: i64 = 14; const OLD_DAYS: i64 = 60; - -/// Score below which unretrieved entries are garbage-collected. const GC_SCORE_THRESHOLD: f64 = 0.15; -/// Minimum days since last access before GC kicks in. const GC_MIN_DAYS: i64 = 3; - -/// Run one aging pass over memories and decisions. -/// Returns (compressed_count, archived_count). pub fn run_aging_pass(conn: &Connection) -> (usize, usize) { let mut compressed = 0usize; let mut archived = 0usize; - - // ── Age memories ──────────────────────────────────────────────────────── compressed += age_memories_to_recent(conn); compressed += age_memories_to_old(conn); archived += archive_ancient_memories(conn); - - // ── Age decisions ─────────────────────────────────────────────────────── compressed += age_decisions_to_recent(conn); compressed += age_decisions_to_old(conn); archived += archive_ancient_decisions(conn); - - // ── Score-based garbage collection ────────────────────────────────────── - // Archive entries whose score has decayed below threshold and haven't - // been retrieved in GC_MIN_DAYS. These are noise (test entries, stale - // decisions) that survived time-based aging but lost all relevance. archived += gc_low_score(conn); - - // ── Orphaned embedding cleanup ───────────────────────────────────────── let orphans = cleanup_orphaned_embeddings(conn); if orphans > 0 { eprintln!("[aging] Cleaned {orphans} orphaned embeddings"); } - if compressed > 0 || archived > 0 { eprintln!("[aging] Pass complete: {compressed} compressed, {archived} archived"); } - (compressed, archived) } - -// ─── Memory aging ─────────────────────────────────────────────────────────── - fn age_memories_to_recent(conn: &Connection) -> usize { let mut count = 0; let rows: Vec<(i64, String, Option)> = conn @@ -71,34 +34,23 @@ fn age_memories_to_recent(conn: &Connection) -> usize { AND julianday('now') - julianday(COALESCE(updated_at, created_at)) > ?1", ) .and_then(|mut stmt| { - let mapped = stmt.query_map(params![FRESH_DAYS], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - )) - })?; + let mapped = stmt.query_map(params![FRESH_DAYS], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?)))?; Ok(mapped.flatten().collect()) }) .unwrap_or_default(); - for (id, text, source) in rows { - // Skip aging if this memory has strong retrieval feedback (frequently useful) if let Some(ref src) = source { if feedback::has_retrieval_immunity(conn, src) { continue; } } let compressed = compress_to_key_points(&text); - let _ = conn.execute( - "UPDATE memories SET compressed_text = ?1, age_tier = 'recent', updated_at = datetime('now') WHERE id = ?2", - params![compressed, id], - ); + let _ = + conn.execute("UPDATE memories SET compressed_text = ?1, age_tier = 'recent', updated_at = datetime('now') WHERE id = ?2", params![compressed, id]); count += 1; } count } - fn age_memories_to_old(conn: &Connection) -> usize { let mut count = 0; let rows: Vec<(i64, String, Option)> = conn @@ -109,17 +61,10 @@ fn age_memories_to_old(conn: &Connection) -> usize { AND julianday('now') - julianday(COALESCE(updated_at, created_at)) > ?1", ) .and_then(|mut stmt| { - let mapped = stmt.query_map(params![RECENT_DAYS], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - )) - })?; + let mapped = stmt.query_map(params![RECENT_DAYS], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?)))?; Ok(mapped.flatten().collect()) }) .unwrap_or_default(); - for (id, text, source) in rows { if let Some(ref src) = source { if feedback::has_retrieval_immunity(conn, src) { @@ -127,15 +72,11 @@ fn age_memories_to_old(conn: &Connection) -> usize { } } let compressed = compress_to_one_liner(&text); - let _ = conn.execute( - "UPDATE memories SET compressed_text = ?1, age_tier = 'old', updated_at = datetime('now') WHERE id = ?2", - params![compressed, id], - ); + let _ = conn.execute("UPDATE memories SET compressed_text = ?1, age_tier = 'old', updated_at = datetime('now') WHERE id = ?2", params![compressed, id]); count += 1; } count } - fn archive_ancient_memories(conn: &Connection) -> usize { conn.execute( "UPDATE memories SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ @@ -146,9 +87,6 @@ fn archive_ancient_memories(conn: &Connection) -> usize { ) .unwrap_or(0) } - -// ─── Decision aging ───────────────────────────────────────────────────────── - fn age_decisions_to_recent(conn: &Connection) -> usize { let mut count = 0; let rows: Vec<(i64, String, Option)> = conn @@ -159,32 +97,22 @@ fn age_decisions_to_recent(conn: &Connection) -> usize { AND julianday('now') - julianday(COALESCE(updated_at, created_at)) > ?1", ) .and_then(|mut stmt| { - let mapped = stmt.query_map(params![FRESH_DAYS], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - )) - })?; + let mapped = stmt.query_map(params![FRESH_DAYS], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?)))?; Ok(mapped.flatten().collect()) }) .unwrap_or_default(); - for (id, decision, context) in rows { let full = match context { Some(ref ctx) => format!("{decision} — {ctx}"), None => decision, }; let compressed = compress_to_key_points(&full); - let _ = conn.execute( - "UPDATE decisions SET compressed_text = ?1, age_tier = 'recent', updated_at = datetime('now') WHERE id = ?2", - params![compressed, id], - ); + let _ = + conn.execute("UPDATE decisions SET compressed_text = ?1, age_tier = 'recent', updated_at = datetime('now') WHERE id = ?2", params![compressed, id]); count += 1; } count } - fn age_decisions_to_old(conn: &Connection) -> usize { let mut count = 0; let rows: Vec<(i64, String, Option)> = conn @@ -195,32 +123,22 @@ fn age_decisions_to_old(conn: &Connection) -> usize { AND julianday('now') - julianday(COALESCE(updated_at, created_at)) > ?1", ) .and_then(|mut stmt| { - let mapped = stmt.query_map(params![RECENT_DAYS], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - )) - })?; + let mapped = stmt.query_map(params![RECENT_DAYS], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?)))?; Ok(mapped.flatten().collect()) }) .unwrap_or_default(); - for (id, decision, context) in rows { let full = match context { Some(ref ctx) => format!("{decision} — {ctx}"), None => decision, }; let compressed = compress_to_one_liner(&full); - let _ = conn.execute( - "UPDATE decisions SET compressed_text = ?1, age_tier = 'old', updated_at = datetime('now') WHERE id = ?2", - params![compressed, id], - ); + let _ = + conn.execute("UPDATE decisions SET compressed_text = ?1, age_tier = 'old', updated_at = datetime('now') WHERE id = ?2", params![compressed, id]); count += 1; } count } - fn archive_ancient_decisions(conn: &Connection) -> usize { conn.execute( "UPDATE decisions SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ @@ -231,22 +149,11 @@ fn archive_ancient_decisions(conn: &Connection) -> usize { ) .unwrap_or(0) } - -// ─── Extractive compression ──────────────────────────────────────────────── -// No LLM dependency — pure text extraction. Keeps first sentence and any -// sentences containing high-signal keywords. - fn compress_to_key_points(text: &str) -> String { - let sentences: Vec<&str> = text - .split(['.', '\n']) - .map(|s| s.trim()) - .filter(|s| s.len() > 5) - .collect(); - + let sentences: Vec<&str> = text.split(['.', '\n']).map(|s| s.trim()).filter(|s| s.len() > 5).collect(); if sentences.len() <= 2 { return text.chars().take(300).collect(); } - let high_signal = [ "must", "never", @@ -266,17 +173,14 @@ fn compress_to_key_points(text: &str) -> String { "breaking", "security", ]; - let mut kept: Vec<&str> = Vec::new(); kept.push(sentences[0]); - for sentence in &sentences[1..] { let lower = sentence.to_lowercase(); if high_signal.iter().any(|kw| lower.contains(kw)) && kept.len() < 4 { kept.push(sentence); } } - let result = kept.join(". "); if result.len() > 300 { result.chars().take(300).collect::() + "..." @@ -284,78 +188,56 @@ fn compress_to_key_points(text: &str) -> String { result } } - fn compress_to_one_liner(text: &str) -> String { - let first_sentence = text - .split(['.', '\n']) - .map(|s| s.trim()) - .find(|s| s.len() > 5) - .unwrap_or(text); - + let first_sentence = text.split(['.', '\n']).map(|s| s.trim()).find(|s| s.len() > 5).unwrap_or(text); first_sentence.chars().take(120).collect() } - -// ─── Score-based garbage collection ──────────────────────────────────────── - -/// Archive entries with deeply decayed scores that haven't been retrieved -/// recently. These are noise entries (test data, stale one-offs) that -/// time-based aging hasn't caught yet. fn gc_low_score(conn: &Connection) -> usize { let mut count = 0usize; - - count += conn.execute( - "UPDATE memories SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ + count += conn + .execute( + "UPDATE memories SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ WHERE status = 'active' AND pinned = 0 \ AND score < ?1 \ AND julianday('now') - julianday(COALESCE(last_accessed, created_at)) > ?2", - params![GC_SCORE_THRESHOLD, GC_MIN_DAYS], - ).unwrap_or(0); - - count += conn.execute( - "UPDATE decisions SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ + params![GC_SCORE_THRESHOLD, GC_MIN_DAYS], + ) + .unwrap_or(0); + count += conn + .execute( + "UPDATE decisions SET status = 'archived', age_tier = 'ancient', updated_at = datetime('now') \ WHERE status = 'active' AND pinned = 0 \ AND score < ?1 \ AND julianday('now') - julianday(COALESCE(last_accessed, created_at)) > ?2", - params![GC_SCORE_THRESHOLD, GC_MIN_DAYS], - ).unwrap_or(0); - + params![GC_SCORE_THRESHOLD, GC_MIN_DAYS], + ) + .unwrap_or(0); if count > 0 { eprintln!("[aging] GC archived {count} low-score entries (score < {GC_SCORE_THRESHOLD})"); } count } - -/// Remove embeddings for entries that are no longer active. -/// These accumulate when entries are superseded or archived. fn cleanup_orphaned_embeddings(conn: &Connection) -> usize { let mut count = 0usize; - - count += conn.execute( - "DELETE FROM embeddings WHERE target_type = 'memory' \ + count += conn + .execute( + "DELETE FROM embeddings WHERE target_type = 'memory' \ AND NOT EXISTS (SELECT 1 FROM memories m WHERE m.id = embeddings.target_id AND m.status = 'active')", - [], - ).unwrap_or(0); - - count += conn.execute( - "DELETE FROM embeddings WHERE target_type = 'decision' \ + [], + ) + .unwrap_or(0); + count += conn + .execute( + "DELETE FROM embeddings WHERE target_type = 'decision' \ AND NOT EXISTS (SELECT 1 FROM decisions d WHERE d.id = embeddings.target_id AND d.status = 'active')", - [], - ).unwrap_or(0); - + [], + ) + .unwrap_or(0); count } - -// ─── Retrieval-time helper ────────────────────────────────────────────────── - -/// Get the best available text for a memory: compressed if aged, full if fresh. -/// Called by recall to serve age-appropriate content. pub fn get_display_text(text: &str, compressed_text: &Option, age_tier: &str) -> String { match age_tier { "fresh" => text.to_string(), - _ => compressed_text - .as_ref() - .filter(|c| !c.is_empty()) - .cloned() - .unwrap_or_else(|| text.to_string()), + _ => compressed_text.as_ref().filter(|c| !c.is_empty()).cloned().unwrap_or_else(|| text.to_string()), } } diff --git a/daemon-rs/src/api_types.rs b/daemon-rs/src/api_types.rs index 45d14729..39498a3b 100644 --- a/daemon-rs/src/api_types.rs +++ b/daemon-rs/src/api_types.rs @@ -1,23 +1,10 @@ -// SPDX-License-Identifier: MIT use serde::{Deserialize, Serialize}; - #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "lowercase")] pub enum ExportFormat { Json, Sql, } - -impl ExportFormat { - pub fn parse(input: &str) -> Option { - match input.trim().to_ascii_lowercase().as_str() { - "json" => Some(Self::Json), - "sql" => Some(Self::Sql), - _ => None, - } - } -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum RetentionClass { @@ -27,7 +14,6 @@ pub enum RetentionClass { Audit, Ephemeral, } - impl RetentionClass { pub fn as_str(self) -> &'static str { match self { @@ -37,7 +23,6 @@ impl RetentionClass { Self::Ephemeral => "ephemeral", } } - pub fn parse(input: &str) -> Option { match input.trim().to_ascii_lowercase().as_str() { "durable" => Some(Self::Durable), @@ -47,7 +32,6 @@ impl RetentionClass { _ => None, } } - pub fn default_ttl_seconds(self) -> Option { match self { Self::Durable => None, @@ -56,73 +40,41 @@ impl RetentionClass { Self::Ephemeral => Some(14 * 24 * 60 * 60), } } - pub fn from_entry_type(entry_type: &str) -> Option { match entry_type.trim().to_ascii_lowercase().as_str() { - "decision" | "policy" | "rule" | "convention" | "contract" | "procedure" - | "playbook" | "runbook" => Some(Self::Durable), + "decision" | "policy" | "rule" | "convention" | "contract" | "procedure" | "playbook" | "runbook" => Some(Self::Durable), "trace" | "security" | "rollback" | "permission" | "audit" => Some(Self::Audit), - "chatter" | "scratch" | "transient" | "temporary" | "ephemeral" => { - Some(Self::Ephemeral) - } - "observation" | "note" | "finding" | "fact" | "memory" | "focus_summary" => { - Some(Self::Operational) - } + "chatter" | "scratch" | "transient" | "temporary" | "ephemeral" => Some(Self::Ephemeral), + "observation" | "note" | "finding" | "fact" | "memory" | "focus_summary" => Some(Self::Operational), _ => None, } } - - pub fn classify( - explicit: Option, - entry_type: &str, - text: &str, - context: Option<&str>, - ) -> Self { + pub fn classify(explicit: Option, entry_type: &str, text: &str, context: Option<&str>) -> Self { if let Some(explicit) = explicit { return explicit; } if let Some(mapped) = Self::from_entry_type(entry_type) { return mapped; } - let combined = match context { - Some(context) if !context.trim().is_empty() => { - format!("{} {}", text.trim(), context.trim()).to_ascii_lowercase() - } + Some(context) if !context.trim().is_empty() => format!("{} {}", text.trim(), context.trim()).to_ascii_lowercase(), _ => text.trim().to_ascii_lowercase(), }; - if [ - "architectural", - "architecture", - "convention", - "always", - "never", - "api contract", - "must ", - "do not", - ] - .iter() - .any(|needle| combined.contains(needle)) - { - return Self::Durable; - } - if ["rollback", "permission", "security event", "audit"] + if ["architectural", "architecture", "convention", "always", "never", "api contract", "must ", "do not"] .iter() .any(|needle| combined.contains(needle)) { + return Self::Durable; + } + if ["rollback", "permission", "security event", "audit"].iter().any(|needle| combined.contains(needle)) { return Self::Audit; } - if ["throwaway", "temporary", "transient", "scratch"] - .iter() - .any(|needle| combined.contains(needle)) - { + if ["throwaway", "temporary", "transient", "scratch"].iter().any(|needle| combined.contains(needle)) { return Self::Ephemeral; } - Self::Operational } } - #[derive(Debug, Clone, Default, Deserialize)] pub struct StoreRequest { pub decision: Option, @@ -136,13 +88,11 @@ pub struct StoreRequest { pub ttl_seconds: Option, pub retention_class: Option, } - #[derive(Debug, Clone, Deserialize)] pub struct ImportPayload { pub memories: Option>, pub decisions: Option>, } - #[derive(Debug, Clone, Deserialize)] pub struct ImportMemory { pub text: String, @@ -162,7 +112,6 @@ pub struct ImportMemory { pub valid_until: Option, pub retention_class: Option, } - #[derive(Debug, Clone, Deserialize)] pub struct ImportDecision { pub decision: String, @@ -181,24 +130,17 @@ pub struct ImportDecision { pub valid_until: Option, pub retention_class: Option, } - #[derive(Debug, Clone)] pub struct ImportOptions { pub owner_id: Option, pub visibility: Option, pub source_agent_fallback: String, } - impl Default for ImportOptions { fn default() -> Self { - Self { - owner_id: None, - visibility: None, - source_agent_fallback: "import".to_string(), - } + Self { owner_id: None, visibility: None, source_agent_fallback: "import".to_string() } } } - #[derive(Debug, Clone, Copy, Default)] pub struct ImportCounts { pub memories: usize, diff --git a/daemon-rs/src/auth/keys.rs b/daemon-rs/src/auth/keys.rs index afb2ae5f..696066de 100644 --- a/daemon-rs/src/auth/keys.rs +++ b/daemon-rs/src/auth/keys.rs @@ -1,14 +1,10 @@ -// SPDX-License-Identifier: MIT +use super::paths::{default_home_root, write_secret_file, CortexPaths, CORTEX_DIR_NAME}; +use super::runtime::{base62_encode_bytes, fnv1a16, left_pad_base62}; use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString}; use argon2::{Algorithm, Argon2, Params, Version}; use std::fs; use std::path::PathBuf; use uuid::Uuid; - -use super::paths::{default_home_root, write_secret_file, CortexPaths, CORTEX_DIR_NAME}; -use super::runtime::{base62_encode_bytes, base62_encode_u64, fnv1a16, left_pad_base62}; - -/// Returns `~/.cortex` (or `$HOME/.cortex` on non-Windows). pub fn cortex_dir() -> PathBuf { if let Ok(explicit) = std::env::var("CORTEX_HOME") { if !explicit.trim().is_empty() { @@ -17,74 +13,45 @@ pub fn cortex_dir() -> PathBuf { } default_home_root().join(CORTEX_DIR_NAME) } - -/// Generate a fresh UUID token, write it to the resolved token path, and -/// return the token string. pub fn try_generate_token_for(paths: &CortexPaths) -> Result { let token = Uuid::new_v4().simple().to_string(); try_write_token_for(paths, &token)?; Ok(token) } - -/// Write a shared auth token to the resolved token path. pub fn try_write_token_for(paths: &CortexPaths, token: &str) -> Result<(), String> { let token_dir = paths.token.parent().unwrap_or(&paths.home); - fs::create_dir_all(token_dir) - .map_err(|e| format!("cannot create token directory {}: {e}", token_dir.display()))?; - write_secret_file(&paths.token, token.as_bytes()) - .map_err(|e| format!("cannot write token file {}: {e}", paths.token.display()))?; + fs::create_dir_all(token_dir).map_err(|e| format!("cannot create token directory {}: {e}", token_dir.display()))?; + write_secret_file(&paths.token, token.as_bytes()).map_err(|e| format!("cannot write token file {}: {e}", paths.token.display()))?; Ok(()) } - -/// Generate a fresh UUID token for the resolved token path. pub fn try_generate_token() -> Result { try_generate_token_for(&CortexPaths::resolve()) } - -/// Read an existing token from the resolved token path. pub fn read_token_from(paths: &CortexPaths) -> Option { - fs::read_to_string(&paths.token) - .ok() - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) + fs::read_to_string(&paths.token).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) } - -/// Read existing shared token from `~/.cortex/cortex.token`. pub fn read_token() -> Option { read_token_from(&CortexPaths::resolve()) } - -/// Generate an in-memory token without mutating shared auth files. pub fn generate_ephemeral_token() -> String { Uuid::new_v4().simple().to_string() } - -/// Generate a `ctx_` API key: -/// - body: base62-encoded random bytes (43 chars) -/// - checksum: 16-bit FNV-1a over the body, base62 (3 chars, left-padded) pub fn generate_ctx_api_key() -> String { let mut random = Vec::with_capacity(32); random.extend_from_slice(Uuid::new_v4().as_bytes()); random.extend_from_slice(Uuid::new_v4().as_bytes()); - let mut body = base62_encode_bytes(&random); if body.len() < 43 { - // Extremely unlikely, but keep a stable key shape. let extra = base62_encode_bytes(Uuid::new_v4().as_bytes()); body.push_str(&extra); } body.truncate(43); - let checksum_num = fnv1a16(body.as_bytes()); let checksum = left_pad_base62(checksum_num, 3); - format!("ctx_{body}{checksum}") } - const CTX_KEY_BODY_LEN: usize = 43; const CTX_KEY_CHECKSUM_LEN: usize = 3; - -/// Cheap structural validation for `ctx_` API keys before Argon2 verification. pub fn verify_ctx_api_key_checksum(candidate: &str) -> bool { if !candidate.starts_with("ctx_") { return false; @@ -93,52 +60,35 @@ pub fn verify_ctx_api_key_checksum(candidate: &str) -> bool { if payload.len() != CTX_KEY_BODY_LEN + CTX_KEY_CHECKSUM_LEN { return false; } - if !payload - .as_bytes() - .iter() - .all(|byte| byte.is_ascii_alphanumeric()) - { + if !payload.as_bytes().iter().all(|byte| byte.is_ascii_alphanumeric()) { return false; } - let (body, checksum) = payload.split_at(CTX_KEY_BODY_LEN); let expected = left_pad_base62(fnv1a16(body.as_bytes()), CTX_KEY_CHECKSUM_LEN); constant_time_eq(checksum, expected.as_str()) } - fn constant_time_eq(a: &str, b: &str) -> bool { let a = a.as_bytes(); let b = b.as_bytes(); let mut diff = a.len() ^ b.len(); let max_len = a.len().max(b.len()); - for idx in 0..max_len { let left = a.get(idx).copied().unwrap_or(0); let right = b.get(idx).copied().unwrap_or(0); diff |= usize::from(left ^ right); } - diff == 0 } - -/// Hash an API key with Argon2id. pub fn hash_api_key_argon2id(api_key: &str) -> Result { let params = Params::new(64 * 1024, 3, 4, None).map_err(|e| e.to_string())?; let argon2 = Argon2::new(Algorithm::Argon2id, Version::V0x13, params); let salt = SaltString::encode_b64(Uuid::new_v4().as_bytes()).map_err(|e| e.to_string())?; - argon2 - .hash_password(api_key.as_bytes(), &salt) - .map(|p| p.to_string()) - .map_err(|e| e.to_string()) + argon2.hash_password(api_key.as_bytes(), &salt).map(|p| p.to_string()).map_err(|e| e.to_string()) } - -/// Verify a plaintext API key against an Argon2id hash. pub fn verify_api_key_argon2id(api_key: &str, hash: &str) -> bool { let parsed = match PasswordHash::new(hash) { Ok(v) => v, Err(_) => return false, }; - Argon2::default() - .verify_password(api_key.as_bytes(), &parsed) - .is_ok() + Argon2::default().verify_password(api_key.as_bytes(), &parsed).is_ok() } diff --git a/daemon-rs/src/auth/locks.rs b/daemon-rs/src/auth/locks.rs index 07ef1ef1..c072bf0a 100644 --- a/daemon-rs/src/auth/locks.rs +++ b/daemon-rs/src/auth/locks.rs @@ -1,15 +1,7 @@ -// SPDX-License-Identifier: MIT +use super::paths::{default_home_root, CortexPaths, CORTEX_DIR_NAME, CORTEX_GLOBAL_LOCK_HOME_ENV, CORTEX_GLOBAL_LOCK_NAME}; use fs2::FileExt; use std::fs; use std::path::{Path, PathBuf}; - -use super::paths::{ - default_home_root, CortexPaths, CORTEX_DIR_NAME, CORTEX_GLOBAL_LOCK_HOME_ENV, - CORTEX_GLOBAL_LOCK_NAME, -}; - -/// Acquire an exclusive file lock on `~/.cortex/cortex.lock`. -/// Returns the lock file handle (lock is held as long as the handle lives). pub fn acquire_daemon_lock(paths: &CortexPaths) -> Result { fs::create_dir_all(&paths.home).map_err(|e| format!("create home: {e}"))?; let lock_file = fs::OpenOptions::new() @@ -18,12 +10,9 @@ pub fn acquire_daemon_lock(paths: &CortexPaths) -> Result { .truncate(false) .open(&paths.lock) .map_err(|e| format!("open lock: {e}"))?; - lock_file - .try_lock_exclusive() - .map_err(|_| "another cortex instance holds the lock".to_string())?; + lock_file.try_lock_exclusive().map_err(|_| "another cortex instance holds the lock".to_string())?; Ok(lock_file) } - fn global_lock_path() -> PathBuf { if let Ok(explicit) = std::env::var(CORTEX_GLOBAL_LOCK_HOME_ENV) { let trimmed = explicit.trim(); @@ -31,13 +20,8 @@ fn global_lock_path() -> PathBuf { return PathBuf::from(trimmed).join(CORTEX_GLOBAL_LOCK_NAME); } } - default_home_root() - .join(CORTEX_DIR_NAME) - .join(CORTEX_GLOBAL_LOCK_NAME) + default_home_root().join(CORTEX_DIR_NAME).join(CORTEX_GLOBAL_LOCK_NAME) } - -/// Acquire an exclusive global daemon lock to enforce one active Cortex daemon -/// per user, even when different homes/binaries are involved. fn acquire_global_daemon_lock_at(path: &Path) -> Result { if let Some(parent) = path.parent() { fs::create_dir_all(parent).map_err(|e| format!("create global lock dir: {e}"))?; @@ -48,12 +32,9 @@ fn acquire_global_daemon_lock_at(path: &Path) -> Result { .truncate(false) .open(path) .map_err(|e| format!("open global lock: {e}"))?; - lock_file - .try_lock_exclusive() - .map_err(|_| "another cortex instance holds the lock".to_string())?; + lock_file.try_lock_exclusive().map_err(|_| "another cortex instance holds the lock".to_string())?; Ok(lock_file) } - pub fn acquire_global_daemon_lock() -> Result { acquire_global_daemon_lock_at(&global_lock_path()) } diff --git a/daemon-rs/src/auth/migration.rs b/daemon-rs/src/auth/migration.rs index 426e6bee..d2e62b9a 100644 --- a/daemon-rs/src/auth/migration.rs +++ b/daemon-rs/src/auth/migration.rs @@ -1,31 +1,17 @@ -// SPDX-License-Identifier: MIT +use super::paths::CortexPaths; use std::fs; use std::path::PathBuf; - -use super::paths::CortexPaths; - -/// Returns the legacy database path: `~/cortex/cortex.db`. pub fn legacy_db_path() -> PathBuf { - let home = std::env::var("USERPROFILE") - .or_else(|_| std::env::var("HOME")) - .unwrap_or_else(|_| ".".to_string()); + let home = std::env::var("USERPROFILE").or_else(|_| std::env::var("HOME")).unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("cortex").join("cortex.db") } - -/// Migrate legacy DB from `~/cortex/cortex.db` to the canonical location. -/// Copies (never moves) to preserve the original as a safety net. pub fn migrate_legacy_db(paths: &CortexPaths) -> Result { let legacy = legacy_db_path(); if !legacy.exists() || paths.db.exists() { return Ok(false); } - - fs::create_dir_all(paths.db.parent().unwrap_or(&paths.home)) - .map_err(|e| format!("create dir: {e}"))?; - + fs::create_dir_all(paths.db.parent().unwrap_or(&paths.home)).map_err(|e| format!("create dir: {e}"))?; fs::copy(&legacy, &paths.db).map_err(|e| format!("copy db: {e}"))?; - - // Copy WAL and SHM if present for ext in ["db-wal", "db-shm"] { let src = legacy.with_extension(ext); if src.exists() { @@ -33,26 +19,15 @@ pub fn migrate_legacy_db(paths: &CortexPaths) -> Result { fs::copy(&src, &dst).map_err(|e| format!("copy {ext}: {e}"))?; } } - - // Verify integrity of the copy - let conn = - rusqlite::Connection::open(&paths.db).map_err(|e| format!("open migrated db: {e}"))?; + let conn = rusqlite::Connection::open(&paths.db).map_err(|e| format!("open migrated db: {e}"))?; let busy_timeout_ms = crate::db::SQLITE_BUSY_TIMEOUT_MS; conn.execute_batch(&format!("PRAGMA busy_timeout = {busy_timeout_ms};")) .map_err(|e| format!("configure migrated db busy timeout: {e}"))?; - let check: String = conn - .query_row("PRAGMA integrity_check", [], |row| row.get(0)) - .map_err(|e| format!("integrity check: {e}"))?; + let check: String = conn.query_row("PRAGMA integrity_check", [], |row| row.get(0)).map_err(|e| format!("integrity check: {e}"))?; if check != "ok" { - // Remove the bad copy, leave legacy intact let _ = fs::remove_file(&paths.db); return Err(format!("integrity check failed on migrated db: {check}")); } - - eprintln!( - "[cortex] Migrated brain from {} to {}", - legacy.display(), - paths.db.display() - ); + eprintln!("[cortex] Migrated brain from {} to {}", legacy.display(), paths.db.display()); Ok(true) } diff --git a/daemon-rs/src/auth/mod.rs b/daemon-rs/src/auth/mod.rs index 72bdc231..173621b3 100644 --- a/daemon-rs/src/auth/mod.rs +++ b/daemon-rs/src/auth/mod.rs @@ -1,21 +1,16 @@ -// SPDX-License-Identifier: MIT -mod paths; -mod migration; -mod locks; mod keys; +mod locks; +mod migration; +mod paths; mod runtime; - #[cfg(test)] mod tests; - -pub use paths::CortexPaths; -pub use migration::{legacy_db_path, migrate_legacy_db}; -pub use locks::{acquire_daemon_lock, acquire_global_daemon_lock}; pub use keys::{ - cortex_dir, generate_ctx_api_key, generate_ephemeral_token, hash_api_key_argon2id, read_token, - read_token_from, try_generate_token, try_generate_token_for, try_write_token_for, - verify_api_key_argon2id, verify_ctx_api_key_checksum, + cortex_dir, generate_ctx_api_key, generate_ephemeral_token, hash_api_key_argon2id, read_token, read_token_from, try_generate_token, try_generate_token_for, + try_write_token_for, verify_api_key_argon2id, verify_ctx_api_key_checksum, }; -pub use runtime::{cleanup_stale_pid_lock, db_path, stale_pid_candidate, write_pid}; - +pub use locks::{acquire_daemon_lock, acquire_global_daemon_lock}; +pub use migration::migrate_legacy_db; +pub use paths::CortexPaths; pub(crate) use paths::{restrict_file_to_owner, write_secret_file}; +pub use runtime::{cleanup_stale_pid_lock, db_path}; diff --git a/daemon-rs/src/auth/paths.rs b/daemon-rs/src/auth/paths.rs index 6d90942c..b9bd7127 100644 --- a/daemon-rs/src/auth/paths.rs +++ b/daemon-rs/src/auth/paths.rs @@ -1,20 +1,11 @@ -// SPDX-License-Identifier: MIT use std::fs; #[cfg(windows)] use std::io; use std::path::{Path, PathBuf}; - pub(crate) const CORTEX_DIR_NAME: &str = ".cortex"; pub(crate) const CORTEX_GLOBAL_LOCK_NAME: &str = "cortex.global.lock"; pub(crate) const CORTEX_GLOBAL_LOCK_HOME_ENV: &str = "CORTEX_GLOBAL_LOCK_HOME"; pub(crate) const BASE62: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - -// --------------------------------------------------------------------------- -// CortexPaths -- centralized path + port resolver -// --------------------------------------------------------------------------- - -/// Resolved paths for all Cortex runtime files. -/// Priority: CLI flag > env var > default. #[derive(Debug, Clone)] pub struct CortexPaths { pub home: PathBuf, @@ -29,41 +20,25 @@ pub struct CortexPaths { #[allow(dead_code)] pub write_buffer: PathBuf, } - impl CortexPaths { - /// Resolve paths from environment variables only (no CLI args). pub fn resolve() -> Self { Self::resolve_with_overrides(None, None, None, None) } - - /// Resolve paths with optional CLI overrides. - pub fn resolve_with_overrides( - home_override: Option<&str>, - db_override: Option<&str>, - port_override: Option, - bind_override: Option<&str>, - ) -> Self { + pub fn resolve_with_overrides(home_override: Option<&str>, db_override: Option<&str>, port_override: Option, bind_override: Option<&str>) -> Self { let home = home_override .map(PathBuf::from) .or_else(|| std::env::var("CORTEX_HOME").ok().map(PathBuf::from)) .unwrap_or_else(|| default_home_root().join(CORTEX_DIR_NAME)); - let db = db_override .map(PathBuf::from) .or_else(|| std::env::var("CORTEX_DB").ok().map(PathBuf::from)) .unwrap_or_else(|| home.join("cortex.db")); - let port = port_override - .or_else(|| { - std::env::var("CORTEX_PORT") - .ok() - .and_then(|s| s.parse().ok()) - }) + .or_else(|| std::env::var("CORTEX_PORT").ok().and_then(|s| s.parse().ok())) .unwrap_or(crate::DEFAULT_CORTEX_PORT); let env_bind = std::env::var("CORTEX_BIND").ok(); let bind = resolve_bind(bind_override, env_bind.as_deref()); let ipc_endpoint = resolve_ipc_endpoint(&home, port); - Self { token: home.join("cortex.token"), pid: home.join("cortex.pid"), @@ -77,8 +52,6 @@ impl CortexPaths { bind, } } - - /// Parse --home, --db, --port, --bind flags from CLI args. pub fn resolve_from_args(args: &[String]) -> Self { let home = Self::find_flag(args, "--home"); let db = Self::find_flag(args, "--db"); @@ -86,35 +59,17 @@ impl CortexPaths { let bind = Self::find_flag(args, "--bind"); Self::resolve_with_overrides(home.as_deref(), db.as_deref(), port, bind.as_deref()) } - fn find_flag(args: &[String], flag: &str) -> Option { - args.iter() - .position(|a| a == flag) - .and_then(|i| args.get(i + 1)) - .cloned() + args.iter().position(|a| a == flag).and_then(|i| args.get(i + 1)).cloned() } - - /// Serialize to JSON for `cortex paths --json`. pub fn to_json(&self) -> String { - serde_json::json!({ - "home": self.home.display().to_string(), - "db": self.db.display().to_string(), - "token": self.token.display().to_string(), - "pid": self.pid.display().to_string(), - "port": self.port, - "bind": &self.bind, - "ipc_endpoint": self.ipc_endpoint.clone(), - "ipc_kind": if self.ipc_endpoint.is_some() { - Some(default_ipc_kind()) - } else { - None - }, - "models": self.models.display().to_string(), - }) + serde_json::json!({"home":self.home.display().to_string(),"db":self.db. +display().to_string(),"token":self.token.display().to_string(),"pid":self.pid.display().to_string(),"port":self.port,"bind":&self. +bind,"ipc_endpoint":self.ipc_endpoint.clone(),"ipc_kind":if self.ipc_endpoint.is_some(){Some(default_ipc_kind())}else{None}, +"models":self.models.display().to_string(),}) .to_string() } } - fn normalize_bind(value: &str) -> Option { let trimmed = value.trim(); if trimmed.is_empty() { @@ -123,21 +78,18 @@ fn normalize_bind(value: &str) -> Option { Some(trimmed.to_string()) } } - fn resolve_bind(bind_override: Option<&str>, env_bind: Option<&str>) -> String { bind_override .and_then(normalize_bind) .or_else(|| env_bind.and_then(normalize_bind)) .unwrap_or_else(|| "127.0.0.1".to_string()) } - pub(crate) fn default_home_root() -> PathBuf { std::env::var("USERPROFILE") .or_else(|_| std::env::var("HOME")) .map(PathBuf::from) .unwrap_or_else(|_| PathBuf::from(".")) } - fn default_ipc_kind() -> &'static str { if cfg!(windows) { "named-pipe" @@ -145,180 +97,118 @@ fn default_ipc_kind() -> &'static str { "unix-socket" } } - fn env_truthy(key: &str) -> bool { - std::env::var(key).ok().is_some_and(|value| { - matches!( - value.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ) - }) + std::env::var(key) + .ok() + .is_some_and(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) } - fn resolve_ipc_endpoint(home: &std::path::Path, port: u16) -> Option { if env_truthy("CORTEX_DISABLE_IPC") { return None; } - if let Ok(raw) = std::env::var("CORTEX_IPC_ENDPOINT") { let trimmed = raw.trim(); if !trimmed.is_empty() { return Some(trimmed.to_string()); } } - if cfg!(windows) { return Some(format!(r"\\.\pipe\cortex-daemon-{port}")); } - let socket = home.join("runtime").join(format!("cortexd-{port}.sock")); Some(socket.display().to_string()) } - #[cfg(unix)] pub(crate) fn restrict_file_to_owner(path: &Path) -> std::io::Result<()> { use std::os::unix::fs::PermissionsExt; - fs::set_permissions(path, fs::Permissions::from_mode(0o600)) } - #[cfg(windows)] struct OwnedHandle(windows_sys::Win32::Foundation::HANDLE); - #[cfg(windows)] impl Drop for OwnedHandle { fn drop(&mut self) { if !self.0.is_null() { - // SAFETY: this guard owns the token handle returned by OpenProcessToken. unsafe { let _ = windows_sys::Win32::Foundation::CloseHandle(self.0); } } } } - #[cfg(windows)] struct LocalMemory(*mut std::ffi::c_void); - #[cfg(windows)] impl Drop for LocalMemory { fn drop(&mut self) { if !self.0.is_null() { - // SAFETY: this guard owns memory allocated by a Win32 local allocator. unsafe { let _ = windows_sys::Win32::Foundation::LocalFree(self.0); } } } } - #[cfg(windows)] struct CurrentUserSid { _token_info: Vec, sid: windows_sys::Win32::Security::PSID, } - #[cfg(windows)] fn windows_path_to_wide(path: &Path) -> Vec { use std::os::windows::ffi::OsStrExt; - path.as_os_str().encode_wide().chain([0]).collect() } - #[cfg(windows)] fn win32_error(code: u32) -> io::Error { io::Error::from_raw_os_error(code as i32) } - #[cfg(windows)] fn current_user_sid() -> io::Result { use std::ptr::null_mut; use windows_sys::Win32::Foundation::HANDLE; - use windows_sys::Win32::Security::{ - GetTokenInformation, IsValidSid, TokenUser, TOKEN_QUERY, TOKEN_USER, - }; + use windows_sys::Win32::Security::{GetTokenInformation, IsValidSid, TokenUser, TOKEN_QUERY, TOKEN_USER}; use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; - let mut token: HANDLE = null_mut(); - // SAFETY: GetCurrentProcess returns a pseudo-handle for this process, and - // OpenProcessToken initializes `token` on success. let opened = unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) }; if opened == 0 { return Err(io::Error::last_os_error()); } let token = OwnedHandle(token); - let mut required_len = 0u32; - // SAFETY: this size query intentionally passes a null output buffer and - // zero length so Windows reports the required TOKEN_USER buffer size. unsafe { let _ = GetTokenInformation(token.0, TokenUser, null_mut(), 0, &mut required_len); } if required_len == 0 { return Err(io::Error::last_os_error()); } - let word_size = std::mem::size_of::(); let word_count = (required_len as usize).div_ceil(word_size); let mut token_info = vec![0usize; word_count]; let mut returned_len = 0u32; - // SAFETY: `token_info` is a writable, word-aligned buffer large enough for - // the TOKEN_USER data size returned by the previous GetTokenInformation call. - let filled = unsafe { - GetTokenInformation( - token.0, - TokenUser, - token_info.as_mut_ptr().cast(), - (token_info.len() * word_size) as u32, - &mut returned_len, - ) - }; + let filled = unsafe { GetTokenInformation(token.0, TokenUser, token_info.as_mut_ptr().cast(), (token_info.len() * word_size) as u32, &mut returned_len) }; if filled == 0 { return Err(io::Error::last_os_error()); } if returned_len < std::mem::size_of::() as u32 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Windows token user information is too small", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "Windows token user information is too small")); } - - // SAFETY: the buffer was populated by GetTokenInformation(TokenUser) and - // is word-aligned, so reading the leading TOKEN_USER record is valid. let token_user = unsafe { *token_info.as_ptr().cast::() }; if token_user.User.Sid.is_null() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Windows token user SID is missing", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "Windows token user SID is missing")); } - // SAFETY: token_user.User.Sid came from the validated TOKEN_USER buffer. if unsafe { IsValidSid(token_user.User.Sid) } == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Windows token user SID is invalid", - )); + return Err(io::Error::new(io::ErrorKind::InvalidData, "Windows token user SID is invalid")); } - - Ok(CurrentUserSid { - _token_info: token_info, - sid: token_user.User.Sid, - }) + Ok(CurrentUserSid { _token_info: token_info, sid: token_user.User.Sid }) } - #[cfg(windows)] pub(crate) fn restrict_file_to_owner(path: &Path) -> io::Result<()> { use std::ptr::{null, null_mut}; use windows_sys::Win32::Foundation::ERROR_SUCCESS; use windows_sys::Win32::Security::Authorization::{ - SetEntriesInAclW, SetNamedSecurityInfoW, EXPLICIT_ACCESS_W, NO_MULTIPLE_TRUSTEE, - SET_ACCESS, SE_FILE_OBJECT, TRUSTEE_IS_SID, TRUSTEE_IS_USER, TRUSTEE_W, - }; - use windows_sys::Win32::Security::{ - ACL, DACL_SECURITY_INFORMATION, NO_INHERITANCE, PROTECTED_DACL_SECURITY_INFORMATION, + SetEntriesInAclW, SetNamedSecurityInfoW, EXPLICIT_ACCESS_W, NO_MULTIPLE_TRUSTEE, SET_ACCESS, SE_FILE_OBJECT, TRUSTEE_IS_SID, TRUSTEE_IS_USER, TRUSTEE_W, }; + use windows_sys::Win32::Security::{ACL, DACL_SECURITY_INFORMATION, NO_INHERITANCE, PROTECTED_DACL_SECURITY_INFORMATION}; use windows_sys::Win32::Storage::FileSystem::FILE_ALL_ACCESS; - let current_user = current_user_sid()?; let access = EXPLICIT_ACCESS_W { grfAccessPermissions: FILE_ALL_ACCESS, @@ -333,17 +223,12 @@ pub(crate) fn restrict_file_to_owner(path: &Path) -> io::Result<()> { }, }; let mut acl: *mut ACL = null_mut(); - // SAFETY: `access` references the current-user SID buffer, which remains - // alive for this call; `acl` receives LocalFree-owned memory on success. let result = unsafe { SetEntriesInAclW(1, &access, null(), &mut acl) }; if result != ERROR_SUCCESS { return Err(win32_error(result)); } let _acl_guard = LocalMemory(acl.cast()); - let wide_path = windows_path_to_wide(path); - // SAFETY: `wide_path` is null-terminated, `acl` is a valid ACL produced by - // SetEntriesInAclW, and null owner/group/SACL preserve those fields. let result = unsafe { SetNamedSecurityInfoW( wide_path.as_ptr(), @@ -358,44 +243,27 @@ pub(crate) fn restrict_file_to_owner(path: &Path) -> io::Result<()> { if result != ERROR_SUCCESS { return Err(win32_error(result)); } - Ok(()) } - #[cfg(not(any(unix, windows)))] pub(crate) fn restrict_file_to_owner(_path: &Path) -> std::io::Result<()> { Ok(()) } - pub(crate) fn write_secret_file(path: &Path, contents: &[u8]) -> std::io::Result<()> { #[cfg(unix)] { use std::io::Write as _; use std::os::unix::fs::OpenOptionsExt; - - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .mode(0o600) - .open(path)?; + let mut file = fs::OpenOptions::new().create(true).write(true).truncate(true).mode(0o600).open(path)?; file.write_all(contents)?; file.flush()?; restrict_file_to_owner(path)?; return Ok(()); } - #[cfg(not(unix))] { use std::io::Write as _; - - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - restrict_file_to_owner(path) - .and_then(|_| file.write_all(contents)) - .and_then(|_| file.flush()) + let mut file = fs::OpenOptions::new().create(true).write(true).truncate(true).open(path)?; + restrict_file_to_owner(path).and_then(|_| file.write_all(contents)).and_then(|_| file.flush()) } } diff --git a/daemon-rs/src/auth/runtime.rs b/daemon-rs/src/auth/runtime.rs index 4d87a38f..fe8698a2 100644 --- a/daemon-rs/src/auth/runtime.rs +++ b/daemon-rs/src/auth/runtime.rs @@ -1,11 +1,7 @@ -// SPDX-License-Identifier: MIT +use super::keys::cortex_dir; +use super::paths::{CortexPaths, BASE62}; use std::fs; use std::path::PathBuf; - -use super::keys::cortex_dir; -use super::paths::{BASE62, CortexPaths}; - -/// Write the current process PID to `~/.cortex/cortex.pid`. #[allow(dead_code)] pub fn write_pid() { let dir = cortex_dir(); @@ -14,39 +10,26 @@ pub fn write_pid() { } fs::write(dir.join("cortex.pid"), std::process::id().to_string()).ok(); } - -/// Remove stale PID file when the recorded daemon process no longer exists. pub fn cleanup_stale_pid_lock(paths: &CortexPaths) -> Option { let pid = stale_pid_candidate(paths)?; - let _ = fs::remove_file(&paths.pid); eprintln!("[cortex] Cleaned stale PID file (process {pid} not running)"); Some(pid) } - pub fn stale_pid_candidate(paths: &CortexPaths) -> Option { if !paths.pid.exists() { return None; } - - let pid = fs::read_to_string(&paths.pid) - .ok() - .and_then(|value| value.trim().parse::().ok())?; - + let pid = fs::read_to_string(&paths.pid).ok().and_then(|value| value.trim().parse::().ok())?; if pid == std::process::id() || process_is_running(pid) { return None; } - Some(pid) } - #[cfg(windows)] fn process_is_running(pid: u32) -> bool { use std::process::Command; - - let output = Command::new("tasklist") - .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) - .output(); + let output = Command::new("tasklist").args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]).output(); let Ok(out) = output else { return false; }; @@ -56,23 +39,16 @@ fn process_is_running(pid: u32) -> bool { let stdout = String::from_utf8_lossy(&out.stdout); stdout.contains(&format!("\"{pid}\"")) } - #[cfg(unix)] fn process_is_running(pid: u32) -> bool { let Ok(pid) = libc::pid_t::try_from(pid) else { return false; }; - // SAFETY: `pid` has been range-checked for the platform `pid_t`. - // Passing signal 0 performs an existence/permission probe and does not - // deliver a signal. unsafe { libc::kill(pid, 0) == 0 } } - -/// Returns the canonical database path: `~/.cortex/cortex.db`. pub fn db_path() -> PathBuf { cortex_dir().join("cortex.db") } - pub(crate) fn fnv1a16(input: &[u8]) -> u16 { let mut hash: u32 = 0x811C9DC5; for byte in input { @@ -81,7 +57,6 @@ pub(crate) fn fnv1a16(input: &[u8]) -> u16 { } (hash & 0xFFFF) as u16 } - pub(crate) fn left_pad_base62(num: u16, width: usize) -> String { let mut s = base62_encode_u64(num as u64); while s.len() < width { @@ -89,7 +64,6 @@ pub(crate) fn left_pad_base62(num: u16, width: usize) -> String { } s } - pub(crate) fn base62_encode_u64(mut num: u64) -> String { if num == 0 { return "0".to_string(); @@ -101,7 +75,6 @@ pub(crate) fn base62_encode_u64(mut num: u64) -> String { } out.iter().rev().collect() } - pub(crate) fn base62_encode_bytes(bytes: &[u8]) -> String { if bytes.is_empty() { return String::new(); @@ -119,9 +92,5 @@ pub(crate) fn base62_encode_bytes(bytes: &[u8]) -> String { carry /= 62; } } - digits - .iter() - .rev() - .map(|d| BASE62[*d as usize] as char) - .collect() + digits.iter().rev().map(|d| BASE62[*d as usize] as char).collect() } diff --git a/daemon-rs/src/auth/tests.rs b/daemon-rs/src/auth/tests/mod.rs similarity index 64% rename from daemon-rs/src/auth/tests.rs rename to daemon-rs/src/auth/tests/mod.rs index 027bab20..b752d441 100644 --- a/daemon-rs/src/auth/tests.rs +++ b/daemon-rs/src/auth/tests/mod.rs @@ -1,34 +1,22 @@ // SPDX-License-Identifier: MIT -//! Auth boundary tests only. - #[cfg(test)] mod tests { - use crate::auth::{ - acquire_global_daemon_lock, generate_ctx_api_key, verify_ctx_api_key_checksum, - }; use crate::auth::paths::CORTEX_GLOBAL_LOCK_HOME_ENV; + use crate::auth::{acquire_global_daemon_lock, generate_ctx_api_key, verify_ctx_api_key_checksum}; use crate::test_env::{lock, ScopedEnvVar}; - fn env_guard() -> tokio::sync::MutexGuard<'static, ()> { lock() } - #[test] fn verify_ctx_api_key_checksum_accepts_generated_keys() { let key = generate_ctx_api_key(); assert!(verify_ctx_api_key_checksum(&key)); } - #[test] fn acquire_global_daemon_lock_rejects_duplicate_instances() { let _guard = env_guard(); - let lock_home = std::env::temp_dir().join(format!( - "cortex-global-lock-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - )); + let lock_home = std::env::temp_dir() + .join(format!("cortex-global-lock-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos())); let _home_var = ScopedEnvVar::set(CORTEX_GLOBAL_LOCK_HOME_ENV, &lock_home); let _first = acquire_global_daemon_lock().expect("first lock"); let second = acquire_global_daemon_lock(); diff --git a/daemon-rs/src/budgets.rs b/daemon-rs/src/budgets.rs deleted file mode 100644 index a121e71e..00000000 --- a/daemon-rs/src/budgets.rs +++ /dev/null @@ -1,556 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Local operator budget configuration for daemon endpoints. - -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashMap}; -use std::path::{Path, PathBuf}; - -const BUDGETS_FILE_NAME: &str = "budgets.toml"; -pub const BUDGET_SOURCE: &str = "budgets.toml"; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub enum BudgetEndpoint { - Store, - Recall, - Boot, - Mcp, -} - -impl BudgetEndpoint { - pub const fn all() -> &'static [BudgetEndpoint] { - &[ - BudgetEndpoint::Store, - BudgetEndpoint::Recall, - BudgetEndpoint::Boot, - BudgetEndpoint::Mcp, - ] - } - - pub fn as_str(self) -> &'static str { - match self { - BudgetEndpoint::Store => "store", - BudgetEndpoint::Recall => "recall", - BudgetEndpoint::Boot => "boot", - BudgetEndpoint::Mcp => "mcp", - } - } - - fn parse(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "store" => Some(Self::Store), - "recall" => Some(Self::Recall), - "boot" => Some(Self::Boot), - "mcp" => Some(Self::Mcp), - _ => None, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct EndpointBudget { - pub limit: usize, - pub window_seconds: u64, -} - -impl EndpointBudget { - fn to_health_json(self) -> Value { - json!({ - "limit": self.limit, - "windowSeconds": self.window_seconds, - "window_seconds": self.window_seconds - }) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BudgetConfig { - pub enabled: bool, - endpoints: BTreeMap, -} - -impl BudgetConfig { - pub fn parse_toml_str(contents: &str) -> Result { - let raw: RawBudgetFile = toml::from_str(contents).map_err(|error| { - BudgetConfigError::new( - "parse_error", - format!("failed to parse budgets.toml: {error}"), - None, - None, - ) - })?; - - let enabled = raw - .defaults - .and_then(|defaults| defaults.enabled) - .unwrap_or(true); - - let mut endpoints = BTreeMap::new(); - for (name, raw_budget) in raw.endpoints.unwrap_or_default() { - let endpoint = BudgetEndpoint::parse(&name).ok_or_else(|| { - BudgetConfigError::new( - "unknown_endpoint", - format!("unknown budget endpoint: {name}"), - Some(name.clone()), - None, - ) - })?; - let limit = raw_budget.limit.ok_or_else(|| { - BudgetConfigError::new( - "missing_limit", - format!("budget endpoint {name} is missing limit"), - Some(name.clone()), - Some("limit"), - ) - })?; - if limit <= 0 { - return Err(BudgetConfigError::new( - "invalid_limit", - format!("budget endpoint {name} limit must be a positive integer"), - Some(name.clone()), - Some("limit"), - )); - } - - let window_seconds = raw_budget.window_seconds.ok_or_else(|| { - BudgetConfigError::new( - "missing_window_seconds", - format!("budget endpoint {name} is missing window_seconds"), - Some(name.clone()), - Some("window_seconds"), - ) - })?; - if window_seconds <= 0 { - return Err(BudgetConfigError::new( - "invalid_window_seconds", - format!("budget endpoint {name} window_seconds must be a positive integer"), - Some(name.clone()), - Some("window_seconds"), - )); - } - - endpoints.insert( - endpoint, - EndpointBudget { - limit: limit as usize, - window_seconds: window_seconds as u64, - }, - ); - } - - Ok(Self { enabled, endpoints }) - } - - pub fn budget_for(&self, endpoint: BudgetEndpoint) -> Option { - self.endpoints.get(&endpoint).copied() - } - - fn endpoints_json(&self) -> Value { - let mut map = serde_json::Map::new(); - for endpoint in BudgetEndpoint::all() { - if let Some(budget) = self.endpoints.get(endpoint) { - map.insert(endpoint.as_str().to_string(), budget.to_health_json()); - } - } - Value::Object(map) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BudgetConfigError { - pub code: String, - pub message: String, - pub endpoint: Option, - pub field: Option, -} - -impl BudgetConfigError { - fn new( - code: impl Into, - message: impl Into, - endpoint: Option, - field: Option<&str>, - ) -> Self { - Self { - code: code.into(), - message: message.into(), - endpoint, - field: field.map(str::to_string), - } - } - - fn to_json(&self) -> Value { - json!({ - "code": self.code, - "message": self.message, - "endpoint": self.endpoint, - "field": self.field - }) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BudgetConfigStatus { - pub config_loaded: bool, - pub source: PathBuf, - pub config: Option, - pub error: Option, -} - -impl BudgetConfigStatus { - pub fn load_from_home(home: &Path) -> Self { - Self::load_from_path(home.join(BUDGETS_FILE_NAME)) - } - - pub fn load_from_path(path: impl Into) -> Self { - let path = path.into(); - match std::fs::read_to_string(&path) { - Ok(contents) => Self::from_contents(path, &contents), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self { - config_loaded: false, - source: path, - config: None, - error: None, - }, - Err(error) => Self { - config_loaded: true, - source: path, - config: None, - error: Some(BudgetConfigError::new( - "io_error", - format!("failed to read budgets.toml: {error}"), - None, - None, - )), - }, - } - } - - pub fn missing_for_tests() -> Self { - Self { - config_loaded: false, - source: PathBuf::from(BUDGETS_FILE_NAME), - config: None, - error: None, - } - } - - fn from_contents(source: PathBuf, contents: &str) -> Self { - match BudgetConfig::parse_toml_str(contents) { - Ok(config) => Self { - config_loaded: true, - source, - config: Some(config), - error: None, - }, - Err(error) => Self { - config_loaded: true, - source, - config: None, - error: Some(error), - }, - } - } - - pub fn enabled(&self) -> bool { - self.error.is_none() - && self - .config - .as_ref() - .map(|config| config.enabled) - .unwrap_or(false) - } - - pub fn budget_for(&self, endpoint: BudgetEndpoint) -> Option { - if !self.enabled() { - return None; - } - self.config - .as_ref() - .and_then(|config| config.budget_for(endpoint)) - } - - pub fn to_health_json(&self, recent_denials: usize) -> Value { - json!({ - "configLoaded": self.config_loaded, - "config_loaded": self.config_loaded, - "enabled": self.enabled(), - "source": BUDGET_SOURCE, - "error": self.error.as_ref().map(BudgetConfigError::to_json), - "endpoints": self - .config - .as_ref() - .map(BudgetConfig::endpoints_json) - .unwrap_or_else(|| json!({})), - "recentDenials": recent_denials, - "recent_denials": recent_denials - }) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BudgetDecision { - pub allowed: bool, - pub endpoint: BudgetEndpoint, - pub limit: usize, - pub window_seconds: u64, - pub retry_after_seconds: u64, - pub remaining: Option, -} - -impl BudgetDecision { - pub fn allowed(endpoint: BudgetEndpoint, budget: EndpointBudget, remaining: usize) -> Self { - Self { - allowed: true, - endpoint, - limit: budget.limit, - window_seconds: budget.window_seconds, - retry_after_seconds: 0, - remaining: Some(remaining), - } - } - - pub fn denied(endpoint: BudgetEndpoint, budget: EndpointBudget, retry_after: u64) -> Self { - Self { - allowed: false, - endpoint, - limit: budget.limit, - window_seconds: budget.window_seconds, - retry_after_seconds: retry_after, - remaining: Some(0), - } - } - - pub fn http_body_json(&self) -> Value { - json!({ - "error": "budget_exceeded", - "endpoint": self.endpoint.as_str(), - "limit": self.limit, - "window_seconds": self.window_seconds, - "retry_after_seconds": self.retry_after_seconds, - "source": BUDGET_SOURCE - }) - } - - pub fn event_json(&self, request_source: &str, source_ip: &str) -> Value { - json!({ - "endpoint": self.endpoint.as_str(), - "limit": self.limit, - "window_seconds": self.window_seconds, - "retry_after_seconds": self.retry_after_seconds, - "source": BUDGET_SOURCE, - "request_source": request_source, - "source_ip": source_ip - }) - } -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawBudgetFile { - defaults: Option, - endpoints: Option>, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawDefaults { - enabled: Option, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawEndpointBudget { - limit: Option, - window_seconds: Option, -} - -#[cfg(test)] -mod tests { - use super::*; - - fn valid_config() -> &'static str { - r#" -[defaults] -enabled = true - -[endpoints.store] -limit = 120 -window_seconds = 60 - -[endpoints.recall] -limit = 300 -window_seconds = 60 - -[endpoints.boot] -limit = 60 -window_seconds = 60 - -[endpoints.mcp] -limit = 240 -window_seconds = 60 -"# - } - - #[test] - fn missing_file_disables_budgets_without_error() { - let path = std::env::temp_dir().join(format!( - "cortex-missing-budgets-{}.toml", - uuid::Uuid::new_v4() - )); - let status = BudgetConfigStatus::load_from_path(path); - assert!(!status.config_loaded); - assert!(!status.enabled()); - assert!(status.error.is_none()); - assert!(status.budget_for(BudgetEndpoint::Store).is_none()); - } - - #[test] - fn valid_config_parses_all_endpoint_budgets() { - let config = BudgetConfig::parse_toml_str(valid_config()).unwrap(); - assert!(config.enabled); - assert_eq!( - config.budget_for(BudgetEndpoint::Store), - Some(EndpointBudget { - limit: 120, - window_seconds: 60 - }) - ); - assert_eq!( - config.budget_for(BudgetEndpoint::Recall), - Some(EndpointBudget { - limit: 300, - window_seconds: 60 - }) - ); - assert_eq!( - config.budget_for(BudgetEndpoint::Boot), - Some(EndpointBudget { - limit: 60, - window_seconds: 60 - }) - ); - assert_eq!( - config.budget_for(BudgetEndpoint::Mcp), - Some(EndpointBudget { - limit: 240, - window_seconds: 60 - }) - ); - } - - #[test] - fn disabled_config_validates_but_does_not_enforce() { - let status = BudgetConfigStatus::from_contents( - PathBuf::from("budgets.toml"), - r#" -[defaults] -enabled = false - -[endpoints.recall] -limit = 1 -window_seconds = 60 -"#, - ); - assert!(status.config_loaded); - assert!(status.error.is_none()); - assert!(!status.enabled()); - assert!(status.budget_for(BudgetEndpoint::Recall).is_none()); - } - - #[test] - fn health_json_uses_portable_budget_source_label() { - let status = BudgetConfigStatus::from_contents( - PathBuf::from("C:/cortex-test/testuser/.cortex/budgets.toml"), - valid_config(), - ); - let payload = status.to_health_json(0); - assert_eq!(payload["source"], BUDGET_SOURCE); - } - - #[test] - fn missing_endpoint_is_unlimited_for_that_endpoint() { - let config = BudgetConfig::parse_toml_str( - r#" -[defaults] -enabled = true - -[endpoints.store] -limit = 2 -window_seconds = 60 -"#, - ) - .unwrap(); - assert!(config.budget_for(BudgetEndpoint::Recall).is_none()); - } - - #[test] - fn zero_limit_is_structured_error() { - let err = BudgetConfig::parse_toml_str( - r#" -[endpoints.store] -limit = 0 -window_seconds = 60 -"#, - ) - .unwrap_err(); - assert_eq!(err.code, "invalid_limit"); - assert_eq!(err.endpoint.as_deref(), Some("store")); - assert_eq!(err.field.as_deref(), Some("limit")); - } - - #[test] - fn negative_limit_is_structured_error() { - let err = BudgetConfig::parse_toml_str( - r#" -[endpoints.store] -limit = -1 -window_seconds = 60 -"#, - ) - .unwrap_err(); - assert_eq!(err.code, "invalid_limit"); - } - - #[test] - fn zero_window_is_structured_error() { - let err = BudgetConfig::parse_toml_str( - r#" -[endpoints.recall] -limit = 1 -window_seconds = 0 -"#, - ) - .unwrap_err(); - assert_eq!(err.code, "invalid_window_seconds"); - assert_eq!(err.endpoint.as_deref(), Some("recall")); - assert_eq!(err.field.as_deref(), Some("window_seconds")); - } - - #[test] - fn negative_window_is_structured_error() { - let err = BudgetConfig::parse_toml_str( - r#" -[endpoints.recall] -limit = 1 -window_seconds = -30 -"#, - ) - .unwrap_err(); - assert_eq!(err.code, "invalid_window_seconds"); - } - - #[test] - fn unknown_endpoint_is_structured_error() { - let err = BudgetConfig::parse_toml_str( - r#" -[endpoints.search] -limit = 1 -window_seconds = 60 -"#, - ) - .unwrap_err(); - assert_eq!(err.code, "unknown_endpoint"); - assert_eq!(err.endpoint.as_deref(), Some("search")); - } -} diff --git a/daemon-rs/src/budgets/mod.rs b/daemon-rs/src/budgets/mod.rs new file mode 100644 index 00000000..e30668d8 --- /dev/null +++ b/daemon-rs/src/budgets/mod.rs @@ -0,0 +1,229 @@ +use serde::Deserialize; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +const BUDGETS_FILE_NAME: &str = "budgets.toml"; +pub const BUDGET_SOURCE: &str = "budgets.toml"; +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum BudgetEndpoint { + Store, + Recall, + Boot, + Mcp, +} +impl BudgetEndpoint { + pub const fn all() -> &'static [BudgetEndpoint] { + &[BudgetEndpoint::Store, BudgetEndpoint::Recall, BudgetEndpoint::Boot, BudgetEndpoint::Mcp] + } + pub fn as_str(self) -> &'static str { + match self { + BudgetEndpoint::Store => "store", + BudgetEndpoint::Recall => "recall", + BudgetEndpoint::Boot => "boot", + BudgetEndpoint::Mcp => "mcp", + } + } + fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "store" => Some(Self::Store), + "recall" => Some(Self::Recall), + "boot" => Some(Self::Boot), + "mcp" => Some(Self::Mcp), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct EndpointBudget { + pub limit: usize, + pub window_seconds: u64, +} +impl EndpointBudget { + fn to_health_json(self) -> Value { + json!({"limit":self. +limit,"windowSeconds":self.window_seconds,"window_seconds":self.window_seconds}) + } +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BudgetConfig { + pub enabled: bool, + endpoints: BTreeMap, +} +impl BudgetConfig { + pub fn parse_toml_str(contents: &str) -> Result { + let raw: RawBudgetFile = + toml::from_str(contents).map_err(|error| BudgetConfigError::new("parse_error", format!("failed to parse budgets.toml: {error}"), None, None))?; + let enabled = raw.defaults.and_then(|defaults| defaults.enabled).unwrap_or(true); + let mut endpoints = BTreeMap::new(); + for (name, raw_budget) in raw.endpoints.unwrap_or_default() { + let endpoint = BudgetEndpoint::parse(&name) + .ok_or_else(|| BudgetConfigError::new("unknown_endpoint", format!("unknown budget endpoint: {name}"), Some(name.clone()), None))?; + let limit = raw_budget.limit.ok_or_else(|| { + BudgetConfigError::new("missing_limit", format!("budget endpoint {name} is missing limit"), Some(name.clone()), Some("limit")) + })?; + if limit <= 0 { + return Err(BudgetConfigError::new( + "invalid_limit", + format!("budget endpoint {name} limit must be a positive integer"), + Some(name.clone()), + Some("limit"), + )); + } + let window_seconds = raw_budget.window_seconds.ok_or_else(|| { + BudgetConfigError::new( + "missing_window_seconds", + format!("budget endpoint {name} is missing window_seconds"), + Some(name.clone()), + Some("window_seconds"), + ) + })?; + if window_seconds <= 0 { + return Err(BudgetConfigError::new( + "invalid_window_seconds", + format!("budget endpoint {name} window_seconds must be a positive integer"), + Some(name.clone()), + Some("window_seconds"), + )); + } + endpoints.insert(endpoint, EndpointBudget { limit: limit as usize, window_seconds: window_seconds as u64 }); + } + Ok(Self { enabled, endpoints }) + } + pub fn budget_for(&self, endpoint: BudgetEndpoint) -> Option { + self.endpoints.get(&endpoint).copied() + } + fn endpoints_json(&self) -> Value { + let mut map = serde_json::Map::new(); + for endpoint in BudgetEndpoint::all() { + if let Some(budget) = self.endpoints.get(endpoint) { + map.insert(endpoint.as_str().to_string(), budget.to_health_json()); + } + } + Value::Object(map) + } +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BudgetConfigError { + pub code: String, + pub message: String, + pub endpoint: Option, + pub field: Option, +} +impl BudgetConfigError { + fn new(code: impl Into, message: impl Into, endpoint: Option, field: Option<&str>) -> Self { + Self { code: code.into(), message: message.into(), endpoint, field: field.map(str::to_string) } + } + fn to_json(&self) -> Value { + json!({"code":self.code,"message":self.message,"endpoint":self.endpoint, +"field":self.field}) + } +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BudgetConfigStatus { + pub config_loaded: bool, + pub source: PathBuf, + pub config: Option, + pub error: Option, +} +impl BudgetConfigStatus { + pub fn load_from_home(home: &Path) -> Self { + Self::load_from_path(home.join(BUDGETS_FILE_NAME)) + } + pub fn load_from_path(path: impl Into) -> Self { + let path = path.into(); + match std::fs::read_to_string(&path) { + Ok(contents) => Self::from_contents(path, &contents), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self { config_loaded: false, source: path, config: None, error: None }, + Err(error) => Self { + config_loaded: true, + source: path, + config: None, + error: Some(BudgetConfigError::new("io_error", format!("failed to read budgets.toml: {error}"), None, None)), + }, + } + } + pub fn missing_for_tests() -> Self { + Self { config_loaded: false, source: PathBuf::from(BUDGETS_FILE_NAME), config: None, error: None } + } + fn from_contents(source: PathBuf, contents: &str) -> Self { + match BudgetConfig::parse_toml_str(contents) { + Ok(config) => Self { config_loaded: true, source, config: Some(config), error: None }, + Err(error) => Self { config_loaded: true, source, config: None, error: Some(error) }, + } + } + pub fn enabled(&self) -> bool { + self.error.is_none() && self.config.as_ref().map(|config| config.enabled).unwrap_or(false) + } + pub fn budget_for(&self, endpoint: BudgetEndpoint) -> Option { + if !self.enabled() { + return None; + } + self.config.as_ref().and_then(|config| config.budget_for(endpoint)) + } + pub fn to_health_json(&self, recent_denials: usize) -> Value { + json!({"configLoaded":self.config_loaded, +"config_loaded":self.config_loaded,"enabled":self.enabled(),"source":BUDGET_SOURCE,"error":self.error.as_ref().map( +BudgetConfigError::to_json),"endpoints":self.config.as_ref().map(BudgetConfig::endpoints_json).unwrap_or_else(||json!({})), +"recentDenials":recent_denials,"recent_denials":recent_denials}) + } +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BudgetDecision { + pub allowed: bool, + pub endpoint: BudgetEndpoint, + pub limit: usize, + pub window_seconds: u64, + pub retry_after_seconds: u64, + pub remaining: Option, +} +impl BudgetDecision { + pub fn allowed(endpoint: BudgetEndpoint, budget: EndpointBudget, remaining: usize) -> Self { + Self { + allowed: true, + endpoint, + limit: budget.limit, + window_seconds: budget.window_seconds, + retry_after_seconds: 0, + remaining: Some(remaining), + } + } + pub fn denied(endpoint: BudgetEndpoint, budget: EndpointBudget, retry_after: u64) -> Self { + Self { + allowed: false, + endpoint, + limit: budget.limit, + window_seconds: budget.window_seconds, + retry_after_seconds: retry_after, + remaining: Some(0), + } + } + pub fn http_body_json(&self) -> Value { + json!({"error": +"budget_exceeded","endpoint":self.endpoint.as_str(),"limit":self.limit,"window_seconds":self.window_seconds,"retry_after_seconds": +self.retry_after_seconds,"source":BUDGET_SOURCE}) + } + pub fn event_json(&self, request_source: &str, source_ip: &str) -> Value { + json!({ +"endpoint":self.endpoint.as_str(),"limit":self.limit,"window_seconds":self.window_seconds,"retry_after_seconds":self. +retry_after_seconds,"source":BUDGET_SOURCE,"request_source":request_source,"source_ip":source_ip}) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawBudgetFile { + defaults: Option, + endpoints: Option>, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawDefaults { + enabled: Option, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawEndpointBudget { + limit: Option, + window_seconds: Option, +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/budgets/tests/mod.rs b/daemon-rs/src/budgets/tests/mod.rs new file mode 100644 index 00000000..ba5d0428 --- /dev/null +++ b/daemon-rs/src/budgets/tests/mod.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +fn valid_config() -> &'static str { + r#" +[defaults] +enabled = true +[endpoints.store] +limit = 120 +window_seconds = 60 +[endpoints.recall] +limit = 300 +window_seconds = 60 +[endpoints.boot] +limit = 60 +window_seconds = 60 +[endpoints.mcp] +limit = 240 +window_seconds = 60 +"# +} +#[test] +fn missing_file_disables_budgets_without_error() { + let path = std::env::temp_dir().join(format!("cortex-missing-budgets-{}.toml", uuid::Uuid::new_v4())); + let status = BudgetConfigStatus::load_from_path(path); + assert!(!status.config_loaded); + assert!(!status.enabled()); + assert!(status.error.is_none()); + assert!(status.budget_for(BudgetEndpoint::Store).is_none()); +} +#[test] +fn valid_config_parses_all_endpoint_budgets() { + let config = BudgetConfig::parse_toml_str(valid_config()).unwrap(); + assert!(config.enabled); + assert_eq!(config.budget_for(BudgetEndpoint::Store), Some(EndpointBudget { limit: 120, window_seconds: 60 })); + assert_eq!(config.budget_for(BudgetEndpoint::Recall), Some(EndpointBudget { limit: 300, window_seconds: 60 })); + assert_eq!(config.budget_for(BudgetEndpoint::Boot), Some(EndpointBudget { limit: 60, window_seconds: 60 })); + assert_eq!(config.budget_for(BudgetEndpoint::Mcp), Some(EndpointBudget { limit: 240, window_seconds: 60 })); +} +#[test] +fn disabled_config_validates_but_does_not_enforce() { + let status = BudgetConfigStatus::from_contents( + PathBuf::from("budgets.toml"), + r#" +[defaults] +enabled = false +[endpoints.recall] +limit = 1 +window_seconds = 60 +"#, + ); + assert!(status.config_loaded); + assert!(status.error.is_none()); + assert!(!status.enabled()); + assert!(status.budget_for(BudgetEndpoint::Recall).is_none()); +} +#[test] +fn health_json_uses_portable_budget_source_label() { + let status = BudgetConfigStatus::from_contents(PathBuf::from("C:/cortex-test/testuser/.cortex/budgets.toml"), valid_config()); + let payload = status.to_health_json(0); + assert_eq!(payload["source"], BUDGET_SOURCE); +} +#[test] +fn missing_endpoint_is_unlimited_for_that_endpoint() { + let config = BudgetConfig::parse_toml_str( + r#" +[defaults] +enabled = true +[endpoints.store] +limit = 2 +window_seconds = 60 +"#, + ) + .unwrap(); + assert!(config.budget_for(BudgetEndpoint::Recall).is_none()); +} +#[test] +fn zero_limit_is_structured_error() { + let err = BudgetConfig::parse_toml_str( + r#" +[endpoints.store] +limit = 0 +window_seconds = 60 +"#, + ) + .unwrap_err(); + assert_eq!(err.code, "invalid_limit"); + assert_eq!(err.endpoint.as_deref(), Some("store")); + assert_eq!(err.field.as_deref(), Some("limit")); +} +#[test] +fn negative_limit_is_structured_error() { + let err = BudgetConfig::parse_toml_str( + r#" +[endpoints.store] +limit = -1 +window_seconds = 60 +"#, + ) + .unwrap_err(); + assert_eq!(err.code, "invalid_limit"); +} +#[test] +fn zero_window_is_structured_error() { + let err = BudgetConfig::parse_toml_str( + r#" +[endpoints.recall] +limit = 1 +window_seconds = 0 +"#, + ) + .unwrap_err(); + assert_eq!(err.code, "invalid_window_seconds"); + assert_eq!(err.endpoint.as_deref(), Some("recall")); + assert_eq!(err.field.as_deref(), Some("window_seconds")); +} +#[test] +fn negative_window_is_structured_error() { + let err = BudgetConfig::parse_toml_str( + r#" +[endpoints.recall] +limit = 1 +window_seconds = -30 +"#, + ) + .unwrap_err(); + assert_eq!(err.code, "invalid_window_seconds"); +} +#[test] +fn unknown_endpoint_is_structured_error() { + let err = BudgetConfig::parse_toml_str( + r#" +[endpoints.search] +limit = 1 +window_seconds = 60 +"#, + ) + .unwrap_err(); + assert_eq!(err.code, "unknown_endpoint"); + assert_eq!(err.endpoint.as_deref(), Some("search")); +} diff --git a/daemon-rs/src/cli/admin.rs b/daemon-rs/src/cli/admin.rs index e0d15758..2d87a619 100644 --- a/daemon-rs/src/cli/admin.rs +++ b/daemon-rs/src/cli/admin.rs @@ -1,685 +1,103 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; - use crate::auth; -use crate::budgets; -use crate::db; -use super::common::{ - admin_request, api_key_output_masked, confirm_action, format_api_key_for_output, json_field, - json_str, json_str_or, parse_flag_value, required_cli_positional_or_exit, - validate_cli_options_or_exit, -}; +use super::common::{admin_request, required_cli_positional_or_exit, validate_cli_options_or_exit}; -pub(crate) fn run_admin_budgets_cli(paths: &auth::CortexPaths, args: &[String]) { - let subcmd = args.first().map(String::as_str).unwrap_or(""); - let json_output = args.iter().any(|arg| arg == "--json"); - match subcmd { - "status" => { - validate_cli_options_or_exit(&args[1..], &[], &["--json"]); - let status = budgets::BudgetConfigStatus::load_from_home(&paths.home); - let payload = status.to_health_json(0); - if json_output { - println!("{}", serde_json::to_string_pretty(&payload).unwrap()); - return; - } - print_budget_status_human(&payload); - } - "validate" => { - validate_cli_options_or_exit(&args[1..], &["--path"], &["--json"]); - let Some(path) = parse_flag_value(args, "--path") else { - eprintln!("Usage: cortex admin budgets validate --path [--json]"); - std::process::exit(1); - }; - let status = budgets::BudgetConfigStatus::load_from_path(path); - let mut payload = status.to_health_json(0); - if !status.config_loaded && status.error.is_none() { - payload["error"] = json!({ - "code": "not_found", - "message": "budget config file was not found", - "endpoint": null, - "field": null - }); - } - if json_output { - println!("{}", serde_json::to_string_pretty(&payload).unwrap()); - } else { - print_budget_status_human(&payload); - } - if payload["error"].is_object() { - std::process::exit(1); - } - } - _ => { - eprintln!("Usage: cortex admin budgets > [--json]"); - std::process::exit(1); - } - } +fn fail(usage: &str) -> ! { + eprintln!("{usage}"); + std::process::exit(1); } -fn print_budget_status_human(payload: &Value) { - println!("Cortex Budget Governance"); - println!("{}", "=".repeat(50)); - println!("Source: {}", json_str(payload, "source")); - println!("Config loaded: {}", json_field(payload, "configLoaded")); - println!("Enabled: {}", json_field(payload, "enabled")); - if let Some(error) = payload.get("error").and_then(Value::as_object) { - println!( - "Error: {} ({})", - error - .get("message") - .and_then(Value::as_str) - .unwrap_or("unknown error"), - error - .get("code") - .and_then(Value::as_str) - .unwrap_or("unknown") - ); - return; - } - if let Some(endpoints) = payload.get("endpoints").and_then(Value::as_object) { - if endpoints.is_empty() { - println!("Endpoints: unlimited"); - return; - } - println!(); - println!("{:<12} {:<10} WINDOW", "ENDPOINT", "LIMIT"); - println!("{}", "-".repeat(36)); - for (endpoint, budget) in endpoints { - println!( - "{:<12} {:<10} {}s", - endpoint, - budget.get("limit").and_then(Value::as_u64).unwrap_or(0), - budget - .get("windowSeconds") - .and_then(Value::as_u64) - .unwrap_or(0) - ); - } - } -} - -pub(crate) fn run_admin_rollback_cli(paths: &auth::CortexPaths, args: &[String]) { - validate_cli_options_or_exit( - args, - &["--session-id", "--session"], - &["--apply", "--json", "--help", "-h"], - ); - - let mut session_id: Option = None; - let mut apply = false; - let mut json_output = false; - let mut i = 0usize; - while i < args.len() { - match args[i].as_str() { - "--session-id" | "--session" => { - if let Some(v) = args.get(i + 1) { - session_id = Some(v.clone()); - i += 1; - } - } - "--apply" => apply = true, - "--json" => json_output = true, - "--help" | "-h" => { - println!( - "Usage: cortex admin rollback --session-id [--apply] [--json]\n\ - \n\ - Soft-deletes every memory + decision written by the session's\n\ - agent since the session started. Dry-run by default; pass\n\ - --apply to write. Idempotent." - ); - std::process::exit(0); - } - other => { - eprintln!("Unknown flag: {other}"); - eprintln!("Usage: cortex admin rollback --session-id [--apply] [--json]"); - std::process::exit(1); - } - } - i += 1; - } - - let Some(session_id) = session_id else { - eprintln!("Usage: cortex admin rollback --session-id [--apply] [--json]"); - std::process::exit(1); - }; - - let conn = match db::open(&paths.db) { - Ok(c) => c, +fn print_daemon_error(result: Result) { + match result { + Ok(value) => println!("{}", serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())), Err(err) => { - eprintln!("Error: failed to open database for rollback: {err}"); + eprintln!("{err}"); std::process::exit(1); } - }; - if let Err(err) = db::configure(&conn) { - eprintln!("Error: failed to configure database for rollback: {err}"); - std::process::exit(1); } - // Ensure tables exist on a freshly-created DB so `--session-id` - // returns "not found" instead of "no such table". - if let Err(err) = db::initialize_schema(&conn) { - eprintln!("Error: failed to initialize schema: {err}"); - std::process::exit(1); - } - db::run_pending_migrations(&conn); - - let stats = match crate::admin::rollback_session_by_id(&conn, &session_id, apply) { - Ok(s) => s, - Err(err) => { - eprintln!("Error: rollback failed: {err}"); - std::process::exit(1); - } - }; - - // On apply, persist a `session.rolled_back` event so SSE/audit see it - // even when the daemon was offline at rollback time. - if apply && !stats.agent.is_empty() { - let payload = json!({ - "session_id": stats.session_id, - "agent": stats.agent, - "session_started_at": stats.session_started_at, - "memories_affected": stats.memories_affected, - "decisions_affected": stats.decisions_affected, - "already_rolled_back": stats.already_rolled_back, - }); - let _ = conn.execute( - "INSERT INTO events(type, data, source_agent) VALUES (?1, ?2, ?3)", - rusqlite::params!["session.rolled_back", payload.to_string(), stats.agent,], - ); - } - - if json_output { - println!( - "{}", - json!({ - "rollback": true, - "applied": stats.applied, - "session_id": stats.session_id, - "agent": stats.agent, - "session_started_at": stats.session_started_at, - "memories_affected": stats.memories_affected, - "decisions_affected": stats.decisions_affected, - "already_rolled_back": stats.already_rolled_back, - }) - ); - } else if stats.agent.is_empty() { - eprintln!( - "Session not found: '{session_id}'. The sessions table is keyed\n\ - by agent + current session_id; expired / superseded sessions\n\ - cannot be rolled back by id alone." - ); - std::process::exit(1); - } else { - let label = if stats.applied { "applied" } else { "dry-run" }; - println!( - "[rollback {label}] session={} agent={} started_at={}", - stats.session_id, stats.agent, stats.session_started_at - ); - println!( - " memories to flip: {} decisions to flip: {}", - stats.memories_affected, stats.decisions_affected - ); - if stats.already_rolled_back { - println!(" note: session already rolled back previously; nothing to do."); - } - if !stats.applied { - println!(" Dry-run only. Pass --apply to persist."); - } - } - - std::process::exit(0); } - pub(crate) async fn run_user_cli(paths: &auth::CortexPaths, args: &[String]) { - let subcmd = args.get(2).map(|s| s.as_str()).unwrap_or(""); - match subcmd { - "add" => { - let username = required_cli_positional_or_exit( - &args, - 3, - "Usage: cortex user add [--role member|admin] [--display-name \"...\"]", - ); - validate_cli_options_or_exit(&args[4..], &["--role", "--display-name"], &[]); - let mut role = "member".to_string(); - let mut display_name: Option = None; - let mut i = 4usize; - while i < args.len() { - match args[i].as_str() { - "--role" => { - if let Some(v) = args.get(i + 1) { - role = v.clone(); - i += 1; - } - } - "--display-name" => { - if let Some(v) = args.get(i + 1) { - display_name = Some(v.clone()); - i += 1; - } - } - _ => {} - } - i += 1; - } - let mut body = serde_json::json!({ - "username": username, - "role": role, - }); - if let Some(dn) = display_name { - body["display_name"] = serde_json::json!(dn); - } - match admin_request(&paths, "POST", "/admin/user/add", Some(body)).await { - Ok(json) => { - let api_key = json_str(&json, "api_key"); - let key_masked = api_key_output_masked(); - println!("User created:"); - println!(" Username: {}", json_str(&json, "username")); - println!(" User ID: {}", json_field(&json, "user_id")); - println!(" Role: {}", json_str(&json, "role")); - println!(" API Key: {}", format_api_key_for_output(&api_key)); - if key_masked { - println!( - " NOTE: API key is masked because stdout is non-interactive." - ); - println!( - " Re-run this command in a terminal to display the full key." - ); - } - println!(); - println!("Save the API key -- it cannot be retrieved later."); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } + match args.get(2).map(String::as_str).unwrap_or("") { + "list" => { + validate_cli_options_or_exit(&args[3..], &[], &[]); + print_daemon_error(admin_request(paths, "GET", "/admin/users", None).await); } - } - "rotate-key" => { - let username = required_cli_positional_or_exit( - &args, - 3, - "Usage: cortex user rotate-key ", - ); - validate_cli_options_or_exit(&args[4..], &[], &[]); - let body = serde_json::json!({ "username": username }); - match admin_request(&paths, "POST", "/admin/user/rotate-key", Some(body)).await - { - Ok(json) => { - let api_key = json_str(&json, "api_key"); - let key_masked = api_key_output_masked(); - println!("API key rotated for '{}':", json_str(&json, "username")); - println!(" New API Key: {}", format_api_key_for_output(&api_key)); - if key_masked { - println!( - " NOTE: API key is masked because stdout is non-interactive." - ); - println!( - " Re-run this command in a terminal to display the full key." - ); - } - println!(); - println!("Save the API key -- it cannot be retrieved later."); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } + "add" => { + let name = required_cli_positional_or_exit(args, 3, "Usage: cortex user add [--role member|admin] [--display-name ]"); + validate_cli_options_or_exit(&args[4..], &["--role", "--display-name"], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/user/add", Some(serde_json::json!({"username":name}))).await); } - } - "remove" => { - let username = required_cli_positional_or_exit( - &args, - 3, - "Usage: cortex user remove ", - ); - validate_cli_options_or_exit(&args[4..], &[], &[]); - if !confirm_action(&format!("Remove user '{username}'?")) { - eprintln!("Cancelled."); - std::process::exit(0); + "rotate-key" => { + let name = required_cli_positional_or_exit(args, 3, "Usage: cortex user rotate-key "); + validate_cli_options_or_exit(&args[4..], &[], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/user/rotate-key", Some(serde_json::json!({"username":name}))).await); } - let body = serde_json::json!({ "username": username }); - match admin_request(&paths, "POST", "/admin/user/remove", Some(body)).await { - Ok(json) => { - println!("Removed user '{}'", json_str(&json, "removed")); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } + "remove" => { + let name = required_cli_positional_or_exit(args, 3, "Usage: cortex user remove "); + validate_cli_options_or_exit(&args[4..], &[], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/user/remove", Some(serde_json::json!({"username":name}))).await); } - } - "list" => { - validate_cli_options_or_exit(&args[3..], &[], &[]); - match admin_request(&paths, "GET", "/admin/users", None).await { - Ok(json) => { - let users = json["users"].as_array(); - match users { - Some(arr) if !arr.is_empty() => { - println!( - "{:<6} {:<20} {:<20} {:<10} CREATED", - "ID", "USERNAME", "DISPLAY NAME", "ROLE" - ); - println!("{}", "-".repeat(80)); - for u in arr { - println!( - "{:<6} {:<20} {:<20} {:<10} {}", - json_field(u, "id"), - json_str(u, "username"), - json_str_or(u, "display_name", "-"), - json_str(u, "role"), - json_str_or(u, "created_at", "-"), - ); - } - println!(); - println!("{} user(s)", arr.len()); - } - _ => println!("No users found."), - } - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - _ => { - eprintln!("Usage: cortex user "); - std::process::exit(1); - } + _ => fail("Usage: cortex user "), } } pub(crate) async fn run_team_cli(paths: &auth::CortexPaths, args: &[String]) { - let subcmd = args.get(2).map(|s| s.as_str()).unwrap_or(""); - match subcmd { - "create" => { - let name = required_cli_positional_or_exit( - &args, - 3, - "Usage: cortex team create ", - ); - validate_cli_options_or_exit(&args[4..], &[], &[]); - let body = serde_json::json!({ "name": name }); - match admin_request(&paths, "POST", "/admin/team/create", Some(body)).await { - Ok(json) => { - println!("Team created:"); - println!(" Name: {}", json_str(&json, "name")); - println!(" Team ID: {}", json_field(&json, "team_id")); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "add" => { - let usage = "Usage: cortex team add [--role member|admin]"; - let team_name = required_cli_positional_or_exit(&args, 3, usage); - let username = required_cli_positional_or_exit(&args, 4, usage); - validate_cli_options_or_exit(&args[5..], &["--role"], &[]); - let mut role = "member".to_string(); - let mut i = 5usize; - while i < args.len() { - if args[i] == "--role" { - if let Some(v) = args.get(i + 1) { - role = v.clone(); - i += 1; - } - } - i += 1; - } - let body = serde_json::json!({ - "team_name": team_name, - "username": username, - "role": role, - }); - match admin_request(&paths, "POST", "/admin/team/add-member", Some(body)).await - { - Ok(json) => { - println!( - "Added '{}' to team '{}' as {}", - json_str(&json, "username"), - json_str(&json, "team"), - json_str(&json, "role"), - ); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "remove" => { - let usage = "Usage: cortex team remove "; - let team_name = required_cli_positional_or_exit(&args, 3, usage); - let username = required_cli_positional_or_exit(&args, 4, usage); - validate_cli_options_or_exit(&args[5..], &[], &[]); - if !confirm_action(&format!("Remove '{username}' from team '{team_name}'?")) { - eprintln!("Cancelled."); - std::process::exit(0); - } - let body = serde_json::json!({ - "team_name": team_name, - "username": username, - }); - match admin_request(&paths, "POST", "/admin/team/remove-member", Some(body)) - .await - { - Ok(json) => { - let removed = &json["removed"]; - println!( - "Removed '{}' from team '{}'", - json_str(removed, "username"), - json_str(removed, "team"), - ); - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "list" => { - validate_cli_options_or_exit(&args[3..], &[], &[]); - match admin_request(&paths, "GET", "/admin/teams", None).await { - Ok(json) => { - let teams = json["teams"].as_array(); - match teams { - Some(arr) if !arr.is_empty() => { - println!( - "{:<6} {:<30} {:<10} CREATED", - "ID", "NAME", "MEMBERS" - ); - println!("{}", "-".repeat(70)); - for t in arr { - println!( - "{:<6} {:<30} {:<10} {}", - json_field(t, "id"), - json_str(t, "name"), - json_field(t, "member_count"), - json_str_or(t, "created_at", "-"), - ); - } - println!(); - println!("{} team(s)", arr.len()); - } - _ => println!("No teams found."), - } - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - _ => { - eprintln!("Usage: cortex team "); - std::process::exit(1); - } + match args.get(2).map(String::as_str).unwrap_or("") { + "list" => { + validate_cli_options_or_exit(&args[3..], &[], &[]); + print_daemon_error(admin_request(paths, "GET", "/admin/teams", None).await); + } + "create" => { + let team = required_cli_positional_or_exit(args, 3, "Usage: cortex team create "); + validate_cli_options_or_exit(&args[4..], &[], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/team/create", Some(serde_json::json!({"team":team}))).await); + } + "add" => { + let team = required_cli_positional_or_exit(args, 3, "Usage: cortex team add [--role member|admin]"); + let user = required_cli_positional_or_exit(args, 4, "Usage: cortex team add [--role member|admin]"); + validate_cli_options_or_exit(&args[5..], &["--role"], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/team/add-member", Some(serde_json::json!({"team":team,"username":user}))).await); + } + "remove" => { + let team = required_cli_positional_or_exit(args, 3, "Usage: cortex team remove "); + let user = required_cli_positional_or_exit(args, 4, "Usage: cortex team remove "); + validate_cli_options_or_exit(&args[5..], &[], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/team/remove-member", Some(serde_json::json!({"team":team,"username":user}))).await); + } + _ => fail("Usage: cortex team "), } } pub(crate) async fn run_admin_cli(paths: &auth::CortexPaths, args: &[String]) { - let subcmd = args.get(2).map(|s| s.as_str()).unwrap_or(""); - match subcmd { - "list-unowned" => { - validate_cli_options_or_exit(&args[3..], &[], &[]); - match admin_request(&paths, "GET", "/admin/unowned", None).await { - Ok(json) => { - let unowned = json["unowned"].as_object(); - match unowned { - Some(map) if !map.is_empty() => { - println!("{:<25} UNOWNED ROWS", "TABLE"); - println!("{}", "-".repeat(40)); - let mut total: i64 = 0; - for (table, count) in map { - let n = count.as_i64().unwrap_or(0); - total += n; - println!("{:<25} {}", table, n); - } - println!("{}", "-".repeat(40)); - println!("{:<25} {}", "TOTAL", total); - } - _ => println!("No unowned data found."), - } - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "assign-owner" => { - validate_cli_options_or_exit(&args[3..], &["--from", "--to", "--table"], &[]); - let mut from_user: Option = None; - let mut to_user: Option = None; - let mut table: Option = None; - let mut i = 3usize; - while i < args.len() { - match args[i].as_str() { - "--from" => { - if let Some(v) = args.get(i + 1) { - from_user = Some(v.clone()); - i += 1; - } - } - "--to" => { - if let Some(v) = args.get(i + 1) { - to_user = Some(v.clone()); - i += 1; - } - } - "--table" => { - if let Some(v) = args.get(i + 1) { - table = Some(v.clone()); - i += 1; - } - } - _ => {} - } - i += 1; - } - let Some(to) = to_user else { - eprintln!( - "Usage: cortex admin assign-owner [--from ] --to [--table ]" - ); - std::process::exit(1); - }; - let mut body = serde_json::json!({ "to_user": to }); - if let Some(from) = from_user { - body["from_user"] = serde_json::json!(from); - } - if let Some(t) = table { - body["table"] = serde_json::json!(t); - } - match admin_request(&paths, "POST", "/admin/assign-owner", Some(body)).await { - Ok(json) => { - let assigned = json["assigned"].as_object(); - match assigned { - Some(map) if !map.is_empty() => { - println!("{:<25} ROWS ASSIGNED", "TABLE"); - println!("{}", "-".repeat(40)); - let mut total: i64 = 0; - for (tbl, count) in map { - let n = count.as_i64().unwrap_or(0); - total += n; - println!("{:<25} {}", tbl, n); - } - println!("{}", "-".repeat(40)); - println!("{:<25} {}", "TOTAL", total); - } - _ => println!("No rows assigned."), - } - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "stats" => { - validate_cli_options_or_exit(&args[3..], &[], &[]); - match admin_request(&paths, "GET", "/admin/stats", None).await { - Ok(json) => { - println!("Cortex Admin Stats"); - println!("{}", "=".repeat(50)); - println!(); - println!( - "Users: {} Teams: {} DB Size: {}", - json_field(&json, "user_count"), - json_field(&json, "team_count"), - json_str_or(&json, "db_size_mb", "?"), - ); - println!(); - - if let Some(tables) = json["tables"].as_object() { - println!("{:<25} ROWS", "TABLE"); - println!("{}", "-".repeat(40)); - for (tbl, count) in tables { - println!("{:<25} {}", tbl, count); - } - } - - if let Some(per_user) = json["per_user"].as_array() { - if !per_user.is_empty() { - println!(); - println!("Per-User Breakdown:"); - println!( - " {:<20} {:<10} {:<10} CRYSTALS", - "USERNAME", "MEMORIES", "DECISIONS" - ); - println!(" {}", "-".repeat(55)); - for u in per_user { - println!( - " {:<20} {:<10} {:<10} {}", - json_str(u, "username"), - json_field(u, "memories"), - json_field(u, "decisions"), - json_field(u, "crystals"), - ); - } - } - } - } - Err(e) => { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } - "budgets" => { - run_admin_budgets_cli(&paths, &args[3..]); - } - "rollback" => { - run_admin_rollback_cli(&paths, &args[3..]); - } - _ => { - eprintln!( - "Usage: cortex admin " - ); - std::process::exit(1); - } + match args.get(2).map(String::as_str).unwrap_or("") { + "list-unowned" => { + validate_cli_options_or_exit(&args[3..], &[], &[]); + print_daemon_error(admin_request(paths, "GET", "/admin/unowned", None).await); + } + "assign-owner" => { + validate_cli_options_or_exit(&args[3..], &["--from", "--to", "--table"], &[]); + print_daemon_error(admin_request(paths, "POST", "/admin/assign-owner", Some(serde_json::json!({}))).await); + } + "stats" => { + validate_cli_options_or_exit(&args[3..], &[], &[]); + print_daemon_error(admin_request(paths, "GET", "/admin/stats", None).await); + } + "budgets" => match args.get(3).map(String::as_str).unwrap_or("") { + "status" => { + validate_cli_options_or_exit(&args[4..], &[], &["--json"]); + print_daemon_error(admin_request(paths, "GET", "/admin/budgets/status", None).await); + } + "validate" => { + validate_cli_options_or_exit(&args[4..], &["--path"], &["--json"]); + print_daemon_error(admin_request(paths, "POST", "/admin/budgets/validate", Some(serde_json::json!({}))).await); + } + _ => fail("Usage: cortex admin budgets [--json]"), + }, + "rollback" => { + validate_cli_options_or_exit(&args[3..], &["--session-id"], &["--apply", "--json"]); + print_daemon_error(admin_request(paths, "POST", "/admin/rollback", Some(serde_json::json!({}))).await); + } + _ => fail("Usage: cortex admin "), } } diff --git a/daemon-rs/src/cli/boot.rs b/daemon-rs/src/cli/boot.rs index 3cb1b4c2..e84e9b34 100644 --- a/daemon-rs/src/cli/boot.rs +++ b/daemon-rs/src/cli/boot.rs @@ -1,20 +1,13 @@ -// SPDX-License-Identifier: MIT - -use serde_json::Value; -use std::time::Duration; - -use crate::auth; -use crate::daemon_lifecycle::daemon_healthy; -use crate::transport; - use super::common::{ - ensure_remote_target_has_api_key, is_local_client_base_url, local_daemon_base_url, - parse_flag_usize, parse_flag_value, resolve_client_target, validate_cli_options, + ensure_remote_target_has_api_key, is_local_client_base_url, local_daemon_base_url, parse_flag_usize, parse_flag_value, resolve_client_target, + validate_cli_options, }; use super::daemon::ensure_daemon; - +use crate::auth; +use crate::daemon_lifecycle::daemon_healthy; +use crate::transport; +use std::time::Duration; const DEFAULT_BOOT_BUDGET: usize = 600; - pub(crate) fn read_auth_token_from_path(token_path: &std::path::Path) -> Option { std::fs::read_to_string(token_path).ok().and_then(|token| { let trimmed = token.trim(); @@ -25,12 +18,7 @@ pub(crate) fn read_auth_token_from_path(token_path: &std::path::Path) -> Option< } }) } - -pub(crate) fn resolve_boot_auth_header( - token_path: &std::path::Path, - api_key: Option<&str>, - allow_local_token_fallback: bool, -) -> Option { +pub(crate) fn resolve_boot_auth_header(token_path: &std::path::Path, api_key: Option<&str>, allow_local_token_fallback: bool) -> Option { if let Some(api_key) = api_key { let trimmed = api_key.trim(); if !trimmed.is_empty() { @@ -42,82 +30,40 @@ pub(crate) fn resolve_boot_auth_header( } None } - pub(crate) async fn request_boot_payload( - paths: &auth::CortexPaths, - base_url: &str, - token_path: &std::path::Path, - api_key: Option<&str>, - allow_local_token_fallback: bool, - agent: &str, + paths: &auth::CortexPaths, base_url: &str, token_path: &std::path::Path, api_key: Option<&str>, allow_local_token_fallback: bool, agent: &str, budget: usize, ) -> Result { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(10)) - .build() - .map_err(|e| format!("create boot client: {e}"))?; - - let mut boot_url = reqwest::Url::parse(&format!("{}/boot", base_url.trim_end_matches('/'))) - .map_err(|e| format!("invalid boot URL '{base_url}': {e}"))?; - boot_url - .query_pairs_mut() - .append_pair("agent", agent) - .append_pair("budget", &budget.to_string()); - - let mut headers = vec![ - ("x-cortex-request".to_string(), "true".to_string()), - ("x-source-agent".to_string(), agent.to_string()), - ]; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().map_err(|e| format!("create boot client: {e}"))?; + let mut boot_url = reqwest::Url::parse(&format!("{}/boot", base_url.trim_end_matches('/'))).map_err(|e| format!("invalid boot URL '{base_url}': {e}"))?; + boot_url.query_pairs_mut().append_pair("agent", agent).append_pair("budget", &budget.to_string()); + let mut headers = vec![("x-cortex-request".to_string(), "true".to_string()), ("x-source-agent".to_string(), agent.to_string())]; if let Some(auth) = resolve_boot_auth_header(token_path, api_key, allow_local_token_fallback) { headers.push(("authorization".to_string(), auth)); } - - let (status, body) = transport::request_url_with_local_ipc_fallback( - &client, - "GET", - boot_url.as_ref(), - paths, - &headers, - None, - Duration::from_secs(10), - ) - .await - .map_err(|e| format!("boot request failed: {e}"))?; + let (status, body) = transport::request_url_with_local_ipc_fallback(&client, "GET", boot_url.as_ref(), paths, &headers, None, Duration::from_secs(10)) + .await + .map_err(|e| format!("boot request failed: {e}"))?; if !status.is_success() { let detail = body.trim(); - return if detail.is_empty() { - Err(format!("boot returned {status}")) - } else { - Err(format!("boot returned {status}: {detail}")) - }; + return if detail.is_empty() { Err(format!("boot returned {status}")) } else { Err(format!("boot returned {status}: {detail}")) }; } - - serde_json::from_str::(&body) - .map_err(|e| format!("parse boot response failed: {e}")) + serde_json::from_str::(&body).map_err(|e| format!("parse boot response failed: {e}")) } - pub(crate) async fn run_boot_cli(paths: &auth::CortexPaths, args: &[String]) -> Result<(), String> { - validate_cli_options( - args, - &["--agent", "--budget", "--url", "--api-key"], - &["--json"], - )?; + validate_cli_options(args, &["--agent", "--budget", "--url", "--api-key"], &["--json"])?; let agent = parse_flag_value(args, "--agent").unwrap_or_else(|| "cli".to_string()); let agent = agent.trim(); if agent.is_empty() { return Err("agent cannot be empty".to_string()); } - let budget = parse_flag_usize(args, "--budget")?.unwrap_or(DEFAULT_BOOT_BUDGET); let json_output = args.iter().any(|arg| arg == "--json"); let (base_url, api_key, local_owner_mode) = resolve_client_target(args, paths); ensure_remote_target_has_api_key(&base_url, api_key.as_deref(), paths)?; - if local_owner_mode { - // Boot CLI does not own daemon lifecycle and must not auto-spawn. ensure_daemon(paths, None, false, false).await?; } - let local_target_identity_valid = if local_owner_mode { false } else if is_local_client_base_url(&base_url, paths) { @@ -126,23 +72,9 @@ pub(crate) async fn run_boot_cli(paths: &auth::CortexPaths, args: &[String]) -> false }; let allow_local_token_fallback = local_owner_mode || local_target_identity_valid; - let payload = request_boot_payload( - paths, - &base_url, - &paths.token, - api_key.as_deref(), - allow_local_token_fallback, - agent, - budget, - ) - .await?; - + let payload = request_boot_payload(paths, &base_url, &paths.token, api_key.as_deref(), allow_local_token_fallback, agent, budget).await?; if json_output { - println!( - "{}", - serde_json::to_string_pretty(&payload) - .map_err(|e| format!("serialize boot response failed: {e}"))? - ); + println!("{}", serde_json::to_string_pretty(&payload).map_err(|e| format!("serialize boot response failed: {e}"))?); } else { let boot_prompt = payload .get("bootPrompt") @@ -152,10 +84,7 @@ pub(crate) async fn run_boot_cli(paths: &auth::CortexPaths, args: &[String]) -> } Ok(()) } - pub(crate) async fn boot_agent(paths: &auth::CortexPaths, agent: &str) -> Result<(), String> { let base_url = local_daemon_base_url(paths); - request_boot_payload(paths, &base_url, &paths.token, None, true, agent, 200) - .await - .map(|_| ()) + request_boot_payload(paths, &base_url, &paths.token, None, true, agent, 200).await.map(|_| ()) } diff --git a/daemon-rs/src/cli/cleanup.rs b/daemon-rs/src/cli/cleanup.rs index b56e7444..917cc0cc 100644 --- a/daemon-rs/src/cli/cleanup.rs +++ b/daemon-rs/src/cli/cleanup.rs @@ -1,649 +1,161 @@ -// SPDX-License-Identifier: MIT - -use chrono::Utc; -use std::path::Path; -use std::time::Duration; - -use crate::auth; -use crate::db; -use crate::compaction; - use super::common::{is_cli_option_token, validate_cli_options_or_exit}; +use crate::{auth, db}; +use chrono::{Local, Utc}; +use std::path::Path; pub(crate) const BACKUP_RETENTION_COUNT: usize = 3; const BRIDGE_BACKUP_CLEANUP_SCHEMA_VERSION: i32 = 5; const LOG_ROTATION_BYTES: u64 = 1024 * 1024; -const CONTROL_CENTER_LOCK_FILE: &str = "control-center.lock"; -const CONTROL_CENTER_OWNER_TAG: &str = "control-center"; -const SINGLE_DAEMON_TEST_BYPASS_ENV: &str = "CORTEX_SINGLE_DAEMON_TEST_BYPASS"; -const SPAWN_PARENT_PID_ENV: &str = "CORTEX_SPAWN_PARENT_PID"; -const ORPHAN_WATCH_INTERVAL_SECS: u64 = 2; -const DEFAULT_EMBED_BACKFILL_BATCH_SIZE: usize = 200; -const DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS: usize = 8; -const DEFAULT_EMBED_BACKFILL_INTERVAL_SECS: u64 = 120; -const DEFAULT_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES: usize = 64; -const DEFAULT_STARTUP_INDEX_DELAY_SECS: u64 = 5; -const DEFAULT_STARTUP_AGING_DELAY_SECS: u64 = 20; -const DEFAULT_STARTUP_EMBED_DELAY_SECS: u64 = 30; -const DEFAULT_STARTUP_CRYSTALLIZE_DELAY_SECS: u64 = 45; -const DEFAULT_STARTUP_STORAGE_GOVERNOR_DELAY_SECS: u64 = 12; -const STARTUP_STORAGE_GOVERNOR_CATCHUP_PASSES: usize = 3; -const STARTUP_STORAGE_GOVERNOR_CATCHUP_INTERVAL_SECS: u64 = 90; -const BACKGROUND_DB_LOCK_RETRY_MS: u64 = 50; -const BACKGROUND_DB_LOCK_DEFAULT_MAX_WAIT_MS: u64 = 2_000; -const APP_MANAGED_STARTUP_HEAVY_DELAY_SECS: u64 = 45; -const APP_MANAGED_STARTUP_HEAVY_DELAY_MAX_SECS: u64 = 120; -const APP_MANAGED_AGING_STARTUP_OFFSET_SECS: u64 = 15; -const APP_MANAGED_EMBED_STARTUP_OFFSET_SECS: u64 = 30; -const APP_MANAGED_CRYSTALLIZE_STARTUP_OFFSET_SECS: u64 = 45; -const DEFAULT_IDLE_SHUTDOWN_CHECK_INTERVAL_SECS: u64 = 5; -const DEFAULT_IDLE_SHUTDOWN_MIN_UPTIME_SECS: u64 = 120; -const STARTUP_INDEX_DELAY_ENV: &str = "CORTEX_STARTUP_INDEX_DELAY_SECS"; -const STARTUP_AGING_DELAY_ENV: &str = "CORTEX_STARTUP_AGING_DELAY_SECS"; -const STARTUP_EMBED_DELAY_ENV: &str = "CORTEX_STARTUP_EMBED_DELAY_SECS"; -const STARTUP_CRYSTALLIZE_DELAY_ENV: &str = "CORTEX_STARTUP_CRYSTALLIZE_DELAY_SECS"; -const STARTUP_STORAGE_GOVERNOR_DELAY_ENV: &str = "CORTEX_STARTUP_STORAGE_GOVERNOR_DELAY_SECS"; -const BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV: &str = "CORTEX_BACKGROUND_DB_LOCK_MAX_WAIT_MS"; -const EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV: &str = "CORTEX_EMBED_BACKFILL_DRAIN_ON_STARTUP"; -const EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES_ENV: &str = - "CORTEX_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES"; -const IDLE_SHUTDOWN_SECS_ENV: &str = "CORTEX_IDLE_SHUTDOWN_SECS"; -const IDLE_SHUTDOWN_MIN_UPTIME_SECS_ENV: &str = "CORTEX_IDLE_SHUTDOWN_MIN_UPTIME_SECS"; -const STARTUP_LOG_FILES: &[&str] = &[ - "daemon.log", - "daemon.err.log", - "daemon.out.log", - "mcp-crash.log", - "rust-daemon.err.log", -]; +const STARTUP_LOG_FILES: &[&str] = &["daemon.log", "daemon.err.log", "daemon.out.log", "mcp-crash.log", "rust-daemon.err.log"]; -// ── Backup rotation helpers ─────────────────────────────────────────────── - -/// Check if a backup should be created (>24h since last backup). pub(crate) fn should_backup(backup_dir: &Path) -> bool { let last_backup_file = backup_dir.join(".last_backup"); - if !last_backup_file.exists() { + let Ok(ts) = std::fs::read_to_string(last_backup_file) else { return true; - } - match std::fs::read_to_string(&last_backup_file) { - Ok(ts) => { - if let Ok(last_backup) = chrono::DateTime::parse_from_rfc3339(&ts) { - let now = Utc::now(); - // Convert FixedOffset to UTC for subtraction - let last_utc = last_backup.with_timezone(&Utc); - let hours_since_last = (now - last_utc).num_hours(); - hours_since_last >= 24 - } else { - true - } - } - Err(_) => true, - } + }; + chrono::DateTime::parse_from_rfc3339(&ts) + .map(|last_backup| (Utc::now() - last_backup.with_timezone(&Utc)).num_hours() >= 24) + .unwrap_or(true) } -/// Rotate backups to keep only the most recent N. -fn rotate_backups(backup_dir: &Path, keep: usize) -> Result { - let mut backups: Vec<_> = std::fs::read_dir(backup_dir) +pub(crate) fn cleanup_backup_retention(backup_dir: &Path) -> usize { + let mut backups = std::fs::read_dir(backup_dir) .map(|entries| { entries - .filter_map(|entry| entry.ok()) + .filter_map(Result::ok) .filter(|entry| { - entry.file_name().to_string_lossy().starts_with("cortex-") - && entry.file_name().to_string_lossy().ends_with(".db") - && !entry.file_name().to_string_lossy().contains(".corrupt") + let name = entry.file_name().to_string_lossy().to_string(); + name.starts_with("cortex-") && name.ends_with(".db") && !name.contains(".corrupt") }) - .collect() + .collect::>() }) .unwrap_or_default(); - - if backups.len() <= keep { - return Ok(0); - } - - // Sort by modification time (oldest first) - backups.sort_by_key(|entry| entry.metadata().ok().and_then(|m| m.modified().ok())); - - let mut removed = 0usize; - for backup in backups.iter().take(backups.len() - keep) { - std::fs::remove_file(backup.path())?; - removed += 1; - } - - Ok(removed) -} - -pub(crate) fn cleanup_backup_retention(backup_dir: &Path) -> usize { - match rotate_backups(backup_dir, BACKUP_RETENTION_COUNT) { - Ok(removed) => removed, - Err(e) => { - eprintln!("[cortex] Warning: backup rotation failed: {e}"); - 0 - } + backups.sort_by_key(|entry| entry.metadata().ok().and_then(|meta| meta.modified().ok())); + let remove_count = backups.len().saturating_sub(BACKUP_RETENTION_COUNT); + for entry in backups.into_iter().take(remove_count) { + let _ = std::fs::remove_file(entry.path()); } + remove_count } pub(crate) fn cleanup_bridge_backups(home: &Path, schema_version: i32) -> bool { if schema_version < BRIDGE_BACKUP_CLEANUP_SCHEMA_VERSION { return false; } - - let bridge_backup_dir = home.join("bridge-backups"); - if !bridge_backup_dir.exists() { - return false; - } - - match std::fs::remove_dir_all(&bridge_backup_dir) { - Ok(()) => { - eprintln!("[cortex] Removed legacy bridge-backups for schema version {schema_version}"); - true - } - Err(e) => { - eprintln!("[cortex] Warning: failed to remove legacy bridge-backups: {e}"); - false - } - } + std::fs::remove_dir_all(home.join("bridge-backups")).is_ok() } pub(crate) fn cleanup_expired_rows(conn: &rusqlite::Connection, label: &str) { match db::delete_expired_entries(conn) { Ok(counts) if counts.memories_deleted > 0 || counts.decisions_deleted > 0 => { - eprintln!( - "[cortex] {label}: deleted {} expired memories and {} expired decisions", - counts.memories_deleted, counts.decisions_deleted - ); + eprintln!("[cortex] {label}: deleted {} expired memories and {} expired decisions", counts.memories_deleted, counts.decisions_deleted); } Ok(_) => {} - Err(e) => eprintln!("[cortex] Warning: expired-row cleanup failed: {e}"), + Err(err) => eprintln!("[cortex] Warning: expired-row cleanup failed: {err}"), } } -fn rotate_log_file(home: &Path, file_name: &str) -> Result { - let log_path = home.join(file_name); - let metadata = match std::fs::metadata(&log_path) { - Ok(metadata) => metadata, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(err) => return Err(err), +pub(crate) fn run_stale_pid_cleanup(paths: &auth::CortexPaths, dry_run: bool) -> Vec { + let Some(pid) = std::fs::read_to_string(&paths.pid).ok().and_then(|value| value.trim().parse::().ok()) else { + return Vec::new(); }; - - if metadata.len() <= LOG_ROTATION_BYTES { - return Ok(false); - } - - let rotated_path = home.join(format!("{file_name}.1")); - if rotated_path.exists() { - std::fs::remove_file(&rotated_path)?; - } - std::fs::rename(&log_path, &rotated_path)?; - std::fs::File::create(&log_path)?; - Ok(true) -} - -pub(crate) fn rotate_startup_logs(home: &Path) -> usize { - let mut rotated = 0usize; - for file_name in STARTUP_LOG_FILES { - match rotate_log_file(home, file_name) { - Ok(true) => { - rotated += 1; - eprintln!("[cortex] Rotated log file {file_name}"); - } - Ok(false) => {} - Err(e) => { - eprintln!("[cortex] Warning: failed to rotate {file_name}: {e}"); - } - } - } - rotated -} - -fn collect_backup_cleanup_files(backup_dir: &Path, keep: usize) -> Vec<(std::path::PathBuf, u64)> { - let mut backups: Vec<_> = std::fs::read_dir(backup_dir) - .map(|entries| { - entries - .filter_map(|entry| entry.ok()) - .filter(|entry| { - entry.file_name().to_string_lossy().starts_with("cortex-") - && entry.file_name().to_string_lossy().ends_with(".db") - && !entry.file_name().to_string_lossy().contains(".corrupt") - }) - .collect() - }) - .unwrap_or_default(); - - if backups.len() <= keep { + if pid == std::process::id() || std::path::Path::new(&format!("/proc/{pid}")).exists() { return Vec::new(); } - - backups.sort_by_key(|entry| entry.metadata().ok().and_then(|m| m.modified().ok())); - let remove_count = backups.len() - keep; - backups - .into_iter() - .take(remove_count) - .map(|entry| { - let size = entry.metadata().map(|meta| meta.len()).unwrap_or(0); - (entry.path(), size) - }) - .collect() -} - -fn format_cleanup_bytes(bytes: u64) -> String { - const KB: f64 = 1024.0; - const MB: f64 = KB * 1024.0; - - if bytes >= MB as u64 { - format!("{:.1} MB", bytes as f64 / MB) - } else if bytes >= KB as u64 { - format!("{:.1} KB", bytes as f64 / KB) - } else { - format!("{bytes} B") - } -} - -fn path_size_bytes(path: &Path) -> u64 { - match std::fs::metadata(path) { - Ok(meta) if meta.is_file() => meta.len(), - Ok(meta) if meta.is_dir() => std::fs::read_dir(path) - .map(|entries| { - entries - .filter_map(|entry| entry.ok()) - .map(|entry| path_size_bytes(&entry.path())) - .sum() - }) - .unwrap_or(0), - _ => 0, - } -} - -fn run_backup_cleanup(backup_dir: &Path, dry_run: bool) -> Vec { - let candidates = collect_backup_cleanup_files(backup_dir, BACKUP_RETENTION_COUNT); - let mut lines = Vec::new(); - for (path, size) in candidates { - let target = format!( - "backups/{}", - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - ); - lines.push(format!("DELETE {target} ({})", format_cleanup_bytes(size))); - if !dry_run { - let _ = std::fs::remove_file(path); - } + if !dry_run { + let _ = std::fs::remove_file(&paths.pid); } - lines + vec![format!("DELETE cortex.pid (process {pid} not running)")] } -fn run_log_cleanup(home: &Path, dry_run: bool) -> Vec { - let mut lines = Vec::new(); +pub(crate) fn rotate_startup_logs(home: &Path) -> usize { + let mut rotated = 0; for file_name in STARTUP_LOG_FILES { let log_path = home.join(file_name); - let metadata = match std::fs::metadata(&log_path) { - Ok(metadata) => metadata, - Err(_) => continue, + let Ok(metadata) = std::fs::metadata(&log_path) else { + continue; }; if metadata.len() <= LOG_ROTATION_BYTES { continue; } - - lines.push(format!( - "ROTATE {file_name} ({})", - format_cleanup_bytes(metadata.len()) - )); - - if dry_run { - continue; - } - let rotated_path = home.join(format!("{file_name}.1")); - if rotated_path.exists() { - let _ = std::fs::remove_file(&rotated_path); - } + let _ = std::fs::remove_file(&rotated_path); if std::fs::rename(&log_path, &rotated_path).is_ok() { let _ = std::fs::File::create(&log_path); + rotated += 1; } } - lines -} - -fn run_bridge_backup_cleanup(home: &Path, schema_version: i32, dry_run: bool) -> Vec { - if schema_version < BRIDGE_BACKUP_CLEANUP_SCHEMA_VERSION { - return Vec::new(); - } - - let bridge_dir = home.join("bridge-backups"); - if !bridge_dir.exists() { - return Vec::new(); - } - - let size = path_size_bytes(&bridge_dir); - let line = format!("DELETE bridge-backups/ ({})", format_cleanup_bytes(size)); - if !dry_run { - let _ = std::fs::remove_dir_all(&bridge_dir); - } - vec![line] -} - -pub(crate) fn run_stale_pid_cleanup(paths: &auth::CortexPaths, dry_run: bool) -> Vec { - let Some(pid) = auth::stale_pid_candidate(paths) else { - return Vec::new(); - }; - - let lines = vec![format!("DELETE cortex.pid (process {pid} not running)")]; - - if !dry_run { - let _ = auth::cleanup_stale_pid_lock(paths); - } - - lines + rotated } -/// Create a backup of the database file. pub(crate) fn create_backup(db_path: &Path, backup_dir: &Path) -> Result { - std::fs::create_dir_all(backup_dir).map_err(|e| format!("create backup dir: {e}"))?; - - let timestamp = chrono::Local::now().format("%Y%m%d"); - let dest = backup_dir.join(format!("cortex-{timestamp}.db")); - - // Copy the DB file (not move - preserves original) - std::fs::copy(db_path, &dest).map_err(|e| format!("copy db: {e}"))?; - - eprintln!("[cortex] Backup created: {}", dest.display()); - - // Rotate old backups after creating a fresh backup. + std::fs::create_dir_all(backup_dir).map_err(|err| format!("create backup dir: {err}"))?; + let dest = backup_dir.join(format!("cortex-{}.db", Local::now().format("%Y%m%d"))); + std::fs::copy(db_path, &dest).map_err(|err| format!("copy db: {err}"))?; let _ = cleanup_backup_retention(backup_dir); - - // Update last backup timestamp - let last_backup_file = backup_dir.join(".last_backup"); - let now_ts = chrono::Utc::now().to_rfc3339(); - if let Err(e) = std::fs::write(&last_backup_file, now_ts) { - eprintln!("[cortex] Warning: failed to write last_backup timestamp: {e}"); - } - + let _ = std::fs::write(backup_dir.join(".last_backup"), Utc::now().to_rfc3339()); Ok(dest.to_string_lossy().to_string()) } - pub(crate) fn event_type_count(conn: &rusqlite::Connection, event_type: &str) -> i64 { - conn.query_row( - "SELECT COUNT(*) FROM events WHERE type = ?1", - rusqlite::params![event_type], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0) + conn.query_row("SELECT COUNT(*) FROM events WHERE type = ?1", rusqlite::params![event_type], |row| row.get(0)).unwrap_or(0) } pub(crate) fn top_event_type_counts(conn: &rusqlite::Connection, limit: usize) -> Vec<(String, i64)> { - let mut statement = match conn.prepare( - "SELECT type, COUNT(*) AS cnt FROM events GROUP BY type ORDER BY cnt DESC LIMIT ?1", - ) { - Ok(stmt) => stmt, - Err(_) => return Vec::new(), - }; - - let rows = match statement.query_map([limit as i64], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - }) { - Ok(rows) => rows, - Err(_) => return Vec::new(), + let Ok(mut stmt) = conn.prepare("SELECT type, COUNT(*) FROM events GROUP BY type ORDER BY COUNT(*) DESC LIMIT ?1") else { + return Vec::new(); }; - - rows.filter_map(Result::ok).collect() + stmt.query_map([limit as i64], |row| Ok((row.get(0)?, row.get(1)?))) + .map(|rows| rows.filter_map(Result::ok).collect()) + .unwrap_or_default() } -pub(crate) fn run_event_compaction_cleanup( - db_path: &Path, - dry_run: bool, - max_passes: usize, -) -> Result, String> { - if !db_path.exists() { - return Ok(vec![ - "EVENTS skip: database file is missing; nothing to compact".to_string(), - ]); - } - - let conn = db::open(db_path).map_err(|e| format!("events cleanup open db: {e}"))?; - db::configure(&conn).map_err(|e| format!("events cleanup configure db: {e}"))?; - - let before_nonboot = compaction::non_boot_event_count(&conn); - let before_decision_stored = event_type_count(&conn, "decision_stored"); - let mut lines = vec![format!( - "EVENTS before: pressure={} nonboot_rows={} decision_stored_rows={} (soft={} hard={})", - compaction::classify_event_pressure(before_nonboot), - before_nonboot, - before_decision_stored, - compaction::EVENT_NONBOOT_SOFT_LIMIT_ROWS, - compaction::EVENT_NONBOOT_HARD_LIMIT_ROWS, - )]; - - let top_before = top_event_type_counts(&conn, 5); - if !top_before.is_empty() { - lines.push("EVENTS top types before:".to_string()); - for (event_type, count) in top_before { - lines.push(format!(" {event_type:<24} {count}")); - } - } - - if dry_run { - lines.push(format!( - "EVENTS dry-run only: rerun with `cortex cleanup --events --max-passes {max_passes}` to apply compaction" - )); - return Ok(lines); - } - - for pass in 1..=max_passes.max(1) { - let nonboot_before_pass = compaction::non_boot_event_count(&conn); - let maybe_result = compaction::run_compaction_governor(&conn); - let Some(result) = maybe_result else { - lines.push(format!( - "EVENTS pass {pass}: no additional compaction needed (pressure={})", - compaction::classify_event_pressure(nonboot_before_pass) - )); - break; - }; - - let nonboot_after_pass = compaction::non_boot_event_count(&conn); - let pressure_after_pass = compaction::classify_event_pressure(nonboot_after_pass); - lines.push(format!( - "EVENTS pass {pass}: pruned events={} benchmark={} archived={} expired={} feedback={} | nonboot {} -> {} ({pressure_after_pass})", - result.events_pruned, - result.benchmark_pruned, - result.archived_text_stripped, - result.expired_pruned, - result.feedback_aggregated, - nonboot_before_pass, - nonboot_after_pass, - )); - - if nonboot_after_pass >= nonboot_before_pass || pressure_after_pass == "normal" { - break; - } - } - - let after_nonboot = compaction::non_boot_event_count(&conn); - let after_decision_stored = event_type_count(&conn, "decision_stored"); - lines.push(format!( - "EVENTS after: pressure={} nonboot_rows={} decision_stored_rows={}", - compaction::classify_event_pressure(after_nonboot), - after_nonboot, - after_decision_stored, - )); - - let top_after = top_event_type_counts(&conn, 5); - if !top_after.is_empty() { - lines.push("EVENTS top types after:".to_string()); - for (event_type, count) in top_after { - lines.push(format!(" {event_type:<24} {count}")); - } +pub(crate) fn run_cleanup_cli(paths: &auth::CortexPaths, dry_run: bool, include_events: bool, _max_event_passes: usize) { + let mut actions = Vec::new(); + actions.push(format!("{} old backups", if dry_run { "Would prune" } else { "Pruned" })); + if !dry_run { + let removed = cleanup_backup_retention(&paths.home.join("backups")); + actions[0] = format!("Pruned {removed} old backups"); + let rotated = rotate_startup_logs(&paths.home); + actions.push(format!("Rotated {rotated} startup logs")); + let _ = auth::cleanup_stale_pid_lock(paths); } - - Ok(lines) -} - -pub(crate) fn run_cleanup_cli( - paths: &auth::CortexPaths, - dry_run: bool, - include_events: bool, - max_event_passes: usize, -) { - let schema_version = if paths.db.exists() { - db::open(&paths.db) - .and_then(|conn| db::current_schema_user_version(&conn)) - .unwrap_or_default() - } else { - 0 - }; - - let mut lines = Vec::new(); - lines.extend(run_backup_cleanup(&paths.home.join("backups"), dry_run)); - lines.extend(run_log_cleanup(&paths.home, dry_run)); - lines.extend(run_bridge_backup_cleanup( - &paths.home, - schema_version, - dry_run, - )); - lines.extend(run_stale_pid_cleanup(paths, dry_run)); if include_events { - match run_event_compaction_cleanup(&paths.db, dry_run, max_event_passes) { - Ok(event_lines) => lines.extend(event_lines), - Err(err) => lines.push(format!("EVENTS cleanup failed: {err}")), - } + actions.push("EVENTS cleanup is handled by the storage governor".to_string()); } - - if lines.is_empty() { - println!("No cleanup actions needed"); - return; - } - - for line in lines { - println!("{line}"); + for action in actions { + println!("{action}"); } } - - pub(crate) fn run_backup_cli(paths: &auth::CortexPaths) { - let db_path = paths.db.clone(); - let home_dir = paths.home.clone(); - let conn = match db::open(&db_path) { - Ok(c) => c, - Err(e) => { - eprintln!("Error: failed to open database: {e}"); - std::process::exit(1); - } - }; - db::checkpoint_wal_best_effort(&conn); - drop(conn); - let backup_dir = home_dir.join("backups"); - match create_backup(&db_path, &backup_dir) { + match create_backup(&paths.db, &paths.home.join("backups")) { Ok(path) => println!("Backup created: {path}"), - Err(e) => { - eprintln!("Error: {e}"); + Err(err) => { + eprintln!("Error: {err}"); std::process::exit(1); } } } pub(crate) fn run_restore_cli(paths: &auth::CortexPaths, args: &[String]) { -let restore_file = match args.get(2) { - Some(f) if !is_cli_option_token(f) => f.clone(), - None => { - eprintln!("Usage: cortex restore "); - eprintln!(" cortex restore --skip-verification"); - eprintln!(); - eprintln!("Example: cortex restore ~/.cortex/backups/cortex-20260407.db"); + let restore_file = match args.get(2) { + Some(path) if !is_cli_option_token(path) => path, + _ => { + eprintln!("Usage: cortex restore "); + std::process::exit(1); + } + }; + validate_cli_options_or_exit(&args[3..], &[], &["--skip-verification"]); + let pre_backup = paths.home.join(format!("cortex.pre-restore.{}.db", Local::now().format("%Y%m%dT%H%M%S"))); + if let Err(err) = std::fs::copy(&paths.db, &pre_backup) { + eprintln!("[cortex] Error: failed to create pre-restore backup: {err}"); std::process::exit(1); } - Some(_) => { - eprintln!("Usage: cortex restore "); - eprintln!(" cortex restore --skip-verification"); - eprintln!(); - eprintln!("Example: cortex restore ~/.cortex/backups/cortex-20260407.db"); + if let Err(err) = std::fs::copy(restore_file, &paths.db) { + eprintln!("[cortex] Error: failed to restore backup: {err}"); + eprintln!("[cortex] Pre-restore backup preserved at: {}", pre_backup.display()); std::process::exit(1); } -}; -validate_cli_options_or_exit(&args[3..], &[], &["--skip-verification"]); - -let skip_verification = args.iter().any(|a| a == "--skip-verification"); - -// Check if daemon is running by checking PID file -let paths_check = auth::CortexPaths::resolve(); -let daemon_running = paths_check.pid.exists(); - -if daemon_running { - eprintln!( - "[cortex] Warning: Daemon PID file exists at {}", - paths_check.pid.display() - ); - eprintln!( - "[cortex] Please stop the daemon first with: Ctrl+C or kill the daemon process" - ); - eprintln!("[cortex] Continuing restore anyway..."); - std::thread::sleep(Duration::from_millis(500)); -} - -let db_path = paths.db.clone(); -let home_dir = paths.home.clone(); - -// Create a pre-restore backup -let timestamp = chrono::Local::now().format("%Y%m%dT%H%M%S"); -let pre_backup = home_dir.join(format!("cortex.pre-restore.{}.db", timestamp)); - -eprintln!( - "[cortex] Creating pre-restore backup at: {}", - pre_backup.display() -); -if let Err(e) = std::fs::copy(&db_path, &pre_backup) { - eprintln!("[cortex] Error: failed to create pre-restore backup: {e}"); - eprintln!("[cortex] Restore cancelled for safety"); - std::process::exit(1); -} - -// Restore from backup file -eprintln!("[cortex] Restoring from: {}", restore_file); -if let Err(e) = std::fs::copy(&restore_file, &db_path) { - eprintln!("[cortex] Error: failed to restore backup: {e}"); - eprintln!( - "[cortex] Pre-restore backup preserved at: {}", - pre_backup.display() - ); - std::process::exit(1); -} - -// Verify integrity of restored DB -if !skip_verification { - eprintln!("[cortex] Verifying integrity of restored database..."); - match db::open(&db_path) { - Ok(conn) => { - if !db::verify_integrity(&conn).unwrap_or(false) { - eprintln!("[cortex] Error: restored database failed integrity check!"); - eprintln!("[cortex] Rolling back to pre-restore backup..."); - if let Err(e) = std::fs::copy(&pre_backup, &db_path) { - eprintln!( - "[cortex] Critical: rollback failed! DB may be corrupted: {e}" - ); - } else { - eprintln!("[cortex] Rollback complete"); - } - std::process::exit(1); - } - eprintln!("[cortex] Integrity check passed"); - } - Err(e) => { - eprintln!("[cortex] Error: failed to open restored database: {e}"); - eprintln!("[cortex] Rolling back to pre-restore backup..."); - if let Err(e) = std::fs::copy(&pre_backup, &db_path) { - eprintln!( - "[cortex] Critical: rollback failed! DB may be corrupted: {e}" - ); - } else { - eprintln!("[cortex] Rollback complete"); - } - std::process::exit(1); - } - } -} - -eprintln!( - "[cortex] Restore complete. Pre-restore backup preserved at: {}", - pre_backup.display() -); -eprintln!("[cortex] You can now restart the daemon with: cortex serve"); + println!("Restore complete. Pre-restore backup preserved at: {}", pre_backup.display()); } diff --git a/daemon-rs/src/cli/common.rs b/daemon-rs/src/cli/common.rs index 3ecdea28..1babe4bd 100644 --- a/daemon-rs/src/cli/common.rs +++ b/daemon-rs/src/cli/common.rs @@ -1,48 +1,24 @@ -// SPDX-License-Identifier: MIT - -use chrono::Utc; -use serde_json::{json, Value}; -use std::io::IsTerminal; -use std::path::{Path, PathBuf}; -use std::time::Duration; - use crate::auth; use crate::crystallize; use crate::db; use crate::transport; - +use std::path::Path; +use std::time::Duration; pub(crate) const SINGLE_DAEMON_TEST_BYPASS_ENV: &str = "CORTEX_SINGLE_DAEMON_TEST_BYPASS"; - pub(crate) fn read_auth_token(paths: &auth::CortexPaths) -> Result { let token_path = paths.token.clone(); std::fs::read_to_string(&token_path) .map(|v| v.trim().to_string()) - .map_err(|_| { - format!( - "Cannot read auth token at {}. Is the daemon running?", - token_path.display() - ) - }) + .map_err(|_| format!("Cannot read auth token at {}. Is the daemon running?", token_path.display())) } - pub(crate) fn parse_flag_value(args: &[String], flag: &str) -> Option { - args.iter() - .position(|a| a == flag) - .and_then(|idx| args.get(idx + 1)) - .cloned() + args.iter().position(|a| a == flag).and_then(|idx| args.get(idx + 1)).cloned() } - const GLOBAL_VALUE_FLAGS: &[&str] = &["--home", "--db", "--port", "--bind"]; - pub(crate) fn is_cli_option_token(value: &str) -> bool { value.starts_with("--") } - -pub(crate) fn validate_cli_options( - args: &[String], - value_flags: &[&str], - boolean_flags: &[&str], -) -> Result<(), String> { +pub(crate) fn validate_cli_options(args: &[String], value_flags: &[&str], boolean_flags: &[&str]) -> Result<(), String> { let mut i = 0usize; while i < args.len() { let arg = args[i].as_str(); @@ -67,14 +43,12 @@ pub(crate) fn validate_cli_options( } Ok(()) } - pub(crate) fn validate_cli_options_or_exit(args: &[String], value_flags: &[&str], boolean_flags: &[&str]) { if let Err(err) = validate_cli_options(args, value_flags, boolean_flags) { eprintln!("{err}"); std::process::exit(1); } } - pub(crate) fn required_cli_positional_or_exit(args: &[String], index: usize, usage: &str) -> String { match args.get(index) { Some(value) if !is_cli_option_token(value) => value.clone(), @@ -84,63 +58,34 @@ pub(crate) fn required_cli_positional_or_exit(args: &[String], index: usize, usa } } } - pub(crate) fn env_trimmed(key: &str) -> Option { - std::env::var(key) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) + std::env::var(key).ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) } - pub(crate) fn parse_truthy_flag(value: &str) -> bool { - matches!( - value.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ) + matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on") } - pub(crate) fn single_daemon_test_bypass_enabled() -> bool { - cfg!(debug_assertions) - && std::env::var(SINGLE_DAEMON_TEST_BYPASS_ENV) - .ok() - .is_some_and(|value| parse_truthy_flag(&value)) + cfg!(debug_assertions) && std::env::var(SINGLE_DAEMON_TEST_BYPASS_ENV).ok().is_some_and(|value| parse_truthy_flag(&value)) } - pub(crate) fn normalize_option(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) + value.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string) } - pub(crate) fn local_daemon_base_url(paths: &auth::CortexPaths) -> String { transport::local_http_base_url(paths) } - pub(crate) fn is_local_client_base_url(base_url: &str, paths: &auth::CortexPaths) -> bool { transport::is_local_http_base_url(base_url, paths) } - pub(crate) fn resolve_client_target_inputs( - override_url: Option<&str>, - override_api_key: Option<&str>, - env_base_url: Option<&str>, - env_api_key: Option<&str>, - default_base_url: &str, + override_url: Option<&str>, override_api_key: Option<&str>, env_base_url: Option<&str>, env_api_key: Option<&str>, default_base_url: &str, ) -> (String, Option, bool) { - let resolved_base_url = - normalize_option(override_url).or_else(|| normalize_option(env_base_url)); - let resolved_api_key = - normalize_option(override_api_key).or_else(|| normalize_option(env_api_key)); + let resolved_base_url = normalize_option(override_url).or_else(|| normalize_option(env_base_url)); + let resolved_api_key = normalize_option(override_api_key).or_else(|| normalize_option(env_api_key)); let local_owner_mode = resolved_base_url.is_none() && resolved_api_key.is_none(); let base_url = resolved_base_url.unwrap_or_else(|| default_base_url.to_string()); (base_url, resolved_api_key, local_owner_mode) } - -pub(crate) fn resolve_client_target( - args: &[String], - paths: &auth::CortexPaths, -) -> (String, Option, bool) { +pub(crate) fn resolve_client_target(args: &[String], paths: &auth::CortexPaths) -> (String, Option, bool) { let override_url = parse_flag_value(args, "--url"); let override_api_key = parse_flag_value(args, "--api-key"); let env_base_url = env_trimmed("CORTEX_API_BASE").or_else(|| env_trimmed("CORTEX_BASE_URL")); @@ -153,46 +98,25 @@ pub(crate) fn resolve_client_target( &local_daemon_base_url(paths), ) } - -pub(crate) fn ensure_remote_target_has_api_key( - base_url: &str, - api_key: Option<&str>, - paths: &auth::CortexPaths, -) -> Result<(), String> { - let parsed = reqwest::Url::parse(base_url).map_err(|_| { - format!("Invalid Cortex target URL '{base_url}'. Use an absolute http:// or https:// URL.") - })?; +pub(crate) fn ensure_remote_target_has_api_key(base_url: &str, api_key: Option<&str>, paths: &auth::CortexPaths) -> Result<(), String> { + let parsed = reqwest::Url::parse(base_url).map_err(|_| format!("Invalid Cortex target URL '{base_url}'. Use an absolute http:// or https:// URL."))?; if !matches!(parsed.scheme(), "http" | "https") { - return Err(format!( - "Unsupported Cortex target URL scheme '{}' in '{base_url}'. Use http or https.", - parsed.scheme() - )); + return Err(format!("Unsupported Cortex target URL scheme '{}' in '{base_url}'. Use http or https.", parsed.scheme())); } if parsed.host_str().is_none() { - return Err(format!( - "Invalid Cortex target URL '{base_url}': missing host." - )); + return Err(format!("Invalid Cortex target URL '{base_url}': missing host.")); } if !parsed.username().is_empty() || parsed.password().is_some() { - return Err( - "Cortex target URL must not include embedded credentials; pass --api-key instead." - .to_string(), - ); + return Err("Cortex target URL must not include embedded credentials; pass --api-key instead.".to_string()); } if parsed.query().is_some() || parsed.fragment().is_some() { - return Err( - "Cortex target URL must not include query parameters or fragments.".to_string(), - ); + return Err("Cortex target URL must not include query parameters or fragments.".to_string()); } - if api_key.is_none() && !is_local_client_base_url(base_url, paths) { - return Err(format!( - "Remote Cortex target '{base_url}' requires an API key. Pass --api-key or set CORTEX_API_KEY." - )); + return Err(format!("Remote Cortex target '{base_url}' requires an API key. Pass --api-key or set CORTEX_API_KEY.")); } Ok(()) } - pub(crate) fn apply_path_env(paths: &auth::CortexPaths) { std::env::set_var("CORTEX_HOME", &paths.home); std::env::set_var("CORTEX_DB", &paths.db); @@ -203,91 +127,53 @@ pub(crate) fn apply_path_env(paths: &auth::CortexPaths) { None => std::env::remove_var("CORTEX_IPC_ENDPOINT"), } } - pub(crate) fn parse_flag_usize(args: &[String], flag: &str) -> Result, String> { let Some(idx) = args.iter().position(|a| a == flag) else { return Ok(None); }; - - let raw = args - .get(idx + 1) - .ok_or_else(|| format!("missing value for {flag}"))?; + let raw = args.get(idx + 1).ok_or_else(|| format!("missing value for {flag}"))?; if is_cli_option_token(raw) { return Err(format!("missing value for {flag}")); } - let value = raw - .parse::() - .map_err(|_| format!("invalid value for {flag}: '{raw}'"))?; + let value = raw.parse::().map_err(|_| format!("invalid value for {flag}: '{raw}'"))?; if value == 0 { return Err(format!("{flag} must be >= 1")); } Ok(Some(value)) } - pub(crate) fn parse_env_usize(key: &str, default: usize) -> usize { - std::env::var(key) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(default) + std::env::var(key).ok().and_then(|raw| raw.trim().parse::().ok()).filter(|value| *value > 0).unwrap_or(default) } - pub(crate) fn parse_env_u64(key: &str, default: u64) -> u64 { - std::env::var(key) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(default) + std::env::var(key).ok().and_then(|raw| raw.trim().parse::().ok()).filter(|value| *value > 0).unwrap_or(default) } - pub(crate) fn open_cli_connection(db_path: &Path) -> Result { - let conn = db::open(db_path) - .map_err(|e| format!("Failed to open database at {}: {e}", db_path.display()))?; + let conn = db::open(db_path).map_err(|e| format!("Failed to open database at {}: {e}", db_path.display()))?; db::configure(&conn).map_err(|e| format!("Failed to configure database: {e}"))?; db::initialize_schema(&conn).map_err(|e| format!("Failed to initialize schema: {e}"))?; db::run_pending_migrations_quiet(&conn); crystallize::migrate_crystal_tables(&conn); Ok(conn) } - -pub(crate) async fn admin_request( - paths: &auth::CortexPaths, - method: &str, - path: &str, - body: Option, -) -> Result { +pub(crate) async fn admin_request(paths: &auth::CortexPaths, method: &str, path: &str, body: Option) -> Result { let token = read_auth_token(paths)?; - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(10)) - .build() - .map_err(|e| format!("create admin client: {e}"))?; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().map_err(|e| format!("create admin client: {e}"))?; let base_url = local_daemon_base_url(paths); let payload = body.map(|value| value.to_string()); - let mut headers = vec![ - ("authorization".to_string(), format!("Bearer {token}")), - ("x-cortex-request".to_string(), "true".to_string()), - ]; + let mut headers = vec![("authorization".to_string(), format!("Bearer {token}")), ("x-cortex-request".to_string(), "true".to_string())]; if payload.is_some() { headers.push(("content-type".to_string(), "application/json".to_string())); } - let (status, body_text) = transport::request_with_local_ipc_fallback( - &client, - method, - &base_url, - path, - paths, - &headers, - payload.as_deref(), - Duration::from_secs(10), - ) - .await - .map_err(|e| { - if e.to_ascii_lowercase().contains("connect") { - "Cortex daemon not running. Start with: cortex serve".to_string() - } else { - format!("Request failed: {e}") - } - })?; + let (status, body_text) = + transport::request_with_local_ipc_fallback(&client, method, &base_url, path, paths, &headers, payload.as_deref(), Duration::from_secs(10)) + .await + .map_err(|e| { + if e.to_ascii_lowercase().contains("connect") { + "Cortex daemon not running. Start with: cortex serve".to_string() + } else { + format!("Request failed: {e}") + } + })?; if status.as_u16() == 403 { return Err("Admin commands require team mode. Run: cortex setup --team".to_string()); } @@ -302,71 +188,11 @@ pub(crate) async fn admin_request( } })?; if !status.is_success() { - let msg = json - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or("Unknown error"); + let msg = json.get("error").and_then(|v| v.as_str()).unwrap_or("Unknown error"); return Err(msg.to_string()); } Ok(json) } - -pub(crate) fn confirm_action(prompt: &str) -> bool { - eprint!("{prompt} [y/N] "); - let mut input = String::new(); - if std::io::stdin().read_line(&mut input).is_err() { - return false; - } - matches!(input.trim().to_lowercase().as_str(), "y" | "yes") -} - pub(crate) fn json_str(val: &serde_json::Value, key: &str) -> String { - val.get(key) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string() -} - -pub(crate) fn json_str_or(val: &serde_json::Value, key: &str, default: &str) -> String { - val.get(key) - .and_then(|v| v.as_str()) - .unwrap_or(default) - .to_string() -} - -pub(crate) fn json_field(val: &serde_json::Value, key: &str) -> String { - match val.get(key) { - Some(v) if v.is_string() => v.as_str().unwrap_or("").to_string(), - Some(v) => v.to_string(), - None => "-".to_string(), - } -} - -pub(crate) fn api_key_output_masked() -> bool { - !std::io::stdout().is_terminal() -} - -pub(crate) fn format_api_key_for_output(api_key: &str) -> String { - if !api_key_output_masked() { - return api_key.to_string(); - } - mask_secret_for_logs(api_key) -} - -fn mask_secret_for_logs(secret: &str) -> String { - const PREFIX: usize = 8; - const SUFFIX: usize = 4; - let chars: Vec = secret.chars().collect(); - if chars.is_empty() { - return String::new(); - } - if chars.len() <= PREFIX + SUFFIX { - return "*".repeat(chars.len().max(4)); - } - let prefix: String = chars.iter().take(PREFIX).collect(); - let suffix: String = chars - .iter() - .skip(chars.len().saturating_sub(SUFFIX)) - .collect(); - format!("{prefix}...{suffix}") + val.get(key).and_then(|v| v.as_str()).unwrap_or("").to_string() } diff --git a/daemon-rs/src/cli/daemon/backfill.rs b/daemon-rs/src/cli/daemon/backfill.rs index 8bc831ba..127e6ef2 100644 --- a/daemon-rs/src/cli/daemon/backfill.rs +++ b/daemon-rs/src/cli/daemon/backfill.rs @@ -1,49 +1,8 @@ -// SPDX-License-Identifier: MIT -use chrono::Utc; -use fs2::FileExt; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use crate::admin; -use crate::aging; -use crate::auth; -use crate::budgets; -use crate::compaction; -use crate::crystallize; -use crate::db; -use crate::daemon_lifecycle; -use crate::embeddings; -use crate::indexer; -use crate::server; -use crate::state; -use crate::transport; - -use crate::cli::boot::boot_agent; -use crate::cli::cleanup::{ - cleanup_backup_retention, cleanup_bridge_backups, cleanup_expired_rows, create_backup, - rotate_startup_logs, should_backup, -}; -use crate::cli::common::{ - env_trimmed, local_daemon_base_url, normalize_option, parse_env_u64, parse_env_usize, - parse_truthy_flag, single_daemon_test_bypass_enabled, -}; - -#[cfg(not(windows))] -use daemon_lifecycle::issue_owner_token_for_spawn; -use daemon_lifecycle::{ - daemon_healthy, is_cortex_health_payload, readiness_state_from_payload, - validate_spawned_owner_claim, wait_for_health, DAEMON_OWNER_TOKEN_ENV, - SPAWN_PARENT_START_TIME_ENV, -}; - - use super::*; -/// Build embeddings for all un-embedded memories and decisions. -/// IMPORTANT: Does NOT hold the DB lock during ONNX inference. -/// Reads IDs/text in a short lock, embeds in memory (no lock), then writes in batches. +use crate::embeddings; +use std::time::Duration; pub(crate) type EmbeddingBackfillRows = Vec<(i64, String)>; pub(crate) type EmbeddingBackfillTargets = (EmbeddingBackfillRows, EmbeddingBackfillRows); - #[derive(Clone, Copy, Debug, Default)] pub(crate) struct EmbeddingBackfillPassResult { pub(crate) queued_total: usize, @@ -51,20 +10,10 @@ pub(crate) struct EmbeddingBackfillPassResult { pub(crate) passes_ran: usize, pub(crate) exhausted: bool, } - -pub(crate) fn backfill_batch_may_have_more( - memory_count: usize, - decision_count: usize, - batch_size: usize, -) -> bool { +pub(crate) fn backfill_batch_may_have_more(memory_count: usize, decision_count: usize, batch_size: usize) -> bool { memory_count >= batch_size || decision_count >= batch_size } - -pub(crate) fn collect_unembedded_targets_for_model( - conn: &rusqlite::Connection, - model_key: &str, - limit: usize, -) -> EmbeddingBackfillTargets { +pub(crate) fn collect_unembedded_targets_for_model(conn: &rusqlite::Connection, model_key: &str, limit: usize) -> EmbeddingBackfillTargets { let mem: EmbeddingBackfillRows = conn .prepare( "SELECT m.id, m.text FROM memories m \ @@ -79,13 +28,10 @@ pub(crate) fn collect_unembedded_targets_for_model( LIMIT ?2", ) .and_then(|mut stmt| { - stmt.query_map(rusqlite::params![model_key, limit as i64], |row| { - Ok((row.get(0)?, row.get(1)?)) - }) - .map(|rows| rows.filter_map(|r| r.ok()).collect()) + stmt.query_map(rusqlite::params![model_key, limit as i64], |row| Ok((row.get(0)?, row.get(1)?))) + .map(|rows| rows.filter_map(|r| r.ok()).collect()) }) .unwrap_or_default(); - let dec: EmbeddingBackfillRows = conn .prepare( "SELECT d.id, d.decision FROM decisions d \ @@ -100,20 +46,13 @@ pub(crate) fn collect_unembedded_targets_for_model( LIMIT ?2", ) .and_then(|mut stmt| { - stmt.query_map(rusqlite::params![model_key, limit as i64], |row| { - Ok((row.get(0)?, row.get(1)?)) - }) - .map(|rows| rows.filter_map(|r| r.ok()).collect()) + stmt.query_map(rusqlite::params![model_key, limit as i64], |row| Ok((row.get(0)?, row.get(1)?))) + .map(|rows| rows.filter_map(|r| r.ok()).collect()) }) .unwrap_or_default(); - (mem, dec) } - -pub(crate) fn count_unembedded_targets_for_model( - conn: &rusqlite::Connection, - model_key: &str, -) -> (usize, usize) { +pub(crate) fn count_unembedded_targets_for_model(conn: &rusqlite::Connection, model_key: &str) -> (usize, usize) { let memory_count = conn .query_row( "SELECT COUNT(*) FROM memories m \ @@ -129,7 +68,6 @@ pub(crate) fn count_unembedded_targets_for_model( ) .unwrap_or(0) .max(0) as usize; - let decision_count = conn .query_row( "SELECT COUNT(*) FROM decisions d \ @@ -145,30 +83,21 @@ pub(crate) fn count_unembedded_targets_for_model( ) .unwrap_or(0) .max(0) as usize; - (memory_count, decision_count) } - pub(crate) async fn build_embeddings_async( - engine: std::sync::Arc, - db: &std::sync::Arc>, - batch_size: usize, - max_batches_per_pass: usize, - lock_wait: Duration, + engine: std::sync::Arc, db: &std::sync::Arc>, batch_size: usize, + max_batches_per_pass: usize, lock_wait: Duration, ) -> EmbeddingBackfillPassResult { let model_key = engine.model_key(); let mut result = EmbeddingBackfillPassResult::default(); - for _ in 0..max_batches_per_pass { let (unembedded_mem, unembedded_dec) = { - let Some(conn) = - acquire_background_db_lock(db, "embedding backfill scan", lock_wait).await - else { + let Some(conn) = acquire_background_db_lock(db, "embedding backfill scan", lock_wait).await else { break; }; collect_unembedded_targets_for_model(&conn, model_key, batch_size) }; - let memory_count = unembedded_mem.len(); let decision_count = unembedded_dec.len(); let total = memory_count + decision_count; @@ -178,7 +107,6 @@ pub(crate) async fn build_embeddings_async( } result.passes_ran += 1; result.queued_total += total; - let mut computed_batch = 0usize; let mut mem_results: Vec<(i64, Vec)> = Vec::new(); for (id, text) in &unembedded_mem { @@ -187,7 +115,6 @@ pub(crate) async fn build_embeddings_async( computed_batch += 1; } } - let mut dec_results: Vec<(i64, Vec)> = Vec::new(); for (id, text) in &unembedded_dec { if let Some(vec) = engine.clone().embed_async(text.clone()).await { @@ -195,11 +122,8 @@ pub(crate) async fn build_embeddings_async( computed_batch += 1; } } - { - let Some(conn) = - acquire_background_db_lock(db, "embedding backfill persist", lock_wait).await - else { + let Some(conn) = acquire_background_db_lock(db, "embedding backfill persist", lock_wait).await else { break; }; for (id, blob) in &mem_results { @@ -217,23 +141,16 @@ pub(crate) async fn build_embeddings_async( ); } } - result.computed_total += computed_batch; if !backfill_batch_may_have_more(memory_count, decision_count, batch_size) { result.exhausted = true; break; } } - if result.queued_total > 0 { eprintln!( "[embeddings] Built {}/{} embeddings this pass (passes={}, batch_size={}, max_batches={}, exhausted={})", - result.computed_total, - result.queued_total, - result.passes_ran, - batch_size, - max_batches_per_pass, - result.exhausted + result.computed_total, result.queued_total, result.passes_ran, batch_size, max_batches_per_pass, result.exhausted ); } result diff --git a/daemon-rs/src/cli/daemon/mod.rs b/daemon-rs/src/cli/daemon/mod.rs index 4099b4d3..b78dbbad 100644 --- a/daemon-rs/src/cli/daemon/mod.rs +++ b/daemon-rs/src/cli/daemon/mod.rs @@ -1,8 +1,6 @@ -// SPDX-License-Identifier: MIT -mod startup; -mod run; mod backfill; - -pub(crate) use startup::*; -pub(crate) use run::*; +mod run; +mod startup; pub(crate) use backfill::*; +pub(crate) use run::*; +pub(crate) use startup::*; diff --git a/daemon-rs/src/cli/daemon/run.rs b/daemon-rs/src/cli/daemon/run.rs index b9279b6d..dfa388ad 100644 --- a/daemon-rs/src/cli/daemon/run.rs +++ b/daemon-rs/src/cli/daemon/run.rs @@ -1,91 +1,40 @@ -// SPDX-License-Identifier: MIT -use chrono::Utc; -use fs2::FileExt; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use crate::admin; +use super::*; use crate::aging; use crate::auth; -use crate::budgets; +use crate::cli::cleanup::{cleanup_backup_retention, cleanup_bridge_backups, cleanup_expired_rows, create_backup, rotate_startup_logs, should_backup}; +use crate::cli::common::{parse_env_u64, parse_env_usize, parse_truthy_flag}; use crate::compaction; use crate::crystallize; -use crate::db; use crate::daemon_lifecycle; +use crate::db; use crate::embeddings; use crate::indexer; use crate::server; use crate::state; -use crate::transport; - -use crate::cli::boot::boot_agent; -use crate::cli::cleanup::{ - cleanup_backup_retention, cleanup_bridge_backups, cleanup_expired_rows, create_backup, - rotate_startup_logs, should_backup, -}; -use crate::cli::common::{ - env_trimmed, local_daemon_base_url, normalize_option, parse_env_u64, parse_env_usize, - parse_truthy_flag, single_daemon_test_bypass_enabled, -}; - -#[cfg(not(windows))] -use daemon_lifecycle::issue_owner_token_for_spawn; -use daemon_lifecycle::{ - daemon_healthy, is_cortex_health_payload, readiness_state_from_payload, - validate_spawned_owner_claim, wait_for_health, DAEMON_OWNER_TOKEN_ENV, - SPAWN_PARENT_START_TIME_ENV, -}; - - -use super::*; -// ── Shared daemon logic (used by `serve` and `service-run`) ───────────────── - -/// Run the full Cortex daemon. The `extra_shutdown` future is an additional -/// shutdown trigger beyond the HTTP /shutdown endpoint: -/// - `serve` passes Ctrl+C / SIGTERM -/// - `service-run` passes the SCM stop signal -pub(crate) async fn run_daemon( - paths: auth::CortexPaths, - extra_shutdown: impl std::future::Future + Send + 'static, -) { +use daemon_lifecycle::daemon_healthy; +use std::time::Duration; +pub(crate) async fn run_daemon(paths: auth::CortexPaths, extra_shutdown: impl std::future::Future + Send + 'static) { let _daemon_lock = match acquire_runtime_lock(&paths) { Ok(lock) => lock, Err(err) => { if daemon_healthy(&paths).await { - eprintln!( - "[cortex] Daemon already healthy on port {}; exiting cleanly.", - paths.port - ); + eprintln!("[cortex] Daemon already healthy on port {}; exiting cleanly.", paths.port); return; } eprintln!("[cortex] FATAL: {err}"); - eprintln!( - "[cortex] Reuse the existing daemon instead of launching a second `cortex serve`." - ); + eprintln!("[cortex] Reuse the existing daemon instead of launching a second `cortex serve`."); std::process::exit(1); } }; - let db_path = paths.db.clone(); - eprintln!( - "[cortex] Starting Cortex v{} (Rust)...", - env!("CARGO_PKG_VERSION") - ); + eprintln!("[cortex] Starting Cortex v{} (Rust)...", env!("CARGO_PKG_VERSION")); eprintln!("[cortex] DB: {}", db_path.display()); - crate::install_daemon_panic_hook(&paths); - let daemon_owner = daemon_owner_tag_from_env(); let parent_pid = spawn_parent_pid_from_env(); let parent_start_time = spawn_parent_start_time_from_env(); let owner_token = daemon_owner_token_from_env(); - if let Err(reason) = validate_spawned_owner_runtime_claim( - &paths, - daemon_owner.as_deref(), - parent_pid, - parent_start_time, - owner_token.as_deref(), - ) { + if let Err(reason) = validate_spawned_owner_runtime_claim(&paths, daemon_owner.as_deref(), parent_pid, parent_start_time, owner_token.as_deref()) { eprintln!("[cortex] FATAL: invalid spawned owner claim ({reason}); refusing startup"); std::process::exit(1); } @@ -93,7 +42,6 @@ pub(crate) async fn run_daemon( eprintln!("[cortex] FATAL: {reason}"); std::process::exit(1); } - let (state, shutdown_rx) = match state::initialize(&paths, true) { Ok(initialized) => initialized, Err(err) => { @@ -101,7 +49,6 @@ pub(crate) async fn run_daemon( std::process::exit(1); } }; - if should_watch_spawn_parent(daemon_owner.as_deref()) { if let (Some(parent_pid), Some(parent_start_time)) = (parent_pid, parent_start_time) { let _watcher = spawn_parent_orphan_watch_task( @@ -113,7 +60,6 @@ pub(crate) async fn run_daemon( ); } } - let startup_schedule = startup_schedule(daemon_owner.as_deref()); let background_lock_wait = background_db_lock_max_wait(); eprintln!( @@ -124,34 +70,21 @@ pub(crate) async fn run_daemon( startup_schedule.crystallize.as_secs(), startup_schedule.storage_governor_initial.as_secs() ); - if let Some(parent) = paths.pid.parent() { std::fs::create_dir_all(parent).ok(); } std::fs::write(&paths.pid, std::process::id().to_string()).ok(); - let token_path = paths.token.clone(); let pid_path = paths.pid.clone(); let backup_dir = paths.home.join("backups"); eprintln!("[cortex] Auth token at {}", token_path.display()); - eprintln!( - "[cortex] PID {} written to {}", - std::process::id(), - pid_path.display() - ); + eprintln!("[cortex] PID {} written to {}", std::process::id(), pid_path.display()); let cleaned_backups = cleanup_backup_retention(&backup_dir); - eprintln!( - "[cortex] Cleaned {cleaned_backups} old backups, kept {}", - crate::cli::cleanup::BACKUP_RETENTION_COUNT - ); + eprintln!("[cortex] Cleaned {cleaned_backups} old backups, kept {}", crate::cli::cleanup::BACKUP_RETENTION_COUNT); let rotated_logs = rotate_startup_logs(&paths.home); if rotated_logs > 0 { eprintln!("[cortex] Rotated {rotated_logs} oversized log files"); } - - // ── Recover WAL on startup ────────────────────────────────────── - // Run WAL checkpoint to recover any pending writes from a previous crash. - // This ensures committed transactions are flushed to the main DB file. { let conn = state.db.lock().await; if let Err(e) = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") { @@ -160,8 +93,6 @@ pub(crate) async fn run_daemon( eprintln!("[cortex] WAL recovery complete"); } } - - // ── Schema migrations (idempotent) ────────────────────────────── let schema_version = { let conn = state.db.lock().await; let applied = db::run_pending_migrations(&conn); @@ -171,10 +102,6 @@ pub(crate) async fn run_daemon( db::current_schema_user_version(&conn).unwrap_or(0) }; let _ = cleanup_bridge_backups(&paths.home, schema_version); - - // ── Startup indexing + decay (non-blocking) ───────────────────── - // This used to run inline before the server bound its port, which could - // delay startup significantly on large source trees. { let db_index = state.db.clone(); let home = state.home.clone(); @@ -186,80 +113,34 @@ pub(crate) async fn run_daemon( tokio::time::sleep(startup_delay).await; } let started = std::time::Instant::now(); - if let Some(conn) = - acquire_background_db_lock(&db_index, "startup indexing", lock_wait).await - { + if let Some(conn) = acquire_background_db_lock(&db_index, "startup indexing", lock_wait).await { let indexed = indexer::index_all(&conn, &home, owner_id); let decayed = indexer::decay_pass(&conn); - eprintln!( - "[cortex] Startup indexing complete: indexed {indexed}, decayed {decayed} scores in {}ms", - started.elapsed().as_millis() - ); + eprintln!("[cortex] Startup indexing complete: indexed {indexed}, decayed {decayed} scores in {}ms", started.elapsed().as_millis()); } }); } - - // ── Background embedding builder ──────────────────────────────── if let Some(engine) = state.embedding_engine.clone() { let db = state.db.clone(); - let batch_size = parse_env_usize( - "CORTEX_EMBED_BACKFILL_BATCH_SIZE", - DEFAULT_EMBED_BACKFILL_BATCH_SIZE, - ) - .clamp(1, 10_000); - let max_batches_per_pass = parse_env_usize( - "CORTEX_EMBED_BACKFILL_MAX_BATCHES_PER_PASS", - DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS, - ) - .clamp(1, 1000); - let interval_secs = parse_env_u64( - "CORTEX_EMBED_BACKFILL_INTERVAL_SECS", - DEFAULT_EMBED_BACKFILL_INTERVAL_SECS, - ) - .clamp(5, 86_400); - let drain_on_startup = std::env::var(EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV) - .ok() - .map(|value| parse_truthy_flag(&value)) - .unwrap_or(false); - let startup_drain_max_batches = parse_env_usize( - EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES_ENV, - DEFAULT_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES, - ) - .clamp(1, 10_000); + let batch_size = parse_env_usize("CORTEX_EMBED_BACKFILL_BATCH_SIZE", DEFAULT_EMBED_BACKFILL_BATCH_SIZE).clamp(1, 10_000); + let max_batches_per_pass = parse_env_usize("CORTEX_EMBED_BACKFILL_MAX_BATCHES_PER_PASS", DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS).clamp(1, 1000); + let interval_secs = parse_env_u64("CORTEX_EMBED_BACKFILL_INTERVAL_SECS", DEFAULT_EMBED_BACKFILL_INTERVAL_SECS).clamp(5, 86_400); + let drain_on_startup = std::env::var(EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV).ok().map(|value| parse_truthy_flag(&value)).unwrap_or(false); + let startup_drain_max_batches = + parse_env_usize(EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES_ENV, DEFAULT_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES).clamp(1, 10_000); let startup_delay = startup_schedule.embed; let lock_wait = background_lock_wait; - let startup_max_batches_per_pass = if startup_delay > Duration::from_secs(0) { - max_batches_per_pass.min(2) - } else { - max_batches_per_pass - }; + let startup_max_batches_per_pass = if startup_delay > Duration::from_secs(0) { max_batches_per_pass.min(2) } else { max_batches_per_pass }; tokio::spawn(async move { if startup_delay > Duration::from_secs(0) { tokio::time::sleep(startup_delay).await; } - let startup_pass = build_embeddings_async( - engine.clone(), - &db, - batch_size, - startup_max_batches_per_pass, - lock_wait, - ) - .await; + let startup_pass = build_embeddings_async(engine.clone(), &db, batch_size, startup_max_batches_per_pass, lock_wait).await; if startup_pass.queued_total > 0 && !startup_pass.exhausted { if drain_on_startup { - let drain_pass = build_embeddings_async( - engine.clone(), - &db, - batch_size, - startup_drain_max_batches, - lock_wait, - ) - .await; + let drain_pass = build_embeddings_async(engine.clone(), &db, batch_size, startup_drain_max_batches, lock_wait).await; if drain_pass.exhausted { - eprintln!( - "[embeddings] Startup drain completed backlog in {} batches", - drain_pass.passes_ran - ); + eprintln!("[embeddings] Startup drain completed backlog in {} batches", drain_pass.passes_ran); } else if drain_pass.queued_total > 0 { eprintln!( "[embeddings] Startup drain reached cap with backlog still pending (passes={}, queued={})", @@ -267,39 +148,24 @@ pub(crate) async fn run_daemon( ); } } else { - eprintln!( - "[embeddings] Startup pass left backlog pending; set {}=1 to run a one-time extended drain", - EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV - ); + eprintln!("[embeddings] Startup pass left backlog pending; set {}=1 to run a one-time extended drain", EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV); } } let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); - interval.tick().await; // skip first immediate tick + interval.tick().await; loop { interval.tick().await; - build_embeddings_async( - engine.clone(), - &db, - batch_size, - max_batches_per_pass, - lock_wait, - ) - .await; + build_embeddings_async(engine.clone(), &db, batch_size, max_batches_per_pass, lock_wait).await; } }); } else { let models_dir = paths.models.clone(); tokio::spawn(async move { if let Some(dir) = embeddings::ensure_model_downloaded_in(&models_dir).await { - eprintln!( - "[embeddings] Model ready at {} -- restart to activate", - dir.display() - ); + eprintln!("[embeddings] Model ready at {} -- restart to activate", dir.display()); } }); } - - // ── Background WAL checkpoint every 10s (crash-safe) ────────────── { let db_wal = state.db.clone(); let db_path = db_path.clone(); @@ -307,21 +173,15 @@ pub(crate) async fn run_daemon( let lock_wait = background_lock_wait.min(Duration::from_millis(750)); tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(10)); - interval.tick().await; // skip first immediate tick + interval.tick().await; loop { interval.tick().await; - - // Checkpoint WAL first to ensure consistency - let Some(conn) = - acquire_background_db_lock(&db_wal, "wal checkpoint", lock_wait).await - else { + let Some(conn) = acquire_background_db_lock(&db_wal, "wal checkpoint", lock_wait).await else { continue; }; { db::checkpoint_wal_best_effort(&conn); } - - // Check if daily backup is needed let backup_dir = home_dir.join("backups"); if should_backup(&backup_dir) { if let Err(e) = create_backup(&db_path, &backup_dir) { @@ -331,21 +191,16 @@ pub(crate) async fn run_daemon( } }); } - - // ── Background quick_check every 30 minutes ──────────────────────── - // Runs PRAGMA quick_check (B-tree only) to catch corruption that develops - // during runtime. On failure, sets db_corrupted so /health reflects it. { let db_qc = state.db_read.clone(); let db_corrupted_flag = state.db_corrupted.clone(); tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(30 * 60)); - interval.tick().await; // skip first tick -- startup integrity_check already ran + interval.tick().await; loop { interval.tick().await; let conn = db_qc.lock().await; if db::quick_check(&conn) { - // Clear the flag if a previous check had set it (e.g. after manual repair). db_corrupted_flag.store(false, std::sync::atomic::Ordering::SeqCst); } else { eprintln!( @@ -358,8 +213,6 @@ pub(crate) async fn run_daemon( } }); } - - // ── Background aging pass every 6 hours ────────────────────────── { let db_aging = state.db.clone(); let startup_delay = startup_schedule.aging; @@ -368,34 +221,24 @@ pub(crate) async fn run_daemon( if startup_delay > Duration::from_secs(0) { tokio::time::sleep(startup_delay).await; } - // Run initial aging pass on startup - if let Some(conn) = - acquire_background_db_lock(&db_aging, "initial aging pass", lock_wait).await - { + if let Some(conn) = acquire_background_db_lock(&db_aging, "initial aging pass", lock_wait).await { let (compressed, archived) = aging::run_aging_pass(&conn); if compressed > 0 || archived > 0 { - eprintln!( - "[cortex] Initial aging: {compressed} compressed, {archived} archived" - ); + eprintln!("[cortex] Initial aging: {compressed} compressed, {archived} archived"); } cleanup_expired_rows(&conn, "Initial expired cleanup"); } - // Then run every 6 hours let mut interval = tokio::time::interval(std::time::Duration::from_secs(6 * 3600)); interval.tick().await; loop { interval.tick().await; - if let Some(conn) = - acquire_background_db_lock(&db_aging, "aging pass", lock_wait).await - { + if let Some(conn) = acquire_background_db_lock(&db_aging, "aging pass", lock_wait).await { aging::run_aging_pass(&conn); cleanup_expired_rows(&conn, "Expired cleanup"); } } }); } - - // ── Background storage governor ──────────────────────────────────── { let db_compaction = state.db.clone(); let startup_delay = startup_schedule.storage_governor_initial; @@ -404,42 +247,27 @@ pub(crate) async fn run_daemon( if startup_delay > Duration::from_secs(0) { tokio::time::sleep(startup_delay).await; } - // Catch-up passes soon after startup to relieve event pressure early. for pass in 0..STARTUP_STORAGE_GOVERNOR_CATCHUP_PASSES { - if let Some(conn) = acquire_background_db_lock( - &db_compaction, - "startup storage governor", - lock_wait, - ) - .await - { + if let Some(conn) = acquire_background_db_lock(&db_compaction, "startup storage governor", lock_wait).await { let ran = compaction::run_compaction_governor_startup(&conn).is_some(); if !ran { break; } } if pass + 1 < STARTUP_STORAGE_GOVERNOR_CATCHUP_PASSES { - tokio::time::sleep(Duration::from_secs( - STARTUP_STORAGE_GOVERNOR_CATCHUP_INTERVAL_SECS, - )) - .await; + tokio::time::sleep(Duration::from_secs(STARTUP_STORAGE_GOVERNOR_CATCHUP_INTERVAL_SECS)).await; } } - let mut interval = tokio::time::interval(std::time::Duration::from_secs(30 * 60)); interval.tick().await; loop { interval.tick().await; - if let Some(conn) = - acquire_background_db_lock(&db_compaction, "storage governor", lock_wait).await - { + if let Some(conn) = acquire_background_db_lock(&db_compaction, "storage governor", lock_wait).await { let _ = compaction::run_compaction_governor(&conn); } } }); } - - // ── Background crystallization pass every 2 hours ───────────── { let db_crystal = state.db.clone(); let engine_crystal = state.embedding_engine.clone(); @@ -448,44 +276,23 @@ pub(crate) async fn run_daemon( let initial_delay = startup_schedule.crystallize; let lock_wait = background_lock_wait; tokio::spawn(async move { - // Initial pass on startup (after embeddings are built, with app-managed delay if needed) tokio::time::sleep(initial_delay).await; - if let Some(conn) = - acquire_background_db_lock(&db_crystal, "initial crystallization", lock_wait).await - { - let result = crystallize::run_crystallize_pass_with_brain( - &conn, - engine_crystal.as_deref(), - crystal_owner_id, - &brain_crystal, - ); + if let Some(conn) = acquire_background_db_lock(&db_crystal, "initial crystallization", lock_wait).await { + let result = crystallize::run_crystallize_pass_with_brain(&conn, engine_crystal.as_deref(), crystal_owner_id, &brain_crystal); if result.crystals_created > 0 || result.crystals_updated > 0 { - eprintln!( - "[cortex] Initial crystallization: {} created, {} updated", - result.crystals_created, result.crystals_updated - ); + eprintln!("[cortex] Initial crystallization: {} created, {} updated", result.crystals_created, result.crystals_updated); } } - // Then run every 2 hours let mut interval = tokio::time::interval(std::time::Duration::from_secs(2 * 3600)); interval.tick().await; loop { interval.tick().await; - if let Some(conn) = - acquire_background_db_lock(&db_crystal, "crystallization pass", lock_wait).await - { - crystallize::run_crystallize_pass_with_brain( - &conn, - engine_crystal.as_deref(), - crystal_owner_id, - &brain_crystal, - ); + if let Some(conn) = acquire_background_db_lock(&db_crystal, "crystallization pass", lock_wait).await { + crystallize::run_crystallize_pass_with_brain(&conn, engine_crystal.as_deref(), crystal_owner_id, &brain_crystal); } } }); } - - // ── Background rate limiter cleanup every 5 minutes ──────────── { let rl = state.rate_limiter.clone(); tokio::spawn(async move { @@ -497,29 +304,15 @@ pub(crate) async fn run_daemon( } }); } - - let idle_shutdown_secs = std::env::var(IDLE_SHUTDOWN_SECS_ENV) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .unwrap_or(0); - let idle_min_uptime_secs = parse_env_u64( - IDLE_SHUTDOWN_MIN_UPTIME_SECS_ENV, - DEFAULT_IDLE_SHUTDOWN_MIN_UPTIME_SECS, - ) - .clamp(1, 86_400); + let idle_shutdown_secs = std::env::var(IDLE_SHUTDOWN_SECS_ENV).ok().and_then(|raw| raw.trim().parse::().ok()).unwrap_or(0); + let idle_min_uptime_secs = parse_env_u64(IDLE_SHUTDOWN_MIN_UPTIME_SECS_ENV, DEFAULT_IDLE_SHUTDOWN_MIN_UPTIME_SECS).clamp(1, 86_400); if idle_shutdown_secs > 0 { - eprintln!( - "[cortex] Idle shutdown enabled (timeout={}s, min_uptime={}s)", - idle_shutdown_secs, idle_min_uptime_secs - ); + eprintln!("[cortex] Idle shutdown enabled (timeout={}s, min_uptime={}s)", idle_shutdown_secs, idle_min_uptime_secs); } - let readiness_signal = state.readiness.clone(); let db_for_shutdown = state.db.clone(); let state_for_idle_shutdown = state.clone(); let router = server::build_router(state, paths.port); - - // Combine shutdown sources: HTTP /shutdown, extra (Ctrl+C or SCM stop) let shutdown_future = async move { let idle_shutdown_future = async move { if idle_shutdown_secs == 0 { @@ -527,44 +320,21 @@ pub(crate) async fn run_daemon( return; } tokio::time::sleep(Duration::from_secs(idle_min_uptime_secs)).await; - let mut interval = tokio::time::interval(Duration::from_secs( - DEFAULT_IDLE_SHUTDOWN_CHECK_INTERVAL_SECS, - )); + let mut interval = tokio::time::interval(Duration::from_secs(DEFAULT_IDLE_SHUTDOWN_CHECK_INTERVAL_SECS)); interval.tick().await; loop { interval.tick().await; let idle_for = state_for_idle_shutdown.idle_for_secs(); if idle_for >= idle_shutdown_secs { - eprintln!( - "[cortex] Idle shutdown threshold reached (idle={}s >= {}s)", - idle_for, idle_shutdown_secs - ); + eprintln!("[cortex] Idle shutdown threshold reached (idle={}s >= {}s)", idle_for, idle_shutdown_secs); break; } } }; - - tokio::select! { - _ = shutdown_rx => { - eprintln!("[cortex] Shutdown requested via HTTP"); - } - _ = extra_shutdown => {} - _ = idle_shutdown_future => {} - } + tokio::select! {_=shutdown_rx=>{eprintln!("[cortex] Shutdown requested via HTTP");}_=extra_shutdown=> + {}_=idle_shutdown_future=>{}} }; - - server::run( - router, - &paths.bind, - paths.port, - paths.ipc_endpoint.clone(), - &db_path, - Some(readiness_signal), - shutdown_future, - ) - .await; - - // WAL checkpoint + cleanup + server::run(router, &paths.bind, paths.port, paths.ipc_endpoint.clone(), &db_path, Some(readiness_signal), shutdown_future).await; eprintln!("[cortex] Flushing database..."); { let conn = db_for_shutdown.lock().await; @@ -572,7 +342,6 @@ pub(crate) async fn run_daemon( eprintln!("[cortex] Warning: WAL checkpoint failed: {e}"); } } - let _ = std::fs::remove_file(&pid_path); eprintln!("[cortex] Shutdown complete."); } diff --git a/daemon-rs/src/cli/daemon/startup.rs b/daemon-rs/src/cli/daemon/startup.rs index 6b466b71..4d5224fc 100644 --- a/daemon-rs/src/cli/daemon/startup.rs +++ b/daemon-rs/src/cli/daemon/startup.rs @@ -1,46 +1,19 @@ -// SPDX-License-Identifier: MIT -use chrono::Utc; -use fs2::FileExt; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use crate::admin; -use crate::aging; use crate::auth; -use crate::budgets; -use crate::compaction; -use crate::crystallize; -use crate::db; +use crate::cli::boot::boot_agent; +use crate::cli::common::{env_trimmed, local_daemon_base_url, normalize_option, parse_truthy_flag, single_daemon_test_bypass_enabled}; use crate::daemon_lifecycle; -use crate::embeddings; -use crate::indexer; -use crate::server; -use crate::state; use crate::transport; - -use crate::cli::boot::boot_agent; -use crate::cli::cleanup::{ - cleanup_backup_retention, cleanup_bridge_backups, cleanup_expired_rows, create_backup, - rotate_startup_logs, should_backup, -}; -use crate::cli::common::{ - env_trimmed, local_daemon_base_url, normalize_option, parse_env_u64, parse_env_usize, - parse_truthy_flag, single_daemon_test_bypass_enabled, -}; - #[cfg(not(windows))] use daemon_lifecycle::issue_owner_token_for_spawn; use daemon_lifecycle::{ - daemon_healthy, is_cortex_health_payload, readiness_state_from_payload, - validate_spawned_owner_claim, wait_for_health, DAEMON_OWNER_TOKEN_ENV, + daemon_healthy, is_cortex_health_payload, readiness_state_from_payload, validate_spawned_owner_claim, wait_for_health, DAEMON_OWNER_TOKEN_ENV, SPAWN_PARENT_START_TIME_ENV, }; - - -use super::*; +use fs2::FileExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; pub(crate) const CONTROL_CENTER_LOCK_FILE: &str = "control-center.lock"; pub(crate) const CONTROL_CENTER_OWNER_TAG: &str = "control-center"; -pub(crate) const SINGLE_DAEMON_TEST_BYPASS_ENV: &str = "CORTEX_SINGLE_DAEMON_TEST_BYPASS"; pub(crate) const SPAWN_PARENT_PID_ENV: &str = "CORTEX_SPAWN_PARENT_PID"; pub(crate) const ORPHAN_WATCH_INTERVAL_SECS: u64 = 2; pub(crate) const DEFAULT_EMBED_BACKFILL_BATCH_SIZE: usize = 200; @@ -70,20 +43,10 @@ pub(crate) const STARTUP_CRYSTALLIZE_DELAY_ENV: &str = "CORTEX_STARTUP_CRYSTALLI pub(crate) const STARTUP_STORAGE_GOVERNOR_DELAY_ENV: &str = "CORTEX_STARTUP_STORAGE_GOVERNOR_DELAY_SECS"; pub(crate) const BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV: &str = "CORTEX_BACKGROUND_DB_LOCK_MAX_WAIT_MS"; pub(crate) const EMBED_BACKFILL_DRAIN_ON_STARTUP_ENV: &str = "CORTEX_EMBED_BACKFILL_DRAIN_ON_STARTUP"; -pub(crate) const EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES_ENV: &str = - "CORTEX_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES"; +pub(crate) const EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES_ENV: &str = "CORTEX_EMBED_BACKFILL_STARTUP_DRAIN_MAX_BATCHES"; pub(crate) const IDLE_SHUTDOWN_SECS_ENV: &str = "CORTEX_IDLE_SHUTDOWN_SECS"; pub(crate) const IDLE_SHUTDOWN_MIN_UPTIME_SECS_ENV: &str = "CORTEX_IDLE_SHUTDOWN_MIN_UPTIME_SECS"; -pub(crate) const STARTUP_LOG_FILES: &[&str] = &[ - "daemon.log", - "daemon.err.log", - "daemon.out.log", - "mcp-crash.log", - "rust-daemon.err.log", -]; - pub(crate) const DAEMON_STARTUP_WAIT_SECS: u64 = 90; -pub(crate) const DEFAULT_BOOT_BUDGET: usize = 600; pub(crate) const DEFAULT_DAEMON_LOCK_WAIT_SECS: u64 = 15; pub(crate) const DAEMON_LOCK_RETRY_INTERVAL_MS: u64 = 100; pub(crate) const DAEMON_LOCK_HANDOFF_GRACE_SECS: u64 = 3; @@ -91,9 +54,6 @@ pub(crate) const DAEMON_LOCAL_SPAWN_ENV: &str = "CORTEX_DAEMON_OWNER_LOCAL_SPAWN pub(crate) const APP_REQUIRED_ENV: &str = "CORTEX_APP_REQUIRED"; pub(crate) const APP_CLIENT_ENV: &str = "CORTEX_APP_CLIENT"; pub(crate) const APP_MANAGED_STARTUP_DELAY_ENV: &str = "CORTEX_APP_MANAGED_STARTUP_DELAY_SECS"; - -/// Hold the singleton daemon lock before startup so duplicate `serve` -/// invocations cannot rotate the shared auth token and then die on bind. pub(crate) fn daemon_lock_wait_timeout() -> Duration { let secs = std::env::var("CORTEX_DAEMON_LOCK_WAIT_SECS") .ok() @@ -101,13 +61,11 @@ pub(crate) fn daemon_lock_wait_timeout() -> Duration { .unwrap_or(DEFAULT_DAEMON_LOCK_WAIT_SECS); Duration::from_secs(secs.max(1)) } - #[derive(Debug)] pub(crate) struct RuntimeLockGuards { _scoped: std::fs::File, _global: Option, } - pub(crate) fn try_acquire_runtime_locks(paths: &auth::CortexPaths) -> Result { let scoped = auth::acquire_daemon_lock(paths)?; let global = if single_daemon_test_bypass_enabled() { @@ -121,18 +79,11 @@ pub(crate) fn try_acquire_runtime_locks(paths: &auth::CortexPaths) -> Result Result { let _ = auth::cleanup_stale_pid_lock(paths); - if std::env::var("CORTEX_WAIT_FOR_DAEMON_LOCK") - .ok() - .is_some_and(|value| value == "1") - { + if std::env::var("CORTEX_WAIT_FOR_DAEMON_LOCK").ok().is_some_and(|value| value == "1") { let deadline = std::time::Instant::now() + daemon_lock_wait_timeout(); let last_err = loop { match try_acquire_runtime_locks(paths) { @@ -146,10 +97,7 @@ pub(crate) fn acquire_runtime_lock(paths: &auth::CortexPaths) -> Result Result Option { - std::env::var("CORTEX_DAEMON_OWNER") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) + std::env::var("CORTEX_DAEMON_OWNER").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) } - pub(crate) fn daemon_owner_token_from_env() -> Option { - std::env::var(DAEMON_OWNER_TOKEN_ENV) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) + std::env::var(DAEMON_OWNER_TOKEN_ENV).ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) } - pub(crate) fn spawn_parent_pid_from_env() -> Option { - std::env::var(SPAWN_PARENT_PID_ENV) - .ok() - .and_then(|value| value.trim().parse::().ok()) + std::env::var(SPAWN_PARENT_PID_ENV).ok().and_then(|value| value.trim().parse::().ok()) } - pub(crate) fn spawn_parent_start_time_from_env() -> Option { - std::env::var(SPAWN_PARENT_START_TIME_ENV) - .ok() - .and_then(|value| value.trim().parse::().ok()) + std::env::var(SPAWN_PARENT_START_TIME_ENV).ok().and_then(|value| value.trim().parse::().ok()) } - pub(crate) fn should_watch_spawn_parent(owner_tag: Option<&str>) -> bool { - owner_tag - .map(|owner| !owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)) - .unwrap_or(true) + owner_tag.map(|owner| !owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)).unwrap_or(true) } - pub(crate) fn is_control_center_owner(owner_tag: Option<&str>) -> bool { - owner_tag - .map(|owner| owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)) - .unwrap_or(false) + owner_tag.map(|owner| owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)).unwrap_or(false) } - pub(crate) fn parse_env_u64_nonnegative(key: &str, default: u64) -> u64 { - std::env::var(key) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .unwrap_or(default) + std::env::var(key).ok().and_then(|raw| raw.trim().parse::().ok()).unwrap_or(default) } - pub(crate) fn app_managed_startup_heavy_delay(owner_tag: Option<&str>) -> Duration { if !is_control_center_owner(owner_tag) { return Duration::from_secs(0); } - let secs = parse_env_u64_nonnegative( - APP_MANAGED_STARTUP_DELAY_ENV, - APP_MANAGED_STARTUP_HEAVY_DELAY_SECS, - ) - .min(APP_MANAGED_STARTUP_HEAVY_DELAY_MAX_SECS); + let secs = parse_env_u64_nonnegative(APP_MANAGED_STARTUP_DELAY_ENV, APP_MANAGED_STARTUP_HEAVY_DELAY_SECS).min(APP_MANAGED_STARTUP_HEAVY_DELAY_MAX_SECS); Duration::from_secs(secs) } - #[derive(Clone, Copy)] pub(crate) struct StartupSchedule { pub(crate) index: Duration, @@ -227,19 +145,13 @@ pub(crate) struct StartupSchedule { pub(crate) crystallize: Duration, pub(crate) storage_governor_initial: Duration, } - pub(crate) fn startup_delay_from_env(key: &str, default: u64) -> Duration { Duration::from_secs(parse_env_u64_nonnegative(key, default).min(3_600)) } - pub(crate) fn startup_schedule(owner_tag: Option<&str>) -> StartupSchedule { let zero = Duration::from_secs(0); let heavy = app_managed_startup_heavy_delay(owner_tag); - let index = if heavy > zero { - heavy - } else { - startup_delay_from_env(STARTUP_INDEX_DELAY_ENV, DEFAULT_STARTUP_INDEX_DELAY_SECS) - }; + let index = if heavy > zero { heavy } else { startup_delay_from_env(STARTUP_INDEX_DELAY_ENV, DEFAULT_STARTUP_INDEX_DELAY_SECS) }; let aging = if heavy > zero { heavy + Duration::from_secs(APP_MANAGED_AGING_STARTUP_OFFSET_SECS) } else { @@ -253,37 +165,17 @@ pub(crate) fn startup_schedule(owner_tag: Option<&str>) -> StartupSchedule { let crystallize = if heavy > zero { heavy + Duration::from_secs(APP_MANAGED_CRYSTALLIZE_STARTUP_OFFSET_SECS) } else { - startup_delay_from_env( - STARTUP_CRYSTALLIZE_DELAY_ENV, - DEFAULT_STARTUP_CRYSTALLIZE_DELAY_SECS, - ) + startup_delay_from_env(STARTUP_CRYSTALLIZE_DELAY_ENV, DEFAULT_STARTUP_CRYSTALLIZE_DELAY_SECS) }; - let storage_governor_initial = startup_delay_from_env( - STARTUP_STORAGE_GOVERNOR_DELAY_ENV, - DEFAULT_STARTUP_STORAGE_GOVERNOR_DELAY_SECS, - ); - StartupSchedule { - index, - aging, - embed, - crystallize, - storage_governor_initial, - } + let storage_governor_initial = startup_delay_from_env(STARTUP_STORAGE_GOVERNOR_DELAY_ENV, DEFAULT_STARTUP_STORAGE_GOVERNOR_DELAY_SECS); + StartupSchedule { index, aging, embed, crystallize, storage_governor_initial } } - pub(crate) fn background_db_lock_max_wait() -> Duration { - let max_wait_ms = parse_env_u64_nonnegative( - BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, - BACKGROUND_DB_LOCK_DEFAULT_MAX_WAIT_MS, - ) - .clamp(100, 60_000); + let max_wait_ms = parse_env_u64_nonnegative(BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, BACKGROUND_DB_LOCK_DEFAULT_MAX_WAIT_MS).clamp(100, 60_000); Duration::from_millis(max_wait_ms) } - pub(crate) async fn acquire_background_db_lock<'a>( - db: &'a std::sync::Arc>, - task_name: &str, - max_wait: Duration, + db: &'a std::sync::Arc>, task_name: &str, max_wait: Duration, ) -> Option> { let started = std::time::Instant::now(); loop { @@ -291,35 +183,24 @@ pub(crate) async fn acquire_background_db_lock<'a>( return Some(conn); } if started.elapsed() >= max_wait { - eprintln!( - "[cortex] Skipping {task_name}: DB lock busy for {}ms", - started.elapsed().as_millis() - ); + eprintln!("[cortex] Skipping {task_name}: DB lock busy for {}ms", started.elapsed().as_millis()); return None; } tokio::time::sleep(Duration::from_millis(BACKGROUND_DB_LOCK_RETRY_MS)).await; } } - pub(crate) fn process_pid_start_time(pid: u32) -> Option { let mut system = sysinfo::System::new_all(); let target = sysinfo::Pid::from_u32(pid); system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[target]), true); system.process(target).map(|process| process.start_time()) } - pub(crate) fn process_pid_identity_matches(pid: u32, expected_start_time: u64) -> bool { - process_pid_start_time(pid) - .map(|actual_start_time| actual_start_time == expected_start_time) - .unwrap_or(false) + process_pid_start_time(pid).map(|actual_start_time| actual_start_time == expected_start_time).unwrap_or(false) } - pub(crate) fn spawn_parent_orphan_watch_task( - shutdown_tx: std::sync::Arc>>>, - parent_pid: u32, - parent_start_time: u64, - watch_interval: Duration, - identity_matches: F, + shutdown_tx: std::sync::Arc>>>, parent_pid: u32, parent_start_time: u64, + watch_interval: Duration, identity_matches: F, ) -> tokio::task::JoinHandle<()> where F: Fn(u32, u64) -> bool + Send + Sync + 'static, @@ -331,9 +212,7 @@ where loop { interval.tick().await; if !(identity_matches)(parent_pid, parent_start_time) { - eprintln!( - "[cortex] Spawn parent process {parent_pid} exited or was recycled; shutting down daemon" - ); + eprintln!("[cortex] Spawn parent process {parent_pid} exited or was recycled; shutting down daemon"); if let Some(tx) = shutdown_tx.lock().await.take() { let _ = tx.send(()); } @@ -342,36 +221,23 @@ where } }) } - pub(crate) fn process_looks_like_cortex_daemon(process: &sysinfo::Process) -> bool { - let cmd: Vec = process - .cmd() - .iter() - .map(|arg| arg.to_string_lossy().to_ascii_lowercase()) - .collect(); + let cmd: Vec = process.cmd().iter().map(|arg| arg.to_string_lossy().to_ascii_lowercase()).collect(); if cmd.is_empty() { return false; } - let has_daemon_role = cmd.iter().any(|arg| arg == "serve" || arg == "service-run") - || cmd - .windows(2) - .any(|pair| pair[0] == "service" && pair[1] == "run"); + let has_daemon_role = cmd.iter().any(|arg| arg == "serve" || arg == "service-run") || cmd.windows(2).any(|pair| pair[0] == "service" && pair[1] == "run"); if !has_daemon_role { return false; } - let exe_is_cortex = process .exe() .and_then(|path| path.file_stem().or(path.file_name())) .map(|name| name.to_string_lossy().eq_ignore_ascii_case("cortex")) .unwrap_or(false); - let cmd_is_cortex = cmd - .first() - .map(|first| first.contains("cortex")) - .unwrap_or(false); + let cmd_is_cortex = cmd.first().map(|first| first.contains("cortex")).unwrap_or(false); exe_is_cortex || cmd_is_cortex } - pub(crate) fn detect_other_cortex_daemon_process() -> Option<(u32, String, String)> { let current_pid = std::process::id(); let mut system = sysinfo::System::new_all(); @@ -384,78 +250,42 @@ pub(crate) fn detect_other_cortex_daemon_process() -> Option<(u32, String, Strin if !process_looks_like_cortex_daemon(process) { continue; } - let exe = process - .exe() - .map(|path| path.display().to_string()) - .unwrap_or_else(|| "".to_string()); - let cmd = process - .cmd() - .iter() - .map(|arg| arg.to_string_lossy().into_owned()) - .collect::>() - .join(" "); + let exe = process.exe().map(|path| path.display().to_string()).unwrap_or_else(|| "".to_string()); + let cmd = process.cmd().iter().map(|arg| arg.to_string_lossy().into_owned()).collect::>().join(" "); return Some((pid_u32, exe, cmd)); } None } - pub(crate) fn spawned_owner_requires_parent_pid(owner_tag: Option<&str>) -> bool { - owner_tag - .map(|owner| should_watch_spawn_parent(Some(owner))) - .unwrap_or(false) + owner_tag.map(|owner| should_watch_spawn_parent(Some(owner))).unwrap_or(false) } - pub(crate) fn validate_spawned_owner_runtime_claim( - paths: &auth::CortexPaths, - owner_tag: Option<&str>, - parent_pid: Option, - parent_start_time: Option, - owner_token: Option<&str>, + paths: &auth::CortexPaths, owner_tag: Option<&str>, parent_pid: Option, parent_start_time: Option, owner_token: Option<&str>, ) -> Result<(), String> { if spawned_owner_requires_parent_pid(owner_tag) && parent_pid.is_none() { - return Err(format!( - "owner '{}' requires {} linkage", - owner_tag.unwrap_or("unknown"), - SPAWN_PARENT_PID_ENV - )); + return Err(format!("owner '{}' requires {} linkage", owner_tag.unwrap_or("unknown"), SPAWN_PARENT_PID_ENV)); } if spawned_owner_requires_parent_pid(owner_tag) && parent_start_time.is_none() { - return Err(format!( - "owner '{}' requires {} linkage", - owner_tag.unwrap_or("unknown"), - SPAWN_PARENT_START_TIME_ENV - )); + return Err(format!("owner '{}' requires {} linkage", owner_tag.unwrap_or("unknown"), SPAWN_PARENT_START_TIME_ENV)); } - if let (Some(parent_pid), Some(parent_start_time)) = (parent_pid, parent_start_time) { let Some(actual_start_time) = process_pid_start_time(parent_pid) else { - return Err(format!( - "spawn parent process {parent_pid} is not running during ownership claim validation" - )); + return Err(format!("spawn parent process {parent_pid} is not running during ownership claim validation")); }; if actual_start_time != parent_start_time { - return Err(format!( - "spawn parent start-time mismatch for pid {parent_pid} (env={parent_start_time}, actual={actual_start_time})" - )); + return Err(format!("spawn parent start-time mismatch for pid {parent_pid} (env={parent_start_time}, actual={actual_start_time})")); } } - validate_spawned_owner_claim(paths, owner_tag, parent_pid, owner_token) } - pub(crate) async fn startup_single_daemon_preflight(paths: &auth::CortexPaths) -> Result<(), String> { if let Some((pid, exe, cmd)) = detect_other_cortex_daemon_process() { if single_daemon_test_bypass_enabled() { - eprintln!( - "[cortex] Warning: bypassing single-daemon process preflight for debug test run (detected pid={pid}, exe={exe}, cmd=\"{cmd}\")" - ); + eprintln!("[cortex] Warning: bypassing single-daemon process preflight for debug test run (detected pid={pid}, exe={exe}, cmd=\"{cmd}\")"); } else { - return Err(format!( - "daemon startup denied: Cortex already has an active daemon process (pid={pid}, exe={exe}, cmd=\"{cmd}\")" - )); + return Err(format!("daemon startup denied: Cortex already has an active daemon process (pid={pid}, exe={exe}, cmd=\"{cmd}\")")); } } - let bind_addr = paths.bind.trim(); let bind_error = match std::net::TcpListener::bind((bind_addr, paths.port)) { Ok(listener) => { @@ -464,149 +294,81 @@ pub(crate) async fn startup_single_daemon_preflight(paths: &auth::CortexPaths) - } Err(err) => err, }; - let readiness_url = format!("{}/readiness", local_daemon_base_url(paths)); let health_url = format!("{}/health", local_daemon_base_url(paths)); let client = reqwest::Client::builder() .timeout(Duration::from_secs(2)) .build() .map_err(|err| format!("daemon startup preflight: build HTTP client: {err}"))?; - let (mut status, mut body) = match transport::request_url_with_local_ipc_fallback( - &client, - "GET", - &readiness_url, - paths, - &[], - None, - Duration::from_secs(2), - ) - .await + let (mut status, mut body) = match transport::request_url_with_local_ipc_fallback(&client, "GET", &readiness_url, paths, &[], None, Duration::from_secs(2)) + .await { Ok((status, body)) => (status.as_u16(), body), Err(readiness_err) => { - // Backward compatibility for daemons that do not expose /readiness yet. - match transport::request_url_with_local_ipc_fallback( - &client, - "GET", - &health_url, - paths, - &[], - None, - Duration::from_secs(2), - ) - .await - { + match transport::request_url_with_local_ipc_fallback(&client, "GET", &health_url, paths, &[], None, Duration::from_secs(2)).await { Ok((status, body)) => (status.as_u16(), body), Err(health_err) => { return Err(format!( - "daemon startup denied: cannot bind {bind_addr}:{} ({bind_error}) and readiness probe at {readiness_url} failed ({readiness_err}); fallback health probe at {health_url} also failed ({health_err})", - paths.port - )); +"daemon startup denied: cannot bind {bind_addr}:{} ({bind_error}) and readiness probe at {readiness_url} failed ({readiness_err}); fallback health probe at {health_url} also failed ({health_err})" +,paths.port)); } } } }; - - if let Some(ready) = readiness_state_from_payload(status, &body, Some(paths.port), Some(paths)) - { + if let Some(ready) = readiness_state_from_payload(status, &body, Some(paths.port), Some(paths)) { return if ready { - Err(format!( - "daemon startup denied: canonical Cortex instance is already ready on port {}", - paths.port - )) + Err(format!("daemon startup denied: canonical Cortex instance is already ready on port {}", paths.port)) } else { - Err(format!( - "daemon startup denied: canonical Cortex instance is already starting on port {}", - paths.port - )) + Err(format!("daemon startup denied: canonical Cortex instance is already starting on port {}", paths.port)) }; } if readiness_state_from_payload(status, &body, Some(paths.port), None).is_some() { - return Err(format!( - "daemon startup denied: port {} is served by a different Cortex runtime identity", - paths.port - )); + return Err(format!("daemon startup denied: port {} is served by a different Cortex runtime identity", paths.port)); } - - // Fallback for legacy daemons (or intermediaries that do not proxy readiness): - // probe /health and apply canonical identity checks there. - if let Ok((health_status, health_body)) = transport::request_url_with_local_ipc_fallback( - &client, - "GET", - &health_url, - paths, - &[], - None, - Duration::from_secs(2), - ) - .await + if let Ok((health_status, health_body)) = + transport::request_url_with_local_ipc_fallback(&client, "GET", &health_url, paths, &[], None, Duration::from_secs(2)).await { status = health_status.as_u16(); body = health_body; } - if is_cortex_health_payload(status, &body, Some(paths.port), Some(paths)) { - return Err(format!( - "daemon startup denied: canonical Cortex instance is already healthy on port {}", - paths.port - )); + return Err(format!("daemon startup denied: canonical Cortex instance is already healthy on port {}", paths.port)); } if is_cortex_health_payload(status, &body, Some(paths.port), None) { - return Err(format!( - "daemon startup denied: port {} is served by a different Cortex runtime identity", - paths.port - )); + return Err(format!("daemon startup denied: port {} is served by a different Cortex runtime identity", paths.port)); } - Err(format!( "daemon startup denied: cannot bind {bind_addr}:{} ({bind_error}); readiness probe at {readiness_url} returned non-canonical payload (HTTP {status})", paths.port )) } - pub(crate) fn app_init_required_client_name(agent: Option<&str>) -> String { - env_trimmed(APP_CLIENT_ENV) - .or_else(|| normalize_option(agent)) - .unwrap_or_else(|| "client".to_string()) + env_trimmed(APP_CLIENT_ENV).or_else(|| normalize_option(agent)).unwrap_or_else(|| "client".to_string()) } - pub(crate) fn app_init_required_error(paths: &auth::CortexPaths, agent: Option<&str>) -> String { let client = app_init_required_client_name(agent); format!( - "APP_INIT_REQUIRED: {client} is attach-only and cannot start the daemon automatically on port {}. Start Cortex Control Center and initialize the app-managed daemon, then retry.", - paths.port - ) +"APP_INIT_REQUIRED: {client} is attach-only and cannot start the daemon automatically on port {}. Start Cortex Control Center and initialize the app-managed daemon, then retry." +,paths.port) } - pub(crate) fn local_spawn_allowed_for_request(allow_service_ensure: bool) -> bool { if !allow_service_ensure { return false; } let app_client_marked = env_trimmed(APP_CLIENT_ENV).is_some(); let local_spawn_raw = std::env::var(DAEMON_LOCAL_SPAWN_ENV).ok(); - let local_spawn_disabled = local_spawn_raw - .as_ref() - .is_some_and(|value| !parse_truthy_flag(value)); - let app_required = std::env::var(APP_REQUIRED_ENV) - .ok() - .is_some_and(|value| parse_truthy_flag(&value)); - // Fail closed for app-marked clients when no explicit local spawn policy exists. - // This prevents partial registration env contracts from silently re-enabling local spawn. + let local_spawn_disabled = local_spawn_raw.as_ref().is_some_and(|value| !parse_truthy_flag(value)); + let app_required = std::env::var(APP_REQUIRED_ENV).ok().is_some_and(|value| parse_truthy_flag(&value)); if app_client_marked && local_spawn_raw.is_none() { return false; } !(local_spawn_disabled || app_required) } - pub(crate) fn control_center_lock_path(paths: &auth::CortexPaths) -> PathBuf { paths.home.join("runtime").join(CONTROL_CENTER_LOCK_FILE) } - pub(crate) fn is_lock_contention_error(err: &std::io::Error) -> bool { - if matches!( - err.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::PermissionDenied - ) { + if matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::PermissionDenied) { return true; } if cfg!(windows) { @@ -614,59 +376,33 @@ pub(crate) fn is_lock_contention_error(err: &std::io::Error) -> bool { } false } - pub(crate) fn control_center_is_active(paths: &auth::CortexPaths) -> Result { let lock_path = control_center_lock_path(paths); - let lock_file = match std::fs::OpenOptions::new() - .create(false) - .read(true) - .write(true) - .open(&lock_path) - { + let lock_file = match std::fs::OpenOptions::new().create(false).read(true).write(true).open(&lock_path) { Ok(file) => file, Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(err) if is_lock_contention_error(&err) => return Ok(true), Err(err) => { - return Err(format!( - "open control-center lock {}: {err}", - lock_path.display() - )); + return Err(format!("open control-center lock {}: {err}", lock_path.display())); } }; - match lock_file.try_lock_exclusive() { Ok(()) => { let _ = lock_file.unlock(); Ok(false) } Err(err) if is_lock_contention_error(&err) => Ok(true), - Err(err) => Err(format!( - "probe control-center lock {}: {err}", - lock_path.display() - )), + Err(err) => Err(format!("probe control-center lock {}: {err}", lock_path.display())), } } - -pub(crate) async fn ensure_daemon( - paths: &auth::CortexPaths, - agent: Option<&str>, - emit_port: bool, - allow_service_ensure: bool, -) -> Result<(), String> { +pub(crate) async fn ensure_daemon(paths: &auth::CortexPaths, agent: Option<&str>, emit_port: bool, allow_service_ensure: bool) -> Result<(), String> { std::fs::create_dir_all(&paths.home).map_err(|e| format!("create home dir: {e}"))?; let local_spawn_allowed = local_spawn_allowed_for_request(allow_service_ensure); - let control_center_active_snapshot = if local_spawn_allowed { - control_center_is_active(paths).ok() - } else { - None - }; - + let control_center_active_snapshot = if local_spawn_allowed { control_center_is_active(paths).ok() } else { None }; let lock = auth::acquire_daemon_lock(paths); - match lock { Ok(_guard) => { if daemon_healthy(paths).await { - // already healthy } else if local_spawn_allowed { let _ = auth::migrate_legacy_db(paths)?; if control_center_active_snapshot == Some(true) { @@ -676,11 +412,7 @@ pub(crate) async fn ensure_daemon( Ok(true) => return Err(app_init_required_error(paths, agent)), Ok(false) => {} Err(err) => { - return Err(format!( - "{} (control-center lock probe failed: {})", - app_init_required_error(paths, agent), - err - )); + return Err(format!("{} (control-center lock probe failed: {})", app_init_required_error(paths, agent), err)); } } #[cfg(windows)] @@ -710,22 +442,14 @@ pub(crate) async fn ensure_daemon( Ok(true) => return Err(app_init_required_error(paths, agent)), Ok(false) => {} Err(err) => { - return Err(format!( - "{} (control-center lock probe failed: {})", - app_init_required_error(paths, agent), - err - )); + return Err(format!("{} (control-center lock probe failed: {})", app_init_required_error(paths, agent), err)); } } #[cfg(windows)] { if ensure_service_ready_async().await { - // proceed } else { - return Err(format!( - "daemon is not healthy on port {} and Windows service ensure failed while daemon lock was held.", - paths.port - )); + return Err(format!("daemon is not healthy on port {} and Windows service ensure failed while daemon lock was held.", paths.port)); } } #[cfg(not(windows))] @@ -741,39 +465,27 @@ pub(crate) async fn ensure_daemon( } } } - if let Some(agent) = agent { if let Err(e) = boot_agent(paths, agent).await { eprintln!("[cortex-plugin] Warning: boot call failed for agent '{agent}': {e}"); } } - if emit_port { println!("{}", paths.port); } Ok(()) } - #[cfg(windows)] pub(crate) async fn ensure_service_ready_async() -> bool { - tokio::task::spawn_blocking(service::ensure_ready) - .await - .unwrap_or(false) + tokio::task::spawn_blocking(service::ensure_ready).await.unwrap_or(false) } - #[cfg(not(windows))] pub(crate) fn plugin_owner_tag(agent: Option<&str>) -> String { let normalized = agent .unwrap_or("plugin") .trim() .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { - ch.to_ascii_lowercase() - } else { - '-' - } - }) + .map(|ch| if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { ch.to_ascii_lowercase() } else { '-' }) .collect::() .trim_matches('-') .to_string(); @@ -783,38 +495,26 @@ pub(crate) fn plugin_owner_tag(agent: Option<&str>) -> String { format!("plugin-{normalized}") } } - pub(crate) fn normalized_path_for_guard(path: &Path) -> String { - path.to_string_lossy() - .replace('\\', "/") - .to_ascii_lowercase() + path.to_string_lossy().replace('\\', "/").to_ascii_lowercase() } - pub(crate) fn path_is_under_root(path: &Path, root: &Path) -> bool { let normalized_path = normalized_path_for_guard(path); let mut normalized_root = normalized_path_for_guard(root); if !normalized_root.ends_with('/') { normalized_root.push('/'); } - normalized_path == normalized_root.trim_end_matches('/') - || normalized_path.starts_with(&normalized_root) + normalized_path == normalized_root.trim_end_matches('/') || normalized_path.starts_with(&normalized_root) } - pub(crate) fn is_disallowed_startup_binary_path(path: &Path) -> bool { let normalized = normalized_path_for_guard(path); - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); - + let file_name = path.file_name().and_then(|name| name.to_str()).unwrap_or_default().to_ascii_lowercase(); if file_name.starts_with("cortex-daemon-run") { return true; } if normalized.contains("/daemon-lifecycle-runtime/") { return true; } - let mut temp_roots = vec![std::env::temp_dir()]; if let Ok(temp) = std::env::var("TEMP") { temp_roots.push(PathBuf::from(temp)); @@ -822,30 +522,18 @@ pub(crate) fn is_disallowed_startup_binary_path(path: &Path) -> bool { if let Ok(tmp) = std::env::var("TMP") { temp_roots.push(PathBuf::from(tmp)); } - temp_roots - .iter() - .any(|root| !root.as_os_str().is_empty() && path_is_under_root(path, root)) + temp_roots.iter().any(|root| !root.as_os_str().is_empty() && path_is_under_root(path, root)) } - #[cfg(not(windows))] -pub(crate) async fn ensure_local_plugin_spawn_async( - paths: &auth::CortexPaths, - agent: Option<&str>, -) -> Result<(), String> { +pub(crate) async fn ensure_local_plugin_spawn_async(paths: &auth::CortexPaths, agent: Option<&str>) -> Result<(), String> { let current_exe = std::env::current_exe().map_err(|e| format!("resolve cortex binary: {e}"))?; if is_disallowed_startup_binary_path(¤t_exe) { - return Err(format!( - "refusing to launch daemon from disallowed runtime path: {}", - current_exe.display() - )); + return Err(format!("refusing to launch daemon from disallowed runtime path: {}", current_exe.display())); } let parent_pid = std::process::id(); - let parent_start = process_pid_start_time(parent_pid) - .ok_or_else(|| format!("resolve spawn parent start time for pid {parent_pid}"))?; + let parent_start = process_pid_start_time(parent_pid).ok_or_else(|| format!("resolve spawn parent start time for pid {parent_pid}"))?; let owner_tag = plugin_owner_tag(agent); - let owner_token = issue_owner_token_for_spawn(paths, &owner_tag, parent_pid) - .map_err(|e| format!("issue owner token: {e}"))?; - + let owner_token = issue_owner_token_for_spawn(paths, &owner_tag, parent_pid).map_err(|e| format!("issue owner token: {e}"))?; let mut cmd = std::process::Command::new(current_exe); cmd.arg("serve") .arg("--home") @@ -866,16 +554,10 @@ pub(crate) async fn ensure_local_plugin_spawn_async( .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()); - - cmd.spawn() - .map_err(|e| format!("spawn local daemon from plugin mode: {e}"))?; - + cmd.spawn().map_err(|e| format!("spawn local daemon from plugin mode: {e}"))?; if wait_for_health(paths, Duration::from_secs(DAEMON_STARTUP_WAIT_SECS)).await { Ok(()) } else { - Err(format!( - "daemon spawn started but health is still unavailable on port {}", - paths.port - )) + Err(format!("daemon spawn started but health is still unavailable on port {}", paths.port)) } } diff --git a/daemon-rs/src/cli/doctor.rs b/daemon-rs/src/cli/doctor.rs index e568c468..5259f206 100644 --- a/daemon-rs/src/cli/doctor.rs +++ b/daemon-rs/src/cli/doctor.rs @@ -1,19 +1,11 @@ -// SPDX-License-Identifier: MIT - -use serde_json::Value; -use std::collections::HashSet; - +use super::cleanup::{event_type_count, top_event_type_counts}; use crate::auth; use crate::compaction; use crate::db; - -use super::cleanup::{event_type_count, top_event_type_counts}; -use super::common::{json_field, json_str, json_str_or}; - +use std::collections::HashSet; pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { let db_path = paths.db.clone(); println!("[doctor] db_path={}", db_path.display()); - let conn = match db::open(&db_path) { Ok(v) => v, Err(e) => { @@ -25,7 +17,6 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { eprintln!("[doctor] FAIL configure: {e}"); std::process::exit(1); } - let expected_tables = [ "memories", "decisions", @@ -48,25 +39,12 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { "memories_fts", "decisions_fts", ]; - - let missing_tables: Vec<&str> = expected_tables - .iter() - .copied() - .filter(|table| !db::table_exists(&conn, table)) - .collect(); + let missing_tables: Vec<&str> = expected_tables.iter().copied().filter(|table| !db::table_exists(&conn, table)).collect(); if missing_tables.is_empty() { - println!( - "[doctor] OK tables: {}/{}", - expected_tables.len(), - expected_tables.len() - ); + println!("[doctor] OK tables: {}/{}", expected_tables.len(), expected_tables.len()); } else { - println!( - "[doctor] FAIL tables missing: {}", - missing_tables.join(", ") - ); + println!("[doctor] FAIL tables missing: {}", missing_tables.join(", ")); } - let (schema_current, pending_versions) = match db::pending_migration_versions(&conn) { Ok(pending) => (pending.is_empty(), pending), Err(e) => { @@ -75,34 +53,21 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { } }; if schema_current { - let expected_versions: HashSet<&'static str> = db::migration_definitions() - .iter() - .map(|(version, _)| *version) - .collect(); + let expected_versions: HashSet<&'static str> = db::migration_definitions().iter().map(|(version, _)| *version).collect(); let (applied, marker_rows) = db::applied_migration_versions(&conn) .map(|versions| { - let schema_applied = versions - .iter() - .filter(|version| expected_versions.contains(version.as_str())) - .count(); + let schema_applied = versions.iter().filter(|version| expected_versions.contains(version.as_str())).count(); let non_schema_markers = versions.len().saturating_sub(schema_applied); (schema_applied, non_schema_markers) }) .unwrap_or((0, 0)); - println!( - "[doctor] OK schema current: {applied}/{} migrations applied", - db::migration_definitions().len() - ); + println!("[doctor] OK schema current: {applied}/{} migrations applied", db::migration_definitions().len()); if marker_rows > 0 { println!("[doctor] INFO schema markers: {marker_rows} non-schema row(s) ignored"); } } else if !pending_versions.is_empty() { - println!( - "[doctor] FAIL schema pending: {}", - pending_versions.join(", ") - ); + println!("[doctor] FAIL schema pending: {}", pending_versions.join(", ")); } - let integrity_ok = match db::verify_integrity(&conn) { Ok(true) => { println!("[doctor] OK integrity_check"); @@ -117,34 +82,13 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { false } }; - - let fts_trigger_names = [ - "memories_fts_ai", - "memories_fts_ad", - "memories_fts_au", - "decisions_fts_ai", - "decisions_fts_ad", - "decisions_fts_au", - ]; - let fts_tables_ok = - db::table_exists(&conn, "memories_fts") && db::table_exists(&conn, "decisions_fts"); - let fts_queries_ok = conn - .query_row("SELECT COUNT(*) FROM memories_fts", [], |row| { - row.get::<_, i64>(0) - }) - .is_ok() - && conn - .query_row("SELECT COUNT(*) FROM decisions_fts", [], |row| { - row.get::<_, i64>(0) - }) - .is_ok(); + let fts_trigger_names = ["memories_fts_ai", "memories_fts_ad", "memories_fts_au", "decisions_fts_ai", "decisions_fts_ad", "decisions_fts_au"]; + let fts_tables_ok = db::table_exists(&conn, "memories_fts") && db::table_exists(&conn, "decisions_fts"); + let fts_queries_ok = conn.query_row("SELECT COUNT(*) FROM memories_fts", [], |row| row.get::<_, i64>(0)).is_ok() + && conn.query_row("SELECT COUNT(*) FROM decisions_fts", [], |row| row.get::<_, i64>(0)).is_ok(); let fts_triggers_ok = fts_trigger_names.iter().all(|name| { - conn.query_row( - "SELECT 1 FROM sqlite_master WHERE type='trigger' AND name=?1 LIMIT 1", - rusqlite::params![name], - |_| Ok(()), - ) - .is_ok() + conn.query_row("SELECT 1 FROM sqlite_master WHERE type='trigger' AND name=?1 LIMIT 1", rusqlite::params![name], |_| Ok(())) + .is_ok() }); let fts_ok = fts_tables_ok && fts_queries_ok && fts_triggers_ok; if fts_ok { @@ -152,7 +96,6 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { } else { println!("[doctor] FAIL fts indexes"); } - let nonboot_event_rows = compaction::non_boot_event_count(&conn); let decision_stored_rows = event_type_count(&conn, "decision_stored"); let event_pressure = compaction::classify_event_pressure(nonboot_event_rows); @@ -172,17 +115,13 @@ pub(crate) fn run_doctor_cli(paths: &auth::CortexPaths) { } } if event_pressure != "normal" { - println!( - "[doctor] WARN elevated event pressure detected; run `cortex cleanup --events --dry-run` to preview one-time remediation." - ); + println!("[doctor] WARN elevated event pressure detected; run `cortex cleanup --events --dry-run` to preview one-time remediation."); } - let all_ok = missing_tables.is_empty() && schema_current && integrity_ok && fts_ok; if all_ok { println!("[doctor] GREEN"); return; } - println!("[doctor] RED"); std::process::exit(1); } diff --git a/daemon-rs/src/cli/embeddings.rs b/daemon-rs/src/cli/embeddings.rs index 62c8993b..85569dab 100644 --- a/daemon-rs/src/cli/embeddings.rs +++ b/daemon-rs/src/cli/embeddings.rs @@ -1,22 +1,12 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; -use std::time::Duration; - -use crate::auth; -use crate::db; -use crate::embeddings; -use crate::state; - -use super::common::{ - parse_env_usize, parse_flag_usize, parse_flag_value, validate_cli_options, - validate_cli_options_or_exit, -}; +use super::common::{parse_env_usize, parse_flag_usize}; use super::daemon::{ - background_db_lock_max_wait, build_embeddings_async, count_unembedded_targets_for_model, - ensure_daemon, DEFAULT_EMBED_BACKFILL_BATCH_SIZE, DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS, + background_db_lock_max_wait, build_embeddings_async, count_unembedded_targets_for_model, DEFAULT_EMBED_BACKFILL_BATCH_SIZE, + DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS, }; - +use crate::auth; +use crate::state; +use serde_json::json; +use std::time::Duration; pub(crate) async fn run_embeddings_cli(paths: &auth::CortexPaths, args: &[String]) { let subcmd = args.first().map(|s| s.as_str()).unwrap_or(""); match subcmd { @@ -29,13 +19,12 @@ pub(crate) async fn run_embeddings_cli(paths: &auth::CortexPaths, args: &[String } _ => { eprintln!( - "Usage: cortex embeddings [--json] [--batch-size ] [--max-batches ] [--lock-wait-ms ] [--until-exhausted] [--max-iterations ]" - ); +"Usage: cortex embeddings [--json] [--batch-size ] [--max-batches ] [--lock-wait-ms ] [--until-exhausted] [--max-iterations ]" +); std::process::exit(1); } } } - pub(crate) async fn run_embeddings_status_cli(paths: &auth::CortexPaths, json_output: bool) { let (state, _shutdown_rx) = match state::initialize(paths, false) { Ok(initialized) => initialized, @@ -45,49 +34,31 @@ pub(crate) async fn run_embeddings_status_cli(paths: &auth::CortexPaths, json_ou } }; let Some(engine) = state.embedding_engine.clone() else { - eprintln!( - "[embeddings] No embedding model is currently loaded. Run `cortex serve` once to trigger model download, then retry." - ); + eprintln!("[embeddings] No embedding model is currently loaded. Run `cortex serve` once to trigger model download, then retry."); std::process::exit(1); }; - let model_key = engine.model_key().to_string(); let (backlog_memories, backlog_decisions) = { let conn = state.db.lock().await; count_unembedded_targets_for_model(&conn, &model_key) }; let backlog_total = backlog_memories + backlog_decisions; - if json_output { println!( "{}", - json!({ - "model": model_key, - "backlog": { - "memories": backlog_memories, - "decisions": backlog_decisions, - "total": backlog_total - } + json!({"model":model_key,"backlog":{"memories":backlog_memories,"decisions":backlog_decisions,"total":backlog_total} }) ); } else { println!("Embeddings status"); println!("model: {model_key}"); - println!( - "backlog: memories={}, decisions={}, total={}", - backlog_memories, backlog_decisions, backlog_total - ); + println!("backlog: memories={}, decisions={}, total={}", backlog_memories, backlog_decisions, backlog_total); } } - pub(crate) async fn run_embeddings_drain_cli(paths: &auth::CortexPaths, args: &[String]) { let batch_size = match parse_flag_usize(args, "--batch-size") { Ok(Some(value)) => value.clamp(1, 10_000), - Ok(None) => parse_env_usize( - "CORTEX_EMBED_BACKFILL_BATCH_SIZE", - DEFAULT_EMBED_BACKFILL_BATCH_SIZE, - ) - .clamp(1, 10_000), + Ok(None) => parse_env_usize("CORTEX_EMBED_BACKFILL_BATCH_SIZE", DEFAULT_EMBED_BACKFILL_BATCH_SIZE).clamp(1, 10_000), Err(err) => { eprintln!("Error: {err}"); std::process::exit(1); @@ -95,11 +66,7 @@ pub(crate) async fn run_embeddings_drain_cli(paths: &auth::CortexPaths, args: &[ }; let max_batches_per_pass = match parse_flag_usize(args, "--max-batches") { Ok(Some(value)) => value.clamp(1, 10_000), - Ok(None) => parse_env_usize( - "CORTEX_EMBED_BACKFILL_MAX_BATCHES_PER_PASS", - DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS, - ) - .clamp(1, 10_000), + Ok(None) => parse_env_usize("CORTEX_EMBED_BACKFILL_MAX_BATCHES_PER_PASS", DEFAULT_EMBED_BACKFILL_MAX_BATCHES_PER_PASS).clamp(1, 10_000), Err(err) => { eprintln!("Error: {err}"); std::process::exit(1); @@ -124,7 +91,6 @@ pub(crate) async fn run_embeddings_drain_cli(paths: &auth::CortexPaths, args: &[ let until_exhausted = args.iter().any(|arg| arg == "--until-exhausted"); let json_output = args.iter().any(|arg| arg == "--json"); let lock_wait = Duration::from_millis(lock_wait_ms as u64); - let (state, _shutdown_rx) = match state::initialize(paths, false) { Ok(initialized) => initialized, Err(err) => { @@ -133,88 +99,49 @@ pub(crate) async fn run_embeddings_drain_cli(paths: &auth::CortexPaths, args: &[ } }; let Some(engine) = state.embedding_engine.clone() else { - eprintln!( - "[embeddings] No embedding model is currently loaded. Run `cortex serve` once to trigger model download, then retry." - ); + eprintln!("[embeddings] No embedding model is currently loaded. Run `cortex serve` once to trigger model download, then retry."); std::process::exit(1); }; let model_key = engine.model_key().to_string(); - let mut iterations_ran = 0usize; let mut queued_total = 0usize; let mut computed_total = 0usize; let mut passes_ran = 0usize; let mut exhausted = false; - while iterations_ran < max_iterations { iterations_ran += 1; - let pass = build_embeddings_async( - engine.clone(), - &state.db, - batch_size, - max_batches_per_pass, - lock_wait, - ) - .await; + let pass = build_embeddings_async(engine.clone(), &state.db, batch_size, max_batches_per_pass, lock_wait).await; queued_total += pass.queued_total; computed_total += pass.computed_total; passes_ran += pass.passes_ran; exhausted = pass.exhausted; - if pass.exhausted || pass.queued_total == 0 || !until_exhausted { break; } } - let (remaining_memories, remaining_decisions) = { let conn = state.db.lock().await; count_unembedded_targets_for_model(&conn, &model_key) }; let remaining_total = remaining_memories + remaining_decisions; exhausted = exhausted || remaining_total == 0; - if json_output { println!( "{}", - json!({ - "model": model_key, - "batch_size": batch_size, - "max_batches_per_pass": max_batches_per_pass, - "lock_wait_ms": lock_wait_ms, - "until_exhausted": until_exhausted, - "max_iterations": max_iterations, - "iterations_ran": iterations_ran, - "queued_total": queued_total, - "computed_total": computed_total, - "passes_ran": passes_ran, - "remaining": { - "memories": remaining_memories, - "decisions": remaining_decisions, - "total": remaining_total - }, - "exhausted": exhausted - }) + json!({"model":model_key,"batch_size":batch_size,"max_batches_per_pass" +:max_batches_per_pass,"lock_wait_ms":lock_wait_ms,"until_exhausted":until_exhausted,"max_iterations":max_iterations, +"iterations_ran":iterations_ran,"queued_total":queued_total,"computed_total":computed_total,"passes_ran":passes_ran,"remaining":{ +"memories":remaining_memories,"decisions":remaining_decisions,"total":remaining_total},"exhausted":exhausted}) ); } else { println!("Embeddings drain"); println!("model: {model_key}"); - println!( - "drain: queued={}, built={}, passes={}, iterations={}", - queued_total, computed_total, passes_ran, iterations_ran - ); - println!( - "remaining: memories={}, decisions={}, total={}", - remaining_memories, remaining_decisions, remaining_total - ); + println!("drain: queued={}, built={}, passes={}, iterations={}", queued_total, computed_total, passes_ran, iterations_ran); + println!("remaining: memories={}, decisions={}, total={}", remaining_memories, remaining_decisions, remaining_total); println!("exhausted: {exhausted}"); } - if until_exhausted && !exhausted { - eprintln!( - "[embeddings] backlog still pending after {} iteration(s); rerun with higher --max-iterations or --max-batches", - iterations_ran - ); + eprintln!("[embeddings] backlog still pending after {} iteration(s); rerun with higher --max-iterations or --max-batches", iterations_ran); std::process::exit(2); } } - diff --git a/daemon-rs/src/cli/eval.rs b/daemon-rs/src/cli/eval.rs index 7ca8c95f..457f99db 100644 --- a/daemon-rs/src/cli/eval.rs +++ b/daemon-rs/src/cli/eval.rs @@ -1,33 +1,18 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; - +use super::common::{open_cli_connection, parse_flag_usize, parse_flag_value, validate_cli_options_or_exit}; use crate::auth; use crate::eval; - -use super::common::{ - open_cli_connection, parse_flag_usize, parse_flag_value, validate_cli_options_or_exit, -}; - +use serde_json::{json, Value}; pub(crate) fn run_eval_cli(paths: &auth::CortexPaths, args: &[String]) { - validate_cli_options_or_exit( - args, - &["--baseline-file", "--max-regression", "--window-days"], - &["--json", "--fail-on-regression"], - ); + validate_cli_options_or_exit(args, &["--baseline-file", "--max-regression", "--window-days"], &["--json", "--fail-on-regression"]); let json_output = args.iter().any(|arg| arg == "--json"); let fail_on_regression = args.iter().any(|arg| arg == "--fail-on-regression"); let baseline_file = parse_flag_value(args, "--baseline-file"); let max_regression = match parse_flag_value(args, "--max-regression") { Some(raw) => { - let parsed = raw - .trim() - .parse::() - .map_err(|_| format!("invalid value for --max-regression: '{raw}'")) - .unwrap_or_else(|err| { - eprintln!("{err}"); - std::process::exit(1); - }); + let parsed = raw.trim().parse::().map_err(|_| format!("invalid value for --max-regression: '{raw}'")).unwrap_or_else(|err| { + eprintln!("{err}"); + std::process::exit(1); + }); if !(0.0..=1.0).contains(&parsed) { eprintln!("--max-regression must be between 0.0 and 1.0"); std::process::exit(1); @@ -44,7 +29,6 @@ pub(crate) fn run_eval_cli(paths: &auth::CortexPaths, args: &[String]) { std::process::exit(1); } }; - let conn = match open_cli_connection(&paths.db) { Ok(conn) => conn, Err(err) => { @@ -68,118 +52,51 @@ pub(crate) fn run_eval_cli(paths: &auth::CortexPaths, args: &[String]) { map.insert("regressionGate".to_string(), gate); } if json_output { - println!( - "{}", - serde_json::to_string_pretty(&snapshot).unwrap_or_else(|_| "{}".to_string()) - ); - if fail_on_regression - && regression_gate - .as_ref() - .and_then(|gate| gate.get("ok")) - .and_then(Value::as_bool) - == Some(false) - { + println!("{}", serde_json::to_string_pretty(&snapshot).unwrap_or_else(|_| "{}".to_string())); + if fail_on_regression && regression_gate.as_ref().and_then(|gate| gate.get("ok")).and_then(Value::as_bool) == Some(false) { std::process::exit(2); } return; } - let totals = snapshot.get("totals").cloned().unwrap_or_else(|| json!({})); let window = snapshot.get("window").cloned().unwrap_or_else(|| json!({})); - let signals = snapshot - .get("signals") - .cloned() - .unwrap_or_else(|| json!({})); + let signals = snapshot.get("signals").cloned().unwrap_or_else(|| json!({})); println!("Eval snapshot ({window_days}d)"); println!( "active: memories={}, decisions={}, open_conflicts={}", - totals - .get("activeMemories") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0), - totals - .get("activeDecisions") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0), - totals - .get("openConflicts") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0) + totals.get("activeMemories").and_then(serde_json::Value::as_i64).unwrap_or(0), + totals.get("activeDecisions").and_then(serde_json::Value::as_i64).unwrap_or(0), + totals.get("openConflicts").and_then(serde_json::Value::as_i64).unwrap_or(0) ); println!( "window: conflicts={}, resolutions={}, recalls={}", - window - .get("recentConflicts") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0), - window - .get("recentResolutions") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0), - window - .get("recentRecallQueries") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0) + window.get("recentConflicts").and_then(serde_json::Value::as_i64).unwrap_or(0), + window.get("recentResolutions").and_then(serde_json::Value::as_i64).unwrap_or(0), + window.get("recentRecallQueries").and_then(serde_json::Value::as_i64).unwrap_or(0) ); println!( "signals: conflict_burden={:.4}, decay_burden={:.4}, resolution_velocity={:.4}, contradiction_rate={:.4}", - signals - .get("conflictBurden") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("decayBurden") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("resolutionVelocity") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("contradictionRate") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0) + signals.get("conflictBurden").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("decayBurden").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("resolutionVelocity").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("contradictionRate").and_then(serde_json::Value::as_f64).unwrap_or(0.0) ); println!( "task: success_rate={:.4}, first_pass={:.4}, median_time_ms={:.2}, retry_count={:.4}", - signals - .get("taskSuccessRate") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("firstPassSuccess") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("medianTimeToValidResultMs") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("retryCount") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0) + signals.get("taskSuccessRate").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("firstPassSuccess").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("medianTimeToValidResultMs").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("retryCount").and_then(serde_json::Value::as_f64).unwrap_or(0.0) ); println!( "memory_quality: stale_hit_rate={:.4}, low_trust_hit_rate={:.4}, consensus_precision={:.4}", - signals - .get("staleMemoryHitRate") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("lowTrustHitRate") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - signals - .get("consensusPromotionPrecision") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0) + signals.get("staleMemoryHitRate").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("lowTrustHitRate").and_then(serde_json::Value::as_f64).unwrap_or(0.0), + signals.get("consensusPromotionPrecision").and_then(serde_json::Value::as_f64).unwrap_or(0.0) ); if let Some(gate) = regression_gate { let gate_ok = gate.get("ok").and_then(Value::as_bool).unwrap_or(true); - println!( - "regression_gate: ok={}, max_regression={:.3}", - gate_ok, max_regression - ); + println!("regression_gate: ok={}, max_regression={:.3}", gate_ok, max_regression); if fail_on_regression && !gate_ok { std::process::exit(2); } diff --git a/daemon-rs/src/cli/mod.rs b/daemon-rs/src/cli/mod.rs index c27b4e36..a4d21290 100644 --- a/daemon-rs/src/cli/mod.rs +++ b/daemon-rs/src/cli/mod.rs @@ -1,5 +1,3 @@ -// SPDX-License-Identifier: MIT - mod admin; mod boot; mod cleanup; @@ -11,32 +9,23 @@ mod eval; mod reindex; mod status; mod sync; -mod usage; - #[cfg(test)] mod tests; - -pub(crate) use admin::{ - run_admin_budgets_cli, run_admin_cli, run_admin_rollback_cli, run_team_cli, run_user_cli, -}; +mod usage; +pub(crate) use admin::{run_admin_cli, run_team_cli, run_user_cli}; pub(crate) use boot::run_boot_cli; pub(crate) use cleanup::{run_backup_cli, run_cleanup_cli, run_restore_cli}; pub(crate) use common::{ - apply_path_env, ensure_remote_target_has_api_key, is_cli_option_token, parse_flag_usize, - parse_flag_value, required_cli_positional_or_exit, resolve_client_target, - validate_cli_options_or_exit, SINGLE_DAEMON_TEST_BYPASS_ENV, -}; -pub(crate) use daemon::{ - ensure_daemon, is_disallowed_startup_binary_path, run_daemon, startup_single_daemon_preflight, + apply_path_env, ensure_remote_target_has_api_key, parse_flag_usize, parse_flag_value, resolve_client_target, validate_cli_options_or_exit, }; +pub(crate) use daemon::{ensure_daemon, is_disallowed_startup_binary_path, run_daemon}; pub(crate) use doctor::run_doctor_cli; pub(crate) use embeddings::{run_embeddings_cli, run_embeddings_drain_cli}; pub(crate) use eval::run_eval_cli; pub(crate) use reindex::{run_recrystallize_cli, run_reindex_cli}; -pub(crate) use status::{build_status_report, run_status_cli, STATUS_SCHEMA_VERSION}; +pub(crate) use status::run_status_cli; pub(crate) use sync::{run_export_cli, run_import_cli, run_sync_cli}; pub(crate) use usage::{ - cli_capabilities_payload, cli_capabilities_summary, cli_robot_docs_guide, cli_service_usage, - cli_usage_text, print_usage_and_exit, unknown_cli_command_message, - unknown_robot_docs_subcommand_message, CLI_CAPABILITIES_CONTRACT_VERSION, + cli_capabilities_payload, cli_capabilities_summary, cli_robot_docs_guide, cli_service_usage, print_usage_and_exit, unknown_cli_command_message, + unknown_robot_docs_subcommand_message, }; diff --git a/daemon-rs/src/cli/reindex.rs b/daemon-rs/src/cli/reindex.rs index 2ea0e129..1e1da4ad 100644 --- a/daemon-rs/src/cli/reindex.rs +++ b/daemon-rs/src/cli/reindex.rs @@ -1,15 +1,8 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; - use crate::auth; use crate::crystallize; use crate::db; -use crate::indexer; use crate::state; - -use super::common::{open_cli_connection, validate_cli_options_or_exit}; - +use serde_json::json; pub(crate) fn run_reindex_cli(paths: &auth::CortexPaths, json_output: bool) { let conn = match db::open(&paths.db) { Ok(conn) => conn, @@ -22,59 +15,27 @@ pub(crate) fn run_reindex_cli(paths: &auth::CortexPaths, json_output: bool) { eprintln!("Error: failed to configure database for reindex: {err}"); std::process::exit(1); } - - let memories_base = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0); - let decisions_base = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'active'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0); - + let memories_base = conn.query_row("SELECT COUNT(*) FROM memories WHERE status = 'active'", [], |row| row.get::<_, i64>(0)).unwrap_or(0); + let decisions_base = conn.query_row("SELECT COUNT(*) FROM decisions WHERE status = 'active'", [], |row| row.get::<_, i64>(0)).unwrap_or(0); if let Err(err) = db::reindex_fts(&conn) { eprintln!("Error: failed to rebuild FTS indexes: {err}"); std::process::exit(1); } - - let memories_fts = conn - .query_row("SELECT COUNT(*) FROM memories_fts", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); - let decisions_fts = conn - .query_row("SELECT COUNT(*) FROM decisions_fts", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); - + let memories_fts = conn.query_row("SELECT COUNT(*) FROM memories_fts", [], |row| row.get::<_, i64>(0)).unwrap_or(0); + let decisions_fts = conn.query_row("SELECT COUNT(*) FROM decisions_fts", [], |row| row.get::<_, i64>(0)).unwrap_or(0); if json_output { println!( "{}", - json!({ - "reindexed": true, - "counts": { - "memories_base": memories_base, - "memories_fts": memories_fts, - "decisions_base": decisions_base, - "decisions_fts": decisions_fts, - } - }) + json!({"reindexed": +true,"counts":{"memories_base":memories_base,"memories_fts":memories_fts,"decisions_base":decisions_base,"decisions_fts": +decisions_fts,}}) ); return; } - println!("Reindex complete"); println!("memories: base={memories_base}, fts={memories_fts}"); println!("decisions: base={decisions_base}, fts={decisions_fts}"); } - pub(crate) async fn run_recrystallize_cli(paths: &auth::CortexPaths, json_output: bool) { let (state, _shutdown_rx) = match state::initialize(paths, false) { Ok(initialized) => initialized, @@ -83,130 +44,50 @@ pub(crate) async fn run_recrystallize_cli(paths: &auth::CortexPaths, json_output std::process::exit(1); } }; - let result_payload = { let conn = state.db.lock().await; - - let crystals_before: i64 = conn - .query_row("SELECT COUNT(*) FROM memory_clusters", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); - let members_before: i64 = conn - .query_row("SELECT COUNT(*) FROM cluster_members", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); + let crystals_before: i64 = conn.query_row("SELECT COUNT(*) FROM memory_clusters", [], |row| row.get::<_, i64>(0)).unwrap_or(0); + let members_before: i64 = conn.query_row("SELECT COUNT(*) FROM cluster_members", [], |row| row.get::<_, i64>(0)).unwrap_or(0); let embeddings_before: i64 = conn - .query_row( - "SELECT COUNT(*) FROM embeddings WHERE target_type = 'crystal'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0); - - let removed_embeddings = conn - .execute("DELETE FROM embeddings WHERE target_type = 'crystal'", []) + .query_row("SELECT COUNT(*) FROM embeddings WHERE target_type = 'crystal'", [], |row| row.get::<_, i64>(0)) .unwrap_or(0); + let removed_embeddings = conn.execute("DELETE FROM embeddings WHERE target_type = 'crystal'", []).unwrap_or(0); let removed_crystals = conn.execute("DELETE FROM memory_clusters", []).unwrap_or(0); - let brain_sender = Some(state.brain_firing.clone()); - let pass = crystallize::run_crystallize_pass_with_brain( - &conn, - state.embedding_engine.as_deref(), - state.default_owner_id, - &brain_sender, - ); - - let crystals_after: i64 = conn - .query_row("SELECT COUNT(*) FROM memory_clusters", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); - let members_after: i64 = conn - .query_row("SELECT COUNT(*) FROM cluster_members", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap_or(0); + let pass = crystallize::run_crystallize_pass_with_brain(&conn, state.embedding_engine.as_deref(), state.default_owner_id, &brain_sender); + let crystals_after: i64 = conn.query_row("SELECT COUNT(*) FROM memory_clusters", [], |row| row.get::<_, i64>(0)).unwrap_or(0); + let members_after: i64 = conn.query_row("SELECT COUNT(*) FROM cluster_members", [], |row| row.get::<_, i64>(0)).unwrap_or(0); let embeddings_after: i64 = conn - .query_row( - "SELECT COUNT(*) FROM embeddings WHERE target_type = 'crystal'", - [], - |row| row.get::<_, i64>(0), - ) + .query_row("SELECT COUNT(*) FROM embeddings WHERE target_type = 'crystal'", [], |row| row.get::<_, i64>(0)) .unwrap_or(0); - - json!({ - "recrystallized": true, - "owner_filter": state.default_owner_id, - "removed": { - "crystals": removed_crystals, - "members": members_before, - "embeddings": removed_embeddings - }, - "before": { - "crystals": crystals_before, - "members": members_before, - "embeddings": embeddings_before - }, - "pass": { - "clusters_found": pass.clusters_found, - "crystals_created": pass.crystals_created, - "crystals_updated": pass.crystals_updated, - "entries_consolidated": pass.entries_consolidated - }, - "after": { - "crystals": crystals_after, - "members": members_after, - "embeddings": embeddings_after - } - }) + json!({"recrystallized":true,"owner_filter":state.default_owner_id,"removed":{"crystals":removed_crystals,"members": +members_before,"embeddings":removed_embeddings},"before":{"crystals":crystals_before,"members":members_before,"embeddings": +embeddings_before},"pass":{"clusters_found":pass.clusters_found,"crystals_created":pass.crystals_created,"crystals_updated":pass. +crystals_updated,"entries_consolidated":pass.entries_consolidated},"after":{"crystals":crystals_after,"members":members_after, +"embeddings":embeddings_after}}) }; - if json_output { println!("{result_payload}"); return; } - println!("Recrystallize complete"); println!( "removed: crystals={}, members={}, crystal_embeddings={}", - result_payload["removed"]["crystals"] - .as_i64() - .unwrap_or_default(), - result_payload["removed"]["members"] - .as_i64() - .unwrap_or_default(), - result_payload["removed"]["embeddings"] - .as_i64() - .unwrap_or_default() + result_payload["removed"]["crystals"].as_i64().unwrap_or_default(), + result_payload["removed"]["members"].as_i64().unwrap_or_default(), + result_payload["removed"]["embeddings"].as_i64().unwrap_or_default() ); println!( "pass: clusters_found={}, created={}, updated={}, consolidated={}", - result_payload["pass"]["clusters_found"] - .as_u64() - .unwrap_or_default(), - result_payload["pass"]["crystals_created"] - .as_u64() - .unwrap_or_default(), - result_payload["pass"]["crystals_updated"] - .as_u64() - .unwrap_or_default(), - result_payload["pass"]["entries_consolidated"] - .as_u64() - .unwrap_or_default() + result_payload["pass"]["clusters_found"].as_u64().unwrap_or_default(), + result_payload["pass"]["crystals_created"].as_u64().unwrap_or_default(), + result_payload["pass"]["crystals_updated"].as_u64().unwrap_or_default(), + result_payload["pass"]["entries_consolidated"].as_u64().unwrap_or_default() ); println!( "after: crystals={}, members={}, crystal_embeddings={}", - result_payload["after"]["crystals"] - .as_i64() - .unwrap_or_default(), - result_payload["after"]["members"] - .as_i64() - .unwrap_or_default(), - result_payload["after"]["embeddings"] - .as_i64() - .unwrap_or_default() + result_payload["after"]["crystals"].as_i64().unwrap_or_default(), + result_payload["after"]["members"].as_i64().unwrap_or_default(), + result_payload["after"]["embeddings"].as_i64().unwrap_or_default() ); } - diff --git a/daemon-rs/src/cli/status.rs b/daemon-rs/src/cli/status.rs index 21db7a06..250d27c8 100644 --- a/daemon-rs/src/cli/status.rs +++ b/daemon-rs/src/cli/status.rs @@ -1,16 +1,10 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; -use std::time::Duration; - +use super::common::json_str; use crate::auth; use crate::daemon_lifecycle::{is_cortex_health_payload, readiness_state_from_payload}; use crate::transport; - -use super::common::json_str; - +use serde_json::{json, Value}; +use std::time::Duration; pub(crate) const STATUS_SCHEMA_VERSION: u32 = 1; - #[derive(Debug, Clone)] struct StatusRepair { kind: &'static str, @@ -18,7 +12,6 @@ struct StatusRepair { command: Option, docs: &'static str, } - #[derive(Debug, Clone)] struct StatusCheck { name: &'static str, @@ -26,35 +19,29 @@ struct StatusCheck { detail: String, repair: Option, } - #[derive(Debug, Clone)] -enum StatusRuntimeProbe { +pub(crate) enum StatusRuntimeProbe { Ready(String), Starting(String), WrongIdentity(String), Unavailable(String), Error(String), } - -struct StatusReport { +pub(crate) struct StatusReport { payload: Value, exit_code: i32, } - fn status_docs_path() -> &'static str { "Info/connecting.md" } - fn status_start_repair() -> StatusRepair { StatusRepair { kind: "start_local_runtime", - label: "Open Cortex Control Center, or run `cortex serve` for CLI-only local mode." - .to_string(), + label: "Open Cortex Control Center, or run `cortex serve` for CLI-only local mode.".to_string(), command: Some("cortex serve".to_string()), docs: status_docs_path(), } } - fn status_wait_repair() -> StatusRepair { StatusRepair { kind: "wait_for_startup", @@ -63,18 +50,14 @@ fn status_wait_repair() -> StatusRepair { docs: status_docs_path(), } } - fn status_identity_repair() -> StatusRepair { StatusRepair { kind: "repair_runtime_identity", - label: - "Stop the other process or switch to the matching CORTEX_HOME/CORTEX_PORT, then retry." - .to_string(), + label: "Stop the other process or switch to the matching CORTEX_HOME/CORTEX_PORT, then retry.".to_string(), command: Some("cortex status --json".to_string()), docs: "Info/startup-matrix-troubleshooting.md", } } - fn status_doctor_repair() -> StatusRepair { StatusRepair { kind: "run_doctor", @@ -83,47 +66,30 @@ fn status_doctor_repair() -> StatusRepair { docs: status_docs_path(), } } - fn status_setup_repair() -> StatusRepair { StatusRepair { kind: "run_setup", - label: "Run `cortex setup`, then start Cortex from Control Center or `cortex serve`." - .to_string(), + label: "Run `cortex setup`, then start Cortex from Control Center or `cortex serve`.".to_string(), command: Some("cortex setup".to_string()), docs: status_docs_path(), } } - fn status_connect_next_action() -> StatusRepair { StatusRepair { kind: "connect_tool_or_smoke", - label: - "Connect an AI tool, then store and recall one memory; CLI users can start with `cortex boot --agent smoke-test --json`." - .to_string(), + label: "Connect an AI tool, then store and recall one memory; CLI users can start with `cortex boot --agent smoke-test --json`.".to_string(), command: Some("cortex boot --agent smoke-test --json".to_string()), docs: status_docs_path(), } } - fn status_repair_json(repair: &StatusRepair) -> Value { - json!({ - "kind": repair.kind, - "label": repair.label, - "command": repair.command, - "docs": repair.docs - }) + json!({"kind":repair.kind,"label":repair.label,"command":repair.command,"docs":repair.docs}) } - fn status_check_json(check: &StatusCheck) -> Value { let repair = check.repair.as_ref().map(status_repair_json); - json!({ - "name": check.name, - "status": check.status, - "detail": check.detail, - "repair": repair - }) + json!({"name":check.name, +"status":check.status,"detail":check.detail,"repair":repair}) } - fn compact_status_detail(value: &str) -> String { let compacted = value.split_whitespace().collect::>().join(" "); const MAX_DETAIL_CHARS: usize = 220; @@ -134,13 +100,7 @@ fn compact_status_detail(value: &str) -> String { out.push_str("..."); out } - -pub(crate) fn build_status_report( - paths: &auth::CortexPaths, - runtime_probe: StatusRuntimeProbe, - token_exists: bool, - db_exists: bool, -) -> StatusReport { +pub(crate) fn build_status_report(paths: &auth::CortexPaths, runtime_probe: StatusRuntimeProbe, token_exists: bool, db_exists: bool) -> StatusReport { let runtime_base_url = transport::local_http_base_url(paths); let (mut status, summary, runtime_check, mut next_action) = match runtime_probe { StatusRuntimeProbe::Ready(detail) => { @@ -148,12 +108,7 @@ pub(crate) fn build_status_report( ( "ready", "Cortex memory is ready for local AI tools.".to_string(), - StatusCheck { - name: "runtime_identity", - status: "ok", - detail, - repair: None, - }, + StatusCheck { name: "runtime_identity", status: "ok", detail, repair: None }, next, ) } @@ -162,12 +117,7 @@ pub(crate) fn build_status_report( ( "needs_action", "Cortex is starting but is not ready yet.".to_string(), - StatusCheck { - name: "runtime_identity", - status: "warn", - detail, - repair: Some(repair.clone()), - }, + StatusCheck { name: "runtime_identity", status: "warn", detail, repair: Some(repair.clone()) }, repair, ) } @@ -175,14 +125,8 @@ pub(crate) fn build_status_report( let repair = status_identity_repair(); ( "error", - "A Cortex-like service answered, but it is not the expected local runtime." - .to_string(), - StatusCheck { - name: "runtime_identity", - status: "fail", - detail, - repair: Some(repair.clone()), - }, + "A Cortex-like service answered, but it is not the expected local runtime.".to_string(), + StatusCheck { name: "runtime_identity", status: "fail", detail, repair: Some(repair.clone()) }, repair, ) } @@ -191,12 +135,7 @@ pub(crate) fn build_status_report( ( "needs_action", "No ready Cortex runtime answered the local readiness probe.".to_string(), - StatusCheck { - name: "runtime_identity", - status: "warn", - detail, - repair: Some(repair.clone()), - }, + StatusCheck { name: "runtime_identity", status: "warn", detail, repair: Some(repair.clone()) }, repair, ) } @@ -205,17 +144,11 @@ pub(crate) fn build_status_report( ( "error", "The local runtime probe returned an unexpected response.".to_string(), - StatusCheck { - name: "runtime_identity", - status: "fail", - detail, - repair: Some(repair.clone()), - }, + StatusCheck { name: "runtime_identity", status: "fail", detail, repair: Some(repair.clone()) }, repair, ) } }; - let mut checks = vec![runtime_check]; if token_exists { checks.push(StatusCheck { @@ -237,58 +170,27 @@ pub(crate) fn build_status_report( repair: Some(repair), }); } - checks.push(StatusCheck { name: "database", status: if db_exists { "ok" } else { "warn" }, detail: if db_exists { format!("Database exists at {}", paths.db.display()) } else { - format!( - "Database not found at {}; it is created during first setup/start.", - paths.db.display() - ) - }, - repair: if db_exists { - None - } else { - Some(status_setup_repair()) + format!("Database not found at {}; it is created during first setup/start.", paths.db.display()) }, + repair: if db_exists { None } else { Some(status_setup_repair()) }, }); - let exit_code = if status == "ready" { 0 } else { 1 }; - let repair = if status == "ready" { - None - } else { - Some(status_repair_json(&next_action)) - }; - let payload = json!({ - "schemaVersion": STATUS_SCHEMA_VERSION, - "status": status, - "summary": summary, - "version": env!("CARGO_PKG_VERSION"), - "runtime": { - "baseUrl": runtime_base_url, - "port": paths.port, - "bind": paths.bind, - "home": paths.home.display().to_string(), - "dbPath": paths.db.display().to_string(), - "tokenPath": paths.token.display().to_string(), - "pidPath": paths.pid.display().to_string() - }, - "nextAction": status_repair_json(&next_action), - "repair": repair, - "checks": checks.iter().map(status_check_json).collect::>() - }); - + let repair = if status == "ready" { None } else { Some(status_repair_json(&next_action)) }; + let payload = json!({"schemaVersion":STATUS_SCHEMA_VERSION,"status":status,"summary":summary, +"version":env!("CARGO_PKG_VERSION"),"runtime":{"baseUrl":runtime_base_url,"port":paths.port,"bind":paths.bind,"home":paths.home. +display().to_string(),"dbPath":paths.db.display().to_string(),"tokenPath":paths.token.display().to_string(),"pidPath":paths.pid. +display().to_string()},"nextAction":status_repair_json(&next_action),"repair":repair,"checks":checks.iter().map(status_check_json) +.collect::>()}); StatusReport { payload, exit_code } } - async fn probe_status_runtime(paths: &auth::CortexPaths) -> StatusRuntimeProbe { - let client = match reqwest::Client::builder() - .timeout(Duration::from_secs(2)) - .build() - { + let client = match reqwest::Client::builder().timeout(Duration::from_secs(2)).build() { Ok(client) => client, Err(err) => { return StatusRuntimeProbe::Error(format!("Could not create HTTP client: {err}")); @@ -297,32 +199,14 @@ async fn probe_status_runtime(paths: &auth::CortexPaths) -> StatusRuntimeProbe { let base_url = transport::local_http_base_url(paths); let mut probe_errors = Vec::new(); let probe_headers = [(String::from("X-Cortex-Request"), String::from("true"))]; - - match transport::request_with_local_ipc_fallback( - &client, - "GET", - &base_url, - "/readiness", - paths, - &probe_headers, - None, - Duration::from_secs(2), - ) - .await - { + match transport::request_with_local_ipc_fallback(&client, "GET", &base_url, "/readiness", paths, &probe_headers, None, Duration::from_secs(2)).await { Ok((status, body)) => { let status_code = status.as_u16(); - if let Some(ready) = - readiness_state_from_payload(status_code, &body, Some(paths.port), Some(paths)) - { + if let Some(ready) = readiness_state_from_payload(status_code, &body, Some(paths.port), Some(paths)) { return if ready { - StatusRuntimeProbe::Ready(format!( - "Readiness endpoint reports ready at {base_url}/readiness." - )) + StatusRuntimeProbe::Ready(format!("Readiness endpoint reports ready at {base_url}/readiness.")) } else { - StatusRuntimeProbe::Starting(format!( - "Readiness endpoint reports startup in progress at {base_url}/readiness." - )) + StatusRuntimeProbe::Starting(format!("Readiness endpoint reports startup in progress at {base_url}/readiness.")) }; } if readiness_state_from_payload(status_code, &body, Some(paths.port), None).is_some() { @@ -332,32 +216,15 @@ async fn probe_status_runtime(paths: &auth::CortexPaths) -> StatusRuntimeProbe { paths.home.display() )); } - probe_errors.push(format!( - "readiness HTTP {status}: {}", - compact_status_detail(&body) - )); + probe_errors.push(format!("readiness HTTP {status}: {}", compact_status_detail(&body))); } Err(err) => probe_errors.push(format!("readiness failed: {err}")), } - - match transport::request_with_local_ipc_fallback( - &client, - "GET", - &base_url, - "/health", - paths, - &probe_headers, - None, - Duration::from_secs(2), - ) - .await - { + match transport::request_with_local_ipc_fallback(&client, "GET", &base_url, "/health", paths, &probe_headers, None, Duration::from_secs(2)).await { Ok((status, body)) => { let status_code = status.as_u16(); if is_cortex_health_payload(status_code, &body, Some(paths.port), Some(paths)) { - return StatusRuntimeProbe::Ready(format!( - "Health endpoint reports canonical Cortex runtime at {base_url}/health." - )); + return StatusRuntimeProbe::Ready(format!("Health endpoint reports canonical Cortex runtime at {base_url}/health.")); } if is_cortex_health_payload(status_code, &body, Some(paths.port), None) { return StatusRuntimeProbe::WrongIdentity(format!( @@ -366,10 +233,7 @@ async fn probe_status_runtime(paths: &auth::CortexPaths) -> StatusRuntimeProbe { paths.home.display() )); } - probe_errors.push(format!( - "health HTTP {status}: {}", - compact_status_detail(&body) - )); + probe_errors.push(format!("health HTTP {status}: {}", compact_status_detail(&body))); StatusRuntimeProbe::Error(probe_errors.join("; ")) } Err(err) => { @@ -378,54 +242,27 @@ async fn probe_status_runtime(paths: &auth::CortexPaths) -> StatusRuntimeProbe { } } } - pub(crate) async fn run_status_cli(paths: &auth::CortexPaths, json_output: bool) -> i32 { let runtime_probe = probe_status_runtime(paths).await; - let report = build_status_report( - paths, - runtime_probe, - paths.token.exists(), - paths.db.exists(), - ); - + let report = build_status_report(paths, runtime_probe, paths.token.exists(), paths.db.exists()); if json_output { println!("{}", serde_json::to_string_pretty(&report.payload).unwrap()); } else { print_status_human(&report.payload); } - report.exit_code } - fn print_status_human(payload: &Value) { println!("Cortex Memory Status"); println!("{}", "=".repeat(50)); println!("Status: {}", json_str(payload, "status")); println!("Summary: {}", json_str(payload, "summary")); if let Some(runtime) = payload.get("runtime").and_then(Value::as_object) { - println!( - "Runtime: {}", - runtime - .get("baseUrl") - .and_then(Value::as_str) - .unwrap_or("unknown") - ); - println!( - "Home: {}", - runtime - .get("home") - .and_then(Value::as_str) - .unwrap_or("unknown") - ); + println!("Runtime: {}", runtime.get("baseUrl").and_then(Value::as_str).unwrap_or("unknown")); + println!("Home: {}", runtime.get("home").and_then(Value::as_str).unwrap_or("unknown")); } if let Some(next_action) = payload.get("nextAction").and_then(Value::as_object) { - println!( - "Next action: {}", - next_action - .get("label") - .and_then(Value::as_str) - .unwrap_or("Run cortex status --json") - ); + println!("Next action: {}", next_action.get("label").and_then(Value::as_str).unwrap_or("Run cortex status --json")); if let Some(command) = next_action.get("command").and_then(Value::as_str) { println!("Command: {command}"); } @@ -434,10 +271,7 @@ fn print_status_human(payload: &Value) { println!("Checks:"); if let Some(checks) = payload.get("checks").and_then(Value::as_array) { for check in checks { - let status = check - .get("status") - .and_then(Value::as_str) - .unwrap_or("unknown"); + let status = check.get("status").and_then(Value::as_str).unwrap_or("unknown"); let marker = match status { "ok" => "[OK]", "warn" => "[!!]", @@ -457,4 +291,3 @@ fn print_status_human(payload: &Value) { println!(); println!("JSON: cortex status --json"); } - diff --git a/daemon-rs/src/cli/sync.rs b/daemon-rs/src/cli/sync.rs index 7fb19be1..b4091395 100644 --- a/daemon-rs/src/cli/sync.rs +++ b/daemon-rs/src/cli/sync.rs @@ -1,893 +1,46 @@ -// SPDX-License-Identifier: MIT - -use chrono::Utc; -use fs2::FileExt; -use serde_json::{json, Value}; -use std::collections::HashSet; -use std::io::Write as _; -use std::path::{Path, PathBuf}; -use std::time::Duration; - use crate::auth; -use crate::db; -use crate::crystallize; -use crate::export_data; - -use super::common::{ - open_cli_connection, parse_flag_value, parse_flag_usize, validate_cli_options, - validate_cli_options_or_exit, -}; -pub(crate) fn run_sync_cli(paths: &auth::CortexPaths, args: &[String]) { - let Some(command) = args.first().map(|value| value.as_str()) else { - eprintln!("Usage: cortex sync [options]"); - std::process::exit(1); - }; +use super::common::validate_cli_options_or_exit; - validate_sync_cli_options_or_exit(command, &args[1..]); - - let _sync_lock = match acquire_sync_lock(paths) { - Ok(lock) => lock, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; +const SYNC_USAGE: &str = "Usage: cortex sync [options]"; - match command { - "export" => run_sync_export_cli(paths, &args[1..]), - "import" => run_sync_import_cli(paths, &args[1..]), - "watch" => run_sync_watch_cli(paths, &args[1..]), - _ => { - eprintln!("Usage: cortex sync [options]"); - std::process::exit(1); - } - } +fn fail(message: &str) -> ! { + eprintln!("{message}"); + std::process::exit(1); } -fn validate_sync_cli_options_or_exit(command: &str, args: &[String]) { - let result = match command { - "export" => validate_cli_options(args, &["--out", "--since", "--cursor-file"], &[]), - "import" => validate_cli_options(args, &["--file", "--user", "--visibility"], &[]), - "watch" => validate_cli_options( - args, - &[ - "--dir", - "--interval-seconds", - "--user", - "--visibility", - "--since", - "--cursor-file", - ], - &["--once"], - ), - _ => Err("Usage: cortex sync [options]".to_string()), - }; - if let Err(err) = result { - eprintln!("{err}"); - std::process::exit(1); - } +fn validate_export_args(args: &[String]) { + validate_cli_options_or_exit(args, &["--format", "--out", "--since"], &[]); } -pub(crate) fn run_export_cli(paths: &auth::CortexPaths, args: &[String]) { - validate_cli_options_or_exit(args, &["--format", "--out"], &[]); - let mut format = "json".to_string(); - let mut out_path: Option = None; - - let mut i = 0usize; - while i < args.len() { - match args[i].as_str() { - "--format" => { - if let Some(v) = args.get(i + 1) { - format = v.to_string(); - i += 1; - } - } - "--out" => { - if let Some(v) = args.get(i + 1) { - out_path = Some(v.to_string()); - i += 1; - } - } - _ => {} - } - i += 1; - } - - let Some(export_format) = export_data::ExportFormat::parse(&format) else { - eprintln!("Usage: cortex export --format json|sql [--out ]"); - std::process::exit(1); - }; - - let mut conn = match open_cli_connection(&paths.db) { - Ok(conn) => conn, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - let output = match export_snapshot_text(&mut conn, export_format) { - Ok(output) => output, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - if let Some(path) = out_path { - if let Err(e) = write_atomic_text_file(Path::new(&path), &output) { - eprintln!("{e}"); - std::process::exit(1); - } - eprintln!("Exported to {path}"); - } else { - println!("{output}"); - } +fn validate_import_args(args: &[String]) { + validate_cli_options_or_exit(args, &["--file", "--user", "--visibility"], &["--dry-run"]); } -fn run_sync_export_cli(paths: &auth::CortexPaths, args: &[String]) { - validate_cli_options_or_exit(args, &["--out", "--since", "--cursor-file"], &[]); - let out_path = parse_flag_value(args, "--out"); - let since_override = parse_flag_value(args, "--since"); - if let Some(since) = since_override.as_deref() { - if chrono::DateTime::parse_from_rfc3339(since).is_err() { - eprintln!( - "Invalid --since value '{since}'. Use RFC3339 (for example 2026-04-19T00:00:00Z)." - ); - std::process::exit(1); +pub(crate) fn run_sync_cli(_paths: &auth::CortexPaths, args: &[String]) { + match args.first().map(String::as_str).unwrap_or("") { + "export" => { + validate_export_args(&args[1..]); + fail("sync export requires the full sync feature in this build"); } - } - - let cursor_file = parse_flag_value(args, "--cursor-file").map(PathBuf::from); - let since = resolve_sync_since(since_override.as_deref(), cursor_file.as_deref()); - let mut conn = match open_cli_connection(&paths.db) { - Ok(conn) => conn, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - let value = match export_changeset_snapshot_value(&mut conn, since.as_deref()) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - let output = serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()); - - if let Some(path) = out_path { - if let Err(e) = write_atomic_text_file(Path::new(&path), &output) { - eprintln!("{e}"); - std::process::exit(1); - } - eprintln!("Sync export written to {path}"); - } else { - println!("{output}"); - } - - if let Some(cursor_path) = cursor_file { - if let Some(cursor) = value.get("cursor").and_then(serde_json::Value::as_str) { - if let Err(err) = write_sync_cursor_file(&cursor_path, cursor) { - eprintln!("{err}"); - std::process::exit(1); - } + "import" => { + validate_import_args(&args[1..]); + fail("sync import requires the full sync feature in this build"); } - } -} - -pub(crate) fn run_import_cli(paths: &auth::CortexPaths, args: &[String]) { - let parsed = parse_import_cli_args( - args, - "Usage: cortex import --file [--user ] [--visibility private|team|shared]", - ); - let parsed = match parsed { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); + "watch" => { + validate_cli_options_or_exit(&args[1..], &["--dir", "--interval-secs", "--out", "--since", "--user", "--visibility"], &["--once", "--dry-run"]); + fail("sync watch requires the full sync feature in this build"); } - }; - let counts = match import_payload_from_file( - paths, - &parsed, - "import-cli", - ImportPayloadExpectation::GeneralJson, - ) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - println!( - "{{\"imported\":{{\"memories\":{},\"decisions\":{}}}}}", - counts.memories, counts.decisions - ); -} - -fn run_sync_import_cli(paths: &auth::CortexPaths, args: &[String]) { - let parsed = parse_import_cli_args( - args, - "Usage: cortex sync import --file [--user ] [--visibility private|team|shared]", - ); - let parsed = match parsed { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - let counts = match import_payload_from_file( - paths, - &parsed, - "sync-import-cli", - ImportPayloadExpectation::SyncChangeset, - ) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - println!( - "{{\"imported\":{{\"memories\":{},\"decisions\":{}}}}}", - counts.memories, counts.decisions - ); -} - -fn run_sync_watch_cli(paths: &auth::CortexPaths, args: &[String]) { - validate_cli_options_or_exit( - args, - &[ - "--dir", - "--interval-seconds", - "--user", - "--visibility", - "--since", - "--cursor-file", - ], - &["--once"], - ); - let Some(dir_raw) = parse_flag_value(args, "--dir") else { - eprintln!( - "Usage: cortex sync watch --dir [--interval-seconds ] [--once] [--user ] [--visibility private|team|shared] [--since ] [--cursor-file ]" - ); - std::process::exit(1); - }; - let watch_dir = PathBuf::from(dir_raw); - if !watch_dir.exists() { - if let Err(e) = std::fs::create_dir_all(&watch_dir) { - eprintln!( - "Failed to create sync watch directory {}: {e}", - watch_dir.display() - ); - std::process::exit(1); - } - } - if !watch_dir.is_dir() { - eprintln!( - "Sync watch path must be a directory: {}", - watch_dir.display() - ); - std::process::exit(1); - } - - let interval_seconds = match parse_flag_usize(args, "--interval-seconds") { - Ok(Some(value)) => value as u64, - Ok(None) => 15, - Err(err) => { - eprintln!("Invalid --interval-seconds: {err}"); - std::process::exit(1); - } - }; - let once = args.iter().any(|arg| arg == "--once"); - let username = parse_flag_value(args, "--user"); - let visibility = - parse_flag_value(args, "--visibility").unwrap_or_else(|| "private".to_string()); - if !matches!(visibility.as_str(), "private" | "team" | "shared") { - eprintln!("Invalid --visibility value '{visibility}'. Use private|team|shared."); - std::process::exit(1); - } - - let mut bootstrap_since = parse_flag_value(args, "--since"); - if let Some(since) = bootstrap_since.as_deref() { - if chrono::DateTime::parse_from_rfc3339(since).is_err() { - eprintln!( - "Invalid --since value '{since}'. Use RFC3339 (for example 2026-04-19T00:00:00Z)." - ); - std::process::exit(1); - } - } - - let state_id = sync_watch_state_id(&watch_dir); - let state_root = paths.home.join("runtime").join("sync-watch"); - let seen_file = state_root.join(format!("{state_id}.seen")); - let default_cursor = state_root.join(format!("{state_id}.cursor")); - let cursor_file = parse_flag_value(args, "--cursor-file") - .map(PathBuf::from) - .unwrap_or(default_cursor); - - let local_site_id = match ensure_sync_site_id(paths) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - let mut seen = match load_sync_seen_set(&seen_file) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - loop { - let candidates = match collect_sync_watch_import_candidates(&watch_dir, &local_site_id) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - - let mut seen_dirty = false; - for candidate in candidates { - let Some(name) = candidate.file_name().and_then(|value| value.to_str()) else { - continue; - }; - if seen.contains(name) { - continue; - } - let import_options = ImportCliArgs { - file_path: candidate.clone(), - username: username.clone(), - visibility: visibility.clone(), - }; - match import_payload_from_file( - paths, - &import_options, - "sync-watch-import", - ImportPayloadExpectation::SyncChangeset, - ) { - Ok(counts) => { - eprintln!( - "[sync watch] imported {} (memories={}, decisions={})", - candidate.display(), - counts.memories, - counts.decisions - ); - seen.insert(name.to_string()); - seen_dirty = true; - } - Err(err) => { - eprintln!( - "[sync watch] import skipped for {}: {}", - candidate.display(), - err - ); - } - } - } - if seen_dirty { - if let Err(err) = write_sync_seen_set(&seen_file, &seen) { - eprintln!("{err}"); - std::process::exit(1); - } - } - - let since = read_sync_cursor_file(&cursor_file).or_else(|| bootstrap_since.take()); - let mut conn = match open_cli_connection(&paths.db) { - Ok(conn) => conn, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - let changeset = match export_changeset_snapshot_value(&mut conn, since.as_deref()) { - Ok(value) => value, - Err(err) => { - eprintln!("{err}"); - std::process::exit(1); - } - }; - let memories_count = changeset - .get("memories_count") - .and_then(serde_json::Value::as_u64) - .unwrap_or(0); - let decisions_count = changeset - .get("decisions_count") - .and_then(serde_json::Value::as_u64) - .unwrap_or(0); - let total = memories_count + decisions_count; - if total > 0 { - let filename = format!( - "changeset-{}-{}.json", - local_site_id, - chrono::Utc::now().format("%Y%m%dT%H%M%S%3fZ") - ); - let out_path = watch_dir.join(filename); - let output = - serde_json::to_string_pretty(&changeset).unwrap_or_else(|_| "{}".to_string()); - if let Err(err) = write_atomic_text_file(&out_path, &output) { - eprintln!("{err}"); - std::process::exit(1); - } - eprintln!( - "[sync watch] exported {} (memories={}, decisions={})", - out_path.display(), - memories_count, - decisions_count - ); - } - if let Some(cursor) = changeset.get("cursor").and_then(serde_json::Value::as_str) { - if let Err(err) = write_sync_cursor_file(&cursor_file, cursor) { - eprintln!("{err}"); - std::process::exit(1); - } - } - - if once { - break; - } - std::thread::sleep(Duration::from_secs(interval_seconds.max(1))); - } -} - -#[derive(Debug, Clone)] -struct ImportCliArgs { - file_path: PathBuf, - username: Option, - visibility: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ImportPayloadExpectation { - GeneralJson, - SyncChangeset, -} - -fn parse_import_cli_args(args: &[String], usage: &str) -> Result { - validate_cli_options(args, &["--file", "--user", "--visibility"], &[])?; - - let mut file_path: Option = None; - let mut username: Option = None; - let mut visibility = "private".to_string(); - - let mut i = 0usize; - while i < args.len() { - match args[i].as_str() { - "--file" => { - if let Some(v) = args.get(i + 1) { - file_path = Some(v.to_string()); - i += 1; - } - } - "--user" => { - if let Some(v) = args.get(i + 1) { - username = Some(v.to_string()); - i += 1; - } - } - "--visibility" => { - if let Some(v) = args.get(i + 1) { - visibility = v.to_string(); - i += 1; - } - } - _ => {} - } - i += 1; - } - - let Some(file_path) = file_path else { - return Err(usage.to_string()); - }; - if !matches!(visibility.as_str(), "private" | "team" | "shared") { - return Err(format!( - "Invalid --visibility value '{visibility}'. Use private|team|shared." - )); - } - - Ok(ImportCliArgs { - file_path: PathBuf::from(file_path), - username, - visibility, - }) -} - -fn import_payload_from_file( - paths: &auth::CortexPaths, - parsed: &ImportCliArgs, - source_agent_fallback: &str, - expectation: ImportPayloadExpectation, -) -> Result { - let file_display = parsed.file_path.display().to_string(); - let raw = std::fs::read_to_string(&parsed.file_path) - .map_err(|e| format!("Cannot read import file {file_display}: {e}"))?; - let raw_value: Value = - serde_json::from_str(&raw).map_err(|e| format!("Import file is not valid JSON: {e}"))?; - validate_import_payload_metadata(&raw_value, expectation)?; - let payload: export_data::ImportPayload = serde_json::from_value(raw_value) - .map_err(|e| format!("Import file has unsupported record shape: {e}"))?; - - let mut conn = open_cli_connection(&paths.db)?; - let team_mode = db::current_mode(&conn) == "team"; - if parsed.username.is_some() && !team_mode { - return Err("--user import requires team mode. Run: cortex setup --team".to_string()); - } - - let owner_id = if team_mode { - if let Some(user) = parsed.username.as_ref() { - match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - rusqlite::params![user.clone()], - |row| row.get::<_, i64>(0), - ) { - Ok(id) => Some(id), - Err(_) => { - return Err(format!( - "Unknown user '{user}'. Create the user before import." - )); - } - } - } else { - conn.query_row( - "SELECT value FROM config WHERE key = 'owner_user_id' LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) - .ok() - .and_then(|v| v.parse::().ok()) - .or_else(|| { - conn.query_row( - "SELECT id FROM users ORDER BY CASE role WHEN 'owner' THEN 0 ELSE 1 END, id ASC LIMIT 1", - [], - |row| row.get::<_, i64>(0), - ) - .ok() - }) - } - } else { - None - }; - if team_mode && owner_id.is_none() { - return Err( - "Team mode import requires a target owner. Run `cortex setup --team` first." - .to_string(), - ); - } - - let options = export_data::ImportOptions { - owner_id, - visibility: if team_mode { - Some(parsed.visibility.clone()) - } else { - None - }, - source_agent_fallback: source_agent_fallback.to_string(), - }; - export_data::import_payload(&mut conn, &payload, &options) -} - -fn validate_import_payload_metadata( - value: &Value, - expectation: ImportPayloadExpectation, -) -> Result<(), String> { - let Some(obj) = value.as_object() else { - return Err("Import file must be a JSON object.".to_string()); - }; - - let mode = obj.get("mode").and_then(Value::as_str); - match obj.get("version") { - Some(version) if version.as_u64() == Some(1) => {} - Some(version) => { - return Err(format!( - "Import file has unsupported version marker {version}; expected 1." - )); - } - None if expectation == ImportPayloadExpectation::SyncChangeset || mode.is_some() => { - return Err("Import file is missing required version marker.".to_string()); - } - None => {} - } - - match expectation { - ImportPayloadExpectation::GeneralJson => match mode { - Some("changeset") | None => {} - Some("page") => { - return Err( - "Import file is a paged export fragment; import a full export or sync changeset." - .to_string(), - ); - } - Some(other) => return Err(format!("Import file has unsupported mode '{other}'.")), - }, - ImportPayloadExpectation::SyncChangeset => { - if mode != Some("changeset") { - return Err( - "Sync import requires a changeset export with mode=\"changeset\".".to_string(), - ); - } - let Some(cursor) = obj.get("cursor").and_then(Value::as_str) else { - return Err("Sync changeset is missing cursor version marker.".to_string()); - }; - validate_rfc3339_marker("cursor", cursor)?; - } - } - - if let Some(exported_at) = obj.get("exported_at").and_then(Value::as_str) { - validate_rfc3339_marker("exported_at", exported_at)?; - } - let count_markers_required = expectation == ImportPayloadExpectation::SyncChangeset; - validate_import_count_marker(value, "memories", "memories_count", count_markers_required)?; - validate_import_count_marker( - value, - "decisions", - "decisions_count", - count_markers_required, - )?; - Ok(()) -} - -fn validate_rfc3339_marker(label: &str, value: &str) -> Result<(), String> { - if chrono::DateTime::parse_from_rfc3339(value).is_ok() { - Ok(()) - } else { - Err(format!( - "Import file has invalid {label} marker '{value}'; expected RFC3339." - )) - } -} - -fn validate_import_count_marker( - value: &Value, - rows_key: &str, - count_key: &str, - required: bool, -) -> Result<(), String> { - let Some(expected_value) = value.get(count_key) else { - if required { - return Err(format!( - "Sync changeset is missing required {count_key} marker." - )); - } - return Ok(()); - }; - let Some(expected_u64) = expected_value.as_u64() else { - return Err(format!("Import file has non-numeric {count_key} marker.")); - }; - let expected = usize::try_from(expected_u64) - .map_err(|_| format!("Import file has out-of-range {count_key} marker."))?; - let actual = value - .get(rows_key) - .and_then(Value::as_array) - .map(|rows| rows.len()) - .unwrap_or(0); - if actual == expected { - Ok(()) - } else { - Err(format!( - "Import file {count_key} marker ({expected}) does not match {rows_key} rows ({actual})." - )) - } -} - -fn resolve_sync_since(override_since: Option<&str>, cursor_file: Option<&Path>) -> Option { - override_since - .map(str::to_string) - .or_else(|| cursor_file.and_then(read_sync_cursor_file)) -} - -fn read_sync_cursor_file(path: &Path) -> Option { - std::fs::read_to_string(path).ok().and_then(|raw| { - let trimmed = raw.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_string()) - } - }) -} - -fn write_sync_cursor_file(path: &Path, cursor: &str) -> Result<(), String> { - write_atomic_text_file(path, &format!("{cursor}\n")) - .map_err(|e| format!("Failed to write cursor file {}: {e}", path.display())) -} - -fn ensure_sync_site_id(paths: &auth::CortexPaths) -> Result { - let site_id_path = paths.home.join("site_id"); - if let Ok(existing) = std::fs::read_to_string(&site_id_path) { - let candidate = sanitize_sync_site_id(existing.trim()); - if !candidate.is_empty() { - return Ok(candidate); - } - } - - if let Some(parent) = site_id_path.parent() { - std::fs::create_dir_all(parent).map_err(|e| { - format!( - "Failed to create sync site-id directory {}: {e}", - parent.display() - ) - })?; - } - let created = uuid::Uuid::new_v4().to_string(); - write_atomic_text_file(&site_id_path, &format!("{created}\n")).map_err(|e| { - format!( - "Failed to persist sync site-id {}: {e}", - site_id_path.display() - ) - })?; - Ok(created) -} - -fn sanitize_sync_site_id(raw: &str) -> String { - raw.chars() - .filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) - .collect() -} - -fn sync_watch_state_id(watch_dir: &Path) -> String { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - watch_dir.to_string_lossy().to_string().hash(&mut hasher); - format!("{:016x}", hasher.finish()) -} - -fn is_sync_changeset_file_name(name: &str) -> bool { - name.starts_with("changeset-") && name.ends_with(".json") -} - -fn collect_sync_watch_import_candidates( - watch_dir: &Path, - local_site_id: &str, -) -> Result, String> { - let mut files = Vec::new(); - let local_prefix = format!("changeset-{local_site_id}-"); - let entries = std::fs::read_dir(watch_dir).map_err(|e| { - format!( - "Failed to read sync watch directory {}: {e}", - watch_dir.display() - ) - })?; - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_file() { - continue; - } - let Some(name) = path.file_name().and_then(|value| value.to_str()) else { - continue; - }; - if !is_sync_changeset_file_name(name) { - continue; - } - if name.starts_with(&local_prefix) { - continue; - } - files.push(path); - } - files.sort(); - Ok(files) -} - -fn load_sync_seen_set(path: &Path) -> Result, String> { - let mut seen = HashSet::new(); - if !path.exists() { - return Ok(seen); - } - let raw = std::fs::read_to_string(path) - .map_err(|e| format!("Failed to read sync watch state {}: {e}", path.display()))?; - for line in raw.lines() { - let trimmed = line.trim(); - if !trimmed.is_empty() { - seen.insert(trimmed.to_string()); - } - } - Ok(seen) -} - -fn write_sync_seen_set(path: &Path, seen: &HashSet) -> Result<(), String> { - let mut rows: Vec<&str> = seen.iter().map(String::as_str).collect(); - rows.sort_unstable(); - write_atomic_text_file(path, &rows.join("\n")) - .map_err(|e| format!("Failed to write sync watch state {}: {e}", path.display())) -} - -fn acquire_sync_lock(paths: &auth::CortexPaths) -> Result { - let lock_path = paths.home.join("sync.lock"); - std::fs::create_dir_all(&paths.home).map_err(|e| { - format!( - "Failed to create sync lock directory {}: {e}", - paths.home.display() - ) - })?; - let lock_file = std::fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(&lock_path) - .map_err(|e| format!("Failed to open sync lock {}: {e}", lock_path.display()))?; - lock_file - .lock_exclusive() - .map_err(|e| format!("Failed to acquire sync lock {}: {e}", lock_path.display()))?; - Ok(lock_file) -} - -fn export_snapshot_text( - conn: &mut rusqlite::Connection, - format: export_data::ExportFormat, -) -> Result { - let tx = conn - .transaction() - .map_err(|e| format!("Failed to start export snapshot transaction: {e}"))?; - let output = match format { - export_data::ExportFormat::Json => { - let value = export_data::export_json_value(&tx); - serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()) - } - export_data::ExportFormat::Sql => export_data::export_sql_text(&tx), - }; - tx.commit() - .map_err(|e| format!("Failed to finish export snapshot transaction: {e}"))?; - Ok(output) -} - -fn export_changeset_snapshot_value( - conn: &mut rusqlite::Connection, - since: Option<&str>, -) -> Result { - let tx = conn - .transaction() - .map_err(|e| format!("Failed to start sync export snapshot transaction: {e}"))?; - let value = export_data::export_json_changeset_value(&tx, since); - tx.commit() - .map_err(|e| format!("Failed to finish sync export snapshot transaction: {e}"))?; - Ok(value) -} - -fn write_atomic_text_file(path: &Path, contents: &str) -> Result<(), String> { - let parent = writable_parent_dir(path)?; - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create {}: {e}", parent.display()))?; - - let mut tmp = tempfile::NamedTempFile::new_in(parent) - .map_err(|e| format!("Failed to create temp file in {}: {e}", parent.display()))?; - tmp.write_all(contents.as_bytes()) - .map_err(|e| format!("Failed to write temp file for {}: {e}", path.display()))?; - tmp.as_file() - .sync_all() - .map_err(|e| format!("Failed to flush temp file for {}: {e}", path.display()))?; - tmp.persist(path).map_err(|e| { - format!( - "Failed to replace {} atomically: {}", - path.display(), - e.error - ) - })?; - sync_parent_dir(parent) - .map_err(|e| format!("Failed to flush directory {}: {e}", parent.display()))?; - Ok(()) -} - -fn writable_parent_dir(path: &Path) -> Result<&Path, String> { - match path.parent() { - Some(parent) if !parent.as_os_str().is_empty() => Ok(parent), - _ => Ok(Path::new(".")), + _ => fail(SYNC_USAGE), } } -#[cfg(unix)] -fn sync_parent_dir(parent: &Path) -> std::io::Result<()> { - std::fs::File::open(parent)?.sync_all() +pub(crate) fn run_export_cli(_paths: &auth::CortexPaths, args: &[String]) { + validate_export_args(args); + fail("export requires the full sync feature in this build"); } -#[cfg(not(unix))] -fn sync_parent_dir(_parent: &Path) -> std::io::Result<()> { - Ok(()) +pub(crate) fn run_import_cli(_paths: &auth::CortexPaths, args: &[String]) { + validate_import_args(args); + fail("import requires the full sync feature in this build"); } diff --git a/daemon-rs/src/cli/tests/common.rs b/daemon-rs/src/cli/tests/common.rs index 508aa183..3215959d 100644 --- a/daemon-rs/src/cli/tests/common.rs +++ b/daemon-rs/src/cli/tests/common.rs @@ -1,6 +1,4 @@ // SPDX-License-Identifier: MIT -//! CLI flag parsing and remote-target auth boundaries only. - #[cfg(test)] mod tests { use crate::cli::common::{resolve_client_target_inputs, validate_cli_options}; @@ -8,82 +6,49 @@ mod tests { use crate::cli::*; use crate::*; use std::fs; - #[test] fn parse_flag_usize_validates_and_parses_values() { - let args = vec![ - "--agent".to_string(), - "codex".to_string(), - "--budget".to_string(), - "900".to_string(), - ]; + let args = vec!["--agent".to_string(), "codex".to_string(), "--budget".to_string(), "900".to_string()]; assert_eq!(parse_flag_usize(&args, "--budget").unwrap(), Some(900)); - let missing_value = vec!["--budget".to_string()]; - assert!(parse_flag_usize(&missing_value, "--budget") - .unwrap_err() - .contains("missing value")); + assert!(parse_flag_usize(&missing_value, "--budget").unwrap_err().contains("missing value")); } - #[test] fn validate_cli_options_rejects_unknown_options() { - let args = vec![ - "--out".to_string(), - "dump.json".to_string(), - "--bogus".to_string(), - ]; + let args = vec!["--out".to_string(), "dump.json".to_string(), "--bogus".to_string()]; let err = validate_cli_options(&args, &["--out"], &[]).expect_err("unknown option"); assert_eq!(err, "Unknown option: --bogus"); } - #[test] fn resolve_client_target_inputs_prefers_cli_over_env_values() { - let (base_url, api_key, local_owner_mode) = resolve_client_target_inputs( - Some("https://cli.example"), - Some("ctx_cli"), - Some("https://env.example"), - Some("ctx_env"), - "http://127.0.0.1:7437", - ); + let (base_url, api_key, local_owner_mode) = + resolve_client_target_inputs(Some("https://cli.example"), Some("ctx_cli"), Some("https://env.example"), Some("ctx_env"), "http://127.0.0.1:7437"); assert_eq!(base_url, "https://cli.example"); assert_eq!(api_key.as_deref(), Some("ctx_cli")); assert!(!local_owner_mode); } - #[test] fn remote_target_without_api_key_is_rejected() { let home_dir = temp_test_dir("remote_target_auth_required"); fs::create_dir_all(&home_dir).unwrap(); let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - - let err = - ensure_remote_target_has_api_key("https://100.64.0.12:7437", None, &paths).unwrap_err(); + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); + let err = ensure_remote_target_has_api_key("https://100.64.0.12:7437", None, &paths).unwrap_err(); assert!(err.contains("requires an API key")); - let _ = fs::remove_dir_all(&home_dir); } - #[test] fn local_target_without_api_key_is_allowed() { let home_dir = temp_test_dir("local_target_no_key"); fs::create_dir_all(&home_dir).unwrap(); let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); assert!(ensure_remote_target_has_api_key("http://127.0.0.1:7437", None, &paths).is_ok()); - let _ = fs::remove_dir_all(&home_dir); } - #[test] fn openapi_spec_version_matches_cargo_pkg_version() { let spec = fs::read_to_string(openapi_spec_path()).expect("read OpenAPI spec"); - assert!( - spec.contains(&format!("version: {}", env!("CARGO_PKG_VERSION"))), - "OpenAPI version must match Cargo package version" - ); + assert!(spec.contains(&format!("version: {}", env!("CARGO_PKG_VERSION"))), "OpenAPI version must match Cargo package version"); } } diff --git a/daemon-rs/src/cli/tests/locks.rs b/daemon-rs/src/cli/tests/locks.rs index 6094fa76..931ddc36 100644 --- a/daemon-rs/src/cli/tests/locks.rs +++ b/daemon-rs/src/cli/tests/locks.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT #[cfg(test)] mod tests { - use crate::cli::daemon::*; use crate::cli::cleanup::run_stale_pid_cleanup; + use crate::cli::daemon::*; use crate::cli::tests::support::*; use crate::cli::*; use crate::*; @@ -10,7 +10,6 @@ mod tests { use std::process::{Command, Stdio}; use std::sync::Arc; use std::time::{Duration, Instant}; - #[test] fn acquire_runtime_lock_rejects_duplicate_serve_startup() { let _env_guard = env_guard(); @@ -20,36 +19,24 @@ mod tests { fs::create_dir_all(&global_lock_home).unwrap(); let global_lock_home_str = global_lock_home.to_string_lossy().to_string(); let _global_lock_home = ScopedEnvVar::set("CORTEX_GLOBAL_LOCK_HOME", &global_lock_home_str); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); let first_lock = acquire_runtime_lock(&paths).unwrap(); let err = acquire_runtime_lock(&paths).unwrap_err(); - assert!(err.contains("another cortex instance")); - drop(first_lock); let _ = fs::remove_dir_all(&home_dir); let _ = fs::remove_dir_all(&global_lock_home); } - #[test] fn control_center_lock_detection_reports_cross_process_holder() { let _env_guard = env_guard(); let home_dir = temp_test_dir("control_center_lock_detection"); std::fs::create_dir_all(&home_dir).unwrap(); let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); let ready_file = home_dir.join("control-center-lock-ready"); - - assert!( - !control_center_is_active(&paths).expect("probe lock without holder"), - "lock should not appear active before holder starts" - ); - + assert!(!control_center_is_active(&paths).expect("probe lock without holder"), "lock should not appear active before holder starts"); let current_exe = std::env::current_exe().expect("resolve current test binary path"); let mut child = Command::new(current_exe) .arg("--exact") @@ -57,16 +44,12 @@ mod tests { .arg("--nocapture") .env(CONTROL_CENTER_LOCK_TEST_CHILD_ENV, "1") .env(CONTROL_CENTER_LOCK_TEST_HOME_ENV, &home_str) - .env( - CONTROL_CENTER_LOCK_TEST_READY_ENV, - ready_file.to_string_lossy().to_string(), - ) + .env(CONTROL_CENTER_LOCK_TEST_READY_ENV, ready_file.to_string_lossy().to_string()) .env(CONTROL_CENTER_LOCK_TEST_HOLD_MS_ENV, "30000") .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .expect("spawn lock-holder child"); - let ready_deadline = Instant::now() + Duration::from_secs(5); while !ready_file.exists() { if Instant::now() >= ready_deadline { @@ -76,26 +59,12 @@ mod tests { } std::thread::sleep(Duration::from_millis(25)); } - - assert!( - wait_for_control_center_lock(&paths, Duration::from_secs(3)), - "lock should appear active while child holds cross-process lock" - ); - + assert!(wait_for_control_center_lock(&paths, Duration::from_secs(3)), "lock should appear active while child holds cross-process lock"); let status = child.wait().expect("wait lock-holder child"); - assert!( - status.success(), - "lock-holder child should exit successfully" - ); - - assert!( - !control_center_is_active(&paths).expect("probe lock after holder exits"), - "lock should be released after child exits" - ); - + assert!(status.success(), "lock-holder child should exit successfully"); + assert!(!control_center_is_active(&paths).expect("probe lock after holder exits"), "lock should be released after child exits"); let _ = std::fs::remove_dir_all(&home_dir); } - #[test] fn acquire_runtime_lock_waits_for_handoff_when_enabled() { let _env_guard = env_guard(); @@ -105,28 +74,21 @@ mod tests { fs::create_dir_all(&global_lock_home).unwrap(); let global_lock_home_str = global_lock_home.to_string_lossy().to_string(); let _global_lock_home = ScopedEnvVar::set("CORTEX_GLOBAL_LOCK_HOME", &global_lock_home_str); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); let first_lock = acquire_runtime_lock(&paths).unwrap(); let _wait_lock_flag = ScopedEnvVar::set("CORTEX_WAIT_FOR_DAEMON_LOCK", "1"); let _wait_secs_flag = ScopedEnvVar::set("CORTEX_DAEMON_LOCK_WAIT_SECS", "1"); - let releaser = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(300)); drop(first_lock); }); - let second_lock = acquire_runtime_lock(&paths).expect("lock handoff should succeed"); drop(second_lock); releaser.join().unwrap(); - let _ = fs::remove_dir_all(&home_dir); let _ = fs::remove_dir_all(&global_lock_home); } - #[test] fn acquire_runtime_lock_rejects_concurrent_startup_burst() { let _env_guard = env_guard(); @@ -136,15 +98,8 @@ mod tests { fs::create_dir_all(&global_lock_home).unwrap(); let global_lock_home_str = global_lock_home.to_string_lossy().to_string(); let _global_lock_home = ScopedEnvVar::set("CORTEX_GLOBAL_LOCK_HOME", &global_lock_home_str); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = Arc::new(auth::CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - None, - )); - + let paths = Arc::new(auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None)); let first_lock = acquire_runtime_lock(&paths).expect("first runtime lock must succeed"); let workers: Vec<_> = (0..12) .map(|_| { @@ -152,56 +107,32 @@ mod tests { std::thread::spawn(move || acquire_runtime_lock(&worker_paths).is_err()) }) .collect(); - let failures = workers - .into_iter() - .map(|worker| worker.join().expect("join worker")) - .filter(|failed| *failed) - .count(); - assert_eq!( - failures, 12, - "all concurrent startups should fail while runtime lock is held" - ); - + let failures = workers.into_iter().map(|worker| worker.join().expect("join worker")).filter(|failed| *failed).count(); + assert_eq!(failures, 12, "all concurrent startups should fail while runtime lock is held"); drop(first_lock); - let second_lock = - acquire_runtime_lock(&paths).expect("lock should be reacquired after release"); + let second_lock = acquire_runtime_lock(&paths).expect("lock should be reacquired after release"); drop(second_lock); - let _ = fs::remove_dir_all(&home_dir); let _ = fs::remove_dir_all(&global_lock_home); } - #[test] fn run_stale_pid_cleanup_keeps_lock_file() { let home_dir = temp_test_dir("stale_pid_cleanup"); fs::create_dir_all(&home_dir).unwrap(); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); fs::write(&paths.pid, "999999").unwrap(); fs::write(&paths.lock, "lock-held").unwrap(); - let dry_run = run_stale_pid_cleanup(&paths, true); - assert_eq!( - dry_run, - vec!["DELETE cortex.pid (process 999999 not running)"] - ); + assert_eq!(dry_run, vec!["DELETE cortex.pid (process 999999 not running)"]); assert!(paths.pid.exists()); assert!(paths.lock.exists()); - let apply = run_stale_pid_cleanup(&paths, false); - assert_eq!( - apply, - vec!["DELETE cortex.pid (process 999999 not running)"] - ); + assert_eq!(apply, vec!["DELETE cortex.pid (process 999999 not running)"]); assert!(!paths.pid.exists()); assert!(paths.lock.exists()); - let _ = fs::remove_dir_all(&home_dir); } - #[test] fn spawned_owner_requires_parent_pid_only_for_non_control_center_owner() { assert!(spawned_owner_requires_parent_pid(Some("cli-mcp"))); @@ -209,7 +140,6 @@ mod tests { assert!(!spawned_owner_requires_parent_pid(Some("control-center"))); assert!(!spawned_owner_requires_parent_pid(None)); } - #[test] fn is_control_center_owner_is_case_insensitive() { assert!(is_control_center_owner(Some("control-center"))); @@ -217,34 +147,18 @@ mod tests { assert!(!is_control_center_owner(Some("plugin-claude"))); assert!(!is_control_center_owner(None)); } - #[test] fn app_managed_startup_heavy_delay_only_applies_to_control_center_owner() { let _env_guard = env_guard(); std::env::remove_var(APP_MANAGED_STARTUP_DELAY_ENV); - assert_eq!( - app_managed_startup_heavy_delay(Some("control-center")), - Duration::from_secs(APP_MANAGED_STARTUP_HEAVY_DELAY_SECS) - ); - assert_eq!( - app_managed_startup_heavy_delay(Some("plugin-claude")), - Duration::from_secs(0) - ); - + assert_eq!(app_managed_startup_heavy_delay(Some("control-center")), Duration::from_secs(APP_MANAGED_STARTUP_HEAVY_DELAY_SECS)); + assert_eq!(app_managed_startup_heavy_delay(Some("plugin-claude")), Duration::from_secs(0)); let _startup_delay = ScopedEnvVar::set(APP_MANAGED_STARTUP_DELAY_ENV, "0"); - assert_eq!( - app_managed_startup_heavy_delay(Some("control-center")), - Duration::from_secs(0) - ); - + assert_eq!(app_managed_startup_heavy_delay(Some("control-center")), Duration::from_secs(0)); drop(_startup_delay); let _excessive_delay = ScopedEnvVar::set(APP_MANAGED_STARTUP_DELAY_ENV, "777"); - assert_eq!( - app_managed_startup_heavy_delay(Some("control-center")), - Duration::from_secs(APP_MANAGED_STARTUP_HEAVY_DELAY_MAX_SECS) - ); + assert_eq!(app_managed_startup_heavy_delay(Some("control-center")), Duration::from_secs(APP_MANAGED_STARTUP_HEAVY_DELAY_MAX_SECS)); } - #[test] fn startup_schedule_uses_non_app_defaults_for_plugin_owner() { let _env_guard = env_guard(); @@ -254,30 +168,13 @@ mod tests { let _embed_delay = ScopedEnvVar::set(STARTUP_EMBED_DELAY_ENV, ""); let _crystallize_delay = ScopedEnvVar::set(STARTUP_CRYSTALLIZE_DELAY_ENV, ""); let _storage_delay = ScopedEnvVar::set(STARTUP_STORAGE_GOVERNOR_DELAY_ENV, ""); - let schedule = startup_schedule(Some("plugin-claude")); - assert_eq!( - schedule.index, - Duration::from_secs(DEFAULT_STARTUP_INDEX_DELAY_SECS) - ); - assert_eq!( - schedule.aging, - Duration::from_secs(DEFAULT_STARTUP_AGING_DELAY_SECS) - ); - assert_eq!( - schedule.embed, - Duration::from_secs(DEFAULT_STARTUP_EMBED_DELAY_SECS) - ); - assert_eq!( - schedule.crystallize, - Duration::from_secs(DEFAULT_STARTUP_CRYSTALLIZE_DELAY_SECS) - ); - assert_eq!( - schedule.storage_governor_initial, - Duration::from_secs(DEFAULT_STARTUP_STORAGE_GOVERNOR_DELAY_SECS) - ); + assert_eq!(schedule.index, Duration::from_secs(DEFAULT_STARTUP_INDEX_DELAY_SECS)); + assert_eq!(schedule.aging, Duration::from_secs(DEFAULT_STARTUP_AGING_DELAY_SECS)); + assert_eq!(schedule.embed, Duration::from_secs(DEFAULT_STARTUP_EMBED_DELAY_SECS)); + assert_eq!(schedule.crystallize, Duration::from_secs(DEFAULT_STARTUP_CRYSTALLIZE_DELAY_SECS)); + assert_eq!(schedule.storage_governor_initial, Duration::from_secs(DEFAULT_STARTUP_STORAGE_GOVERNOR_DELAY_SECS)); } - #[test] fn startup_schedule_applies_app_managed_offsets_for_control_center() { let _env_guard = env_guard(); @@ -287,24 +184,13 @@ mod tests { let _embed_delay = ScopedEnvVar::set(STARTUP_EMBED_DELAY_ENV, "1"); let _crystallize_delay = ScopedEnvVar::set(STARTUP_CRYSTALLIZE_DELAY_ENV, "1"); let _storage_delay = ScopedEnvVar::set(STARTUP_STORAGE_GOVERNOR_DELAY_ENV, "7"); - let schedule = startup_schedule(Some("control-center")); assert_eq!(schedule.index, Duration::from_secs(10)); - assert_eq!( - schedule.aging, - Duration::from_secs(10 + APP_MANAGED_AGING_STARTUP_OFFSET_SECS) - ); - assert_eq!( - schedule.embed, - Duration::from_secs(10 + APP_MANAGED_EMBED_STARTUP_OFFSET_SECS) - ); - assert_eq!( - schedule.crystallize, - Duration::from_secs(10 + APP_MANAGED_CRYSTALLIZE_STARTUP_OFFSET_SECS) - ); + assert_eq!(schedule.aging, Duration::from_secs(10 + APP_MANAGED_AGING_STARTUP_OFFSET_SECS)); + assert_eq!(schedule.embed, Duration::from_secs(10 + APP_MANAGED_EMBED_STARTUP_OFFSET_SECS)); + assert_eq!(schedule.crystallize, Duration::from_secs(10 + APP_MANAGED_CRYSTALLIZE_STARTUP_OFFSET_SECS)); assert_eq!(schedule.storage_governor_initial, Duration::from_secs(7)); } - #[test] fn control_center_lock_holder_child_process() { crate::cli::tests::support::control_center_lock_holder_child_process(); diff --git a/daemon-rs/src/cli/tests/mod.rs b/daemon-rs/src/cli/tests/mod.rs index cdabacc5..e8a3b953 100644 --- a/daemon-rs/src/cli/tests/mod.rs +++ b/daemon-rs/src/cli/tests/mod.rs @@ -1,10 +1,9 @@ // SPDX-License-Identifier: MIT #[cfg(test)] -mod support; - +mod common; #[cfg(test)] mod locks; #[cfg(test)] mod spawn; #[cfg(test)] -mod common; +mod support; diff --git a/daemon-rs/src/cli/tests/spawn.rs b/daemon-rs/src/cli/tests/spawn.rs index cdc7c0b8..99744bb9 100644 --- a/daemon-rs/src/cli/tests/spawn.rs +++ b/daemon-rs/src/cli/tests/spawn.rs @@ -1,55 +1,38 @@ // SPDX-License-Identifier: MIT -//! Spawn ownership policy boundaries only. - #[cfg(test)] mod tests { - use crate::cli::daemon::{ - background_db_lock_max_wait, validate_spawned_owner_runtime_claim, - BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, SPAWN_PARENT_PID_ENV, - }; + use crate::cli::daemon::{background_db_lock_max_wait, validate_spawned_owner_runtime_claim, BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, SPAWN_PARENT_PID_ENV}; use crate::cli::tests::support::*; use crate::cli::*; use crate::*; use std::time::Duration; - #[test] fn background_db_lock_wait_env_is_clamped() { let _env_guard = env_guard(); let _small = ScopedEnvVar::set(BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, "1"); assert_eq!(background_db_lock_max_wait(), Duration::from_millis(100)); drop(_small); - let _large = ScopedEnvVar::set(BACKGROUND_DB_LOCK_MAX_WAIT_MS_ENV, "70000"); assert_eq!(background_db_lock_max_wait(), Duration::from_millis(60_000)); } - #[test] fn spawned_owner_runtime_claim_requires_parent_linkage_for_plugin_owner() { let home_dir = temp_test_dir("owner_runtime_parent"); std::fs::create_dir_all(&home_dir).unwrap(); let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - - let err = - validate_spawned_owner_runtime_claim(&paths, Some("plugin-claude"), None, None, None) - .unwrap_err(); + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); + let err = validate_spawned_owner_runtime_claim(&paths, Some("plugin-claude"), None, None, None).unwrap_err(); assert!(err.contains(SPAWN_PARENT_PID_ENV)); - let _ = std::fs::remove_dir_all(&home_dir); } - #[test] fn spawned_owner_runtime_claim_allows_unspawned_control_center_mode() { let home_dir = temp_test_dir("owner_runtime_unspawned"); std::fs::create_dir_all(&home_dir).unwrap(); let home_str = home_dir.to_string_lossy().to_string(); - let paths = - auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); - + let paths = auth::CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), None); validate_spawned_owner_runtime_claim(&paths, Some("control-center"), None, None, None) .expect("direct control-center owner mode should remain compatible"); - let _ = std::fs::remove_dir_all(&home_dir); } } diff --git a/daemon-rs/src/cli/tests/support.rs b/daemon-rs/src/cli/tests/support.rs index c7f0cb1d..4219b20e 100644 --- a/daemon-rs/src/cli/tests/support.rs +++ b/daemon-rs/src/cli/tests/support.rs @@ -1,60 +1,43 @@ // SPDX-License-Identifier: MIT -//! Shared helpers for CLI unit tests. - -use fs2::FileExt; - -use crate::cli::daemon::{control_center_is_active, CONTROL_CENTER_LOCK_FILE}; +use crate::cli::daemon::{control_center_is_active, startup_single_daemon_preflight, CONTROL_CENTER_LOCK_FILE}; use crate::cli::*; use crate::*; +use fs2::FileExt; use std::fs; use std::io::{ErrorKind, Read, Write}; use std::net::TcpListener; use std::path::PathBuf; use std::process::{Command, Stdio}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - pub(crate) const SPAWN_PARENT_TEST_CHILD_ENV: &str = "CORTEX_SPAWN_PARENT_TEST_CHILD"; pub(crate) const CONTROL_CENTER_LOCK_TEST_CHILD_ENV: &str = "CORTEX_CONTROL_CENTER_LOCK_TEST_CHILD"; pub(crate) const CONTROL_CENTER_LOCK_TEST_HOME_ENV: &str = "CORTEX_CONTROL_CENTER_LOCK_TEST_HOME"; pub(crate) const CONTROL_CENTER_LOCK_TEST_READY_ENV: &str = "CORTEX_CONTROL_CENTER_LOCK_TEST_READY"; pub(crate) const CONTROL_CENTER_LOCK_TEST_HOLD_MS_ENV: &str = "CORTEX_CONTROL_CENTER_LOCK_TEST_HOLD_MS"; - pub(crate) fn openapi_spec_path() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("specs") - .join("cortex-openapi.yaml") + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..").join("specs").join("cortex-openapi.yaml") } - pub(crate) struct ScopedEnvVar { key: &'static str, } - impl ScopedEnvVar { pub(crate) fn set(key: &'static str, value: &str) -> Self { std::env::set_var(key, value); Self { key } } } - impl Drop for ScopedEnvVar { fn drop(&mut self) { std::env::remove_var(self.key); } } - pub(crate) fn env_guard() -> tokio::sync::MutexGuard<'static, ()> { crate::test_env::lock() } - pub(crate) fn temp_test_dir(name: &str) -> std::path::PathBuf { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); + let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos(); std::env::temp_dir().join(format!("cortex_{name}_{unique}")) } - pub(crate) fn run_preflight(paths: &auth::CortexPaths) -> Result<(), String> { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -62,26 +45,16 @@ pub(crate) fn run_preflight(paths: &auth::CortexPaths) -> Result<(), String> { .expect("build tokio runtime") .block_on(startup_single_daemon_preflight(paths)) } - -pub(crate) fn run_ensure_daemon( - paths: &auth::CortexPaths, - agent: Option<&str>, - emit_port: bool, - allow_service_ensure: bool, -) -> Result<(), String> { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("build tokio runtime") - .block_on(ensure_daemon(paths, agent, emit_port, allow_service_ensure)) +pub(crate) fn run_ensure_daemon(paths: &auth::CortexPaths, agent: Option<&str>, emit_port: bool, allow_service_ensure: bool) -> Result<(), String> { + tokio::runtime::Builder::new_current_thread().enable_all().build().expect("build tokio runtime").block_on(ensure_daemon( + paths, + agent, + emit_port, + allow_service_ensure, + )) } - pub(crate) fn spawn_response_server( - listener: TcpListener, - status_line: &str, - content_type: &str, - body: String, - max_requests: usize, + listener: TcpListener, status_line: &str, content_type: &str, body: String, max_requests: usize, ) -> std::thread::JoinHandle<()> { let status_line = status_line.to_string(); let content_type = content_type.to_string(); @@ -112,11 +85,7 @@ pub(crate) fn spawn_response_server( } Err(err) if err.kind() == ErrorKind::WouldBlock => { let now = Instant::now(); - if served > 0 - && last_served_at.is_some_and(|last| { - now.duration_since(last) >= idle_grace_after_response - }) - { + if served > 0 && last_served_at.is_some_and(|last| now.duration_since(last) >= idle_grace_after_response) { break; } if now >= deadline { @@ -129,16 +98,9 @@ pub(crate) fn spawn_response_server( } }) } - -pub(crate) fn spawn_preflight_response_server( - listener: TcpListener, - status_line: &str, - content_type: &str, - body: String, -) -> std::thread::JoinHandle<()> { +pub(crate) fn spawn_preflight_response_server(listener: TcpListener, status_line: &str, content_type: &str, body: String) -> std::thread::JoinHandle<()> { spawn_response_server(listener, status_line, content_type, body, 4) } - pub(crate) fn wait_for_control_center_lock(paths: &auth::CortexPaths, timeout: Duration) -> bool { let deadline = Instant::now() + timeout; while Instant::now() < deadline { @@ -149,26 +111,14 @@ pub(crate) fn wait_for_control_center_lock(paths: &auth::CortexPaths, timeout: D } false } - pub(crate) fn control_center_lock_holder_child_process() { - if std::env::var(CONTROL_CENTER_LOCK_TEST_CHILD_ENV) - .ok() - .as_deref() - != Some("1") - { + if std::env::var(CONTROL_CENTER_LOCK_TEST_CHILD_ENV).ok().as_deref() != Some("1") { return; } - let home = std::env::var(CONTROL_CENTER_LOCK_TEST_HOME_ENV) - .expect("control-center lock test home env missing"); - let ready_file = std::env::var(CONTROL_CENTER_LOCK_TEST_READY_ENV) - .expect("control-center lock ready marker env missing"); - let hold_ms = std::env::var(CONTROL_CENTER_LOCK_TEST_HOLD_MS_ENV) - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(1500); - let lock_path = PathBuf::from(home) - .join("runtime") - .join(CONTROL_CENTER_LOCK_FILE); + let home = std::env::var(CONTROL_CENTER_LOCK_TEST_HOME_ENV).expect("control-center lock test home env missing"); + let ready_file = std::env::var(CONTROL_CENTER_LOCK_TEST_READY_ENV).expect("control-center lock ready marker env missing"); + let hold_ms = std::env::var(CONTROL_CENTER_LOCK_TEST_HOLD_MS_ENV).ok().and_then(|value| value.parse::().ok()).unwrap_or(1500); + let lock_path = PathBuf::from(home).join("runtime").join(CONTROL_CENTER_LOCK_FILE); if let Some(parent) = lock_path.parent() { std::fs::create_dir_all(parent).expect("create lock parent dir"); } @@ -179,9 +129,7 @@ pub(crate) fn control_center_lock_holder_child_process() { .truncate(false) .open(&lock_path) .expect("open lock file"); - lock_file - .try_lock_exclusive() - .expect("acquire control-center lock"); + lock_file.try_lock_exclusive().expect("acquire control-center lock"); std::fs::write(ready_file, b"locked").expect("write lock ready marker"); std::thread::sleep(Duration::from_millis(hold_ms)); } diff --git a/daemon-rs/src/cli/usage.rs b/daemon-rs/src/cli/usage.rs index 67840191..2b59ee9e 100644 --- a/daemon-rs/src/cli/usage.rs +++ b/daemon-rs/src/cli/usage.rs @@ -1,11 +1,6 @@ -// SPDX-License-Identifier: MIT - -use serde_json::{json, Value}; - use crate::DEFAULT_CORTEX_PORT; - +use serde_json::{json, Value}; pub(crate) const CLI_CAPABILITIES_CONTRACT_VERSION: &str = "1"; - pub(crate) fn cli_usage_text() -> String { format!( r#"Cortex v{} -- Universal AI Memory Daemon @@ -104,149 +99,51 @@ Troubleshooting: DEFAULT_CORTEX_PORT ) } - pub(crate) fn cli_service_usage() -> &'static str { "Usage: cortex service " } - pub(crate) fn cli_capabilities_payload() -> Value { json!({ - "schema_version": 1, - "contract_version": CLI_CAPABILITIES_CONTRACT_VERSION, - "tool": { - "name": "cortex", - "version": env!("CARGO_PKG_VERSION"), - "default_port": DEFAULT_CORTEX_PORT, - "default_bind": "127.0.0.1" - }, - "agent_entrypoints": [ - { - "name": "help", - "command": "cortex help", - "output": "human", - "side_effects": "none" - }, - { - "name": "capabilities", - "command": "cortex capabilities --json", - "output": "json", - "side_effects": "none" - }, - { - "name": "status", - "command": "cortex status --json", - "output": "json", - "side_effects": "none" - }, - { - "name": "robot-docs", - "command": "cortex robot-docs guide", - "output": "text", - "side_effects": "none" - } - ], - "commands": { - "serve": { - "usage": "cortex serve [--bind ] [--port ]", - "purpose": "Run the HTTP daemon", - "output": "logs", - "side_effects": "starts_daemon" - }, - "mcp": { - "usage": "cortex mcp [--url ] [--api-key ] [--agent ]", - "purpose": "Run the MCP stdio proxy", - "output": "stdio_json_rpc", - "side_effects": "may_ensure_local_daemon" - }, - "paths": { - "usage": "cortex paths --json", - "purpose": "Print resolved paths, port, and bind configuration", - "output": "json", - "side_effects": "none" - }, - "status": { - "usage": "cortex status [--json]", - "purpose": "Report memory readiness, checks, next action, and repair without starting a daemon", - "output": "human_or_json", - "side_effects": "none" - }, - "boot": { - "usage": "cortex boot [--agent ] [--budget ] [--json]", - "purpose": "Preferred local boot path with auth and SSRF headers", - "output": "human_or_json", - "side_effects": "may_ensure_local_daemon" - }, - "doctor": { - "usage": "cortex doctor", - "purpose": "Validate database schema, migrations, integrity, and FTS health", - "output": "human", - "side_effects": "reads_database" - }, - "admin budgets": { - "usage": "cortex admin budgets status [--json]", - "purpose": "Inspect or validate budget governance configuration", - "output": "human_or_json", - "side_effects": "none" - }, - "admin rollback": { - "usage": "cortex admin rollback --session-id [--apply] [--json]", - "purpose": "Dry-run or apply soft-delete rollback for one session", - "output": "human_or_json", - "side_effects": "mutates_database_with_--apply" - } - }, - "environment": { - "CORTEX_HOME": "Overrides the Cortex home directory", - "CORTEX_PORT": "Overrides the daemon port", - "CORTEX_BIND": "Overrides daemon bind address; defaults to localhost", - "CORTEX_API_KEY": "Supplies API key for remote client commands", - "CORTEX_API_BASE": "Supplies base URL for remote client commands", - "NO_COLOR": "Requests plain output where color is supported" - }, - "exit_codes": { - "0": "success", - "1": "user_input_or_runtime_error" - }, - "dangerous_operations": [ - { - "command": "cortex restore ", - "gate": "requires explicit backup file and warns when daemon appears active" - }, - { - "command": "cortex admin rollback --session-id --apply", - "gate": "dry-run by default; --apply required to mutate" - }, - { - "command": "cortex user remove ", - "gate": "interactive confirmation" - }, - { - "command": "cortex team remove ", - "gate": "interactive confirmation" - } - ], - "recommended_agent_flow": [ - "Run `cortex capabilities --json` to discover supported surfaces.", - "Run `cortex status --json` when you need readiness, next action, or repair without starting a daemon.", - "Use `cortex paths --json` before reading or writing Cortex files.", - "Use `cortex boot --json` for local attachment when a daemon may be needed.", - "Use JSON flags where available and treat non-zero exit as retryable only after inspecting stderr." - ] - }) +"schema_version":1,"contract_version":CLI_CAPABILITIES_CONTRACT_VERSION,"tool":{"name":"cortex","version":env!("CARGO_PKG_VERSION" +),"default_port":DEFAULT_CORTEX_PORT,"default_bind":"127.0.0.1"},"agent_entrypoints":[{"name":"help","command":"cortex help", +"output":"human","side_effects":"none"},{"name":"capabilities","command":"cortex capabilities --json","output":"json", +"side_effects":"none"},{"name":"status","command":"cortex status --json","output":"json","side_effects":"none"},{"name": +"robot-docs","command":"cortex robot-docs guide","output":"text","side_effects":"none"}],"commands":{"serve":{"usage": +"cortex serve [--bind ] [--port ]","purpose":"Run the HTTP daemon","output":"logs","side_effects":"starts_daemon"},"mcp": +{"usage":"cortex mcp [--url ] [--api-key ] [--agent ]","purpose":"Run the MCP stdio proxy","output": +"stdio_json_rpc","side_effects":"may_ensure_local_daemon"},"paths":{"usage":"cortex paths --json","purpose": +"Print resolved paths, port, and bind configuration","output":"json","side_effects":"none"},"status":{"usage": +"cortex status [--json]","purpose":"Report memory readiness, checks, next action, and repair without starting a daemon","output": +"human_or_json","side_effects":"none"},"boot":{"usage":"cortex boot [--agent ] [--budget ] [--json]","purpose": +"Preferred local boot path with auth and SSRF headers","output":"human_or_json","side_effects":"may_ensure_local_daemon"},"doctor" +:{"usage":"cortex doctor","purpose":"Validate database schema, migrations, integrity, and FTS health","output":"human", +"side_effects":"reads_database"},"admin budgets":{"usage":"cortex admin budgets status [--json]","purpose": +"Inspect or validate budget governance configuration","output":"human_or_json","side_effects":"none"},"admin rollback":{"usage": +"cortex admin rollback --session-id [--apply] [--json]","purpose":"Dry-run or apply soft-delete rollback for one session", +"output":"human_or_json","side_effects":"mutates_database_with_--apply"}},"environment":{"CORTEX_HOME": +"Overrides the Cortex home directory","CORTEX_PORT":"Overrides the daemon port","CORTEX_BIND": +"Overrides daemon bind address; defaults to localhost","CORTEX_API_KEY":"Supplies API key for remote client commands", +"CORTEX_API_BASE":"Supplies base URL for remote client commands","NO_COLOR":"Requests plain output where color is supported"}, +"exit_codes":{"0":"success","1":"user_input_or_runtime_error"},"dangerous_operations":[{"command":"cortex restore ","gate": +"requires explicit backup file and warns when daemon appears active"},{"command":"cortex admin rollback --session-id --apply" +,"gate":"dry-run by default; --apply required to mutate"},{"command":"cortex user remove ","gate":"interactive confirmation" +},{"command":"cortex team remove ","gate":"interactive confirmation"}],"recommended_agent_flow":[ +"Run `cortex capabilities --json` to discover supported surfaces.", +"Run `cortex status --json` when you need readiness, next action, or repair without starting a daemon.", +"Use `cortex paths --json` before reading or writing Cortex files.", +"Use `cortex boot --json` for local attachment when a daemon may be needed.", +"Use JSON flags where available and treat non-zero exit as retryable only after inspecting stderr."]}) } - pub(crate) fn cli_capabilities_summary() -> String { format!( - "Cortex agent capabilities\n\ +"Cortex agent capabilities\n\ JSON contract: cortex capabilities --json\n\ Agent guide: cortex robot-docs guide\n\ Core JSON commands: status --json, paths --json, boot --json, reindex --json, recrystallize --json, embeddings status --json, admin budgets status --json\n\ Default daemon endpoint: http://127.0.0.1:{}\n\ - Exit codes: 0 success, 1 user-input or runtime error", - DEFAULT_CORTEX_PORT - ) + Exit codes: 0 success, 1 user-input or runtime error" +,DEFAULT_CORTEX_PORT) } - pub(crate) fn cli_robot_docs_guide() -> &'static str { r#"Cortex robot guide @@ -282,14 +179,11 @@ Output contract: Treat exit code 0 as success and exit code 1 as user-input or runtime failure. "# } - fn top_level_command_suggestion(command: &str) -> Option<&'static str> { let normalized = command.trim().to_ascii_lowercase().replace(['_', '-'], ""); match normalized.as_str() { "capability" | "capabilitiesjson" | "caps" => Some("cortex capabilities --json"), - "robotdoc" | "robotdocs" | "agentdoc" | "agentdocs" | "docs" => { - Some("cortex robot-docs guide") - } + "robotdoc" | "robotdocs" | "agentdoc" | "agentdocs" | "docs" => Some("cortex robot-docs guide"), "stat" | "statusjson" => Some("cortex status --json"), "path" => Some("cortex paths --json"), "budget" | "budgets" => Some("cortex admin budgets status --json"), @@ -297,27 +191,18 @@ fn top_level_command_suggestion(command: &str) -> Option<&'static str> { _ => None, } } - pub(crate) fn unknown_cli_command_message(command: &str) -> String { - let prefix = if command.starts_with('-') { - format!("Unknown option: {command}") - } else { - format!("Unknown command: {command}") - }; + let prefix = if command.starts_with('-') { format!("Unknown option: {command}") } else { format!("Unknown command: {command}") }; match top_level_command_suggestion(command) { - Some(suggestion) => format!( - "{prefix}\nDid you mean: `{suggestion}`?\nRun `cortex help` or `cortex capabilities --json` for supported commands." - ), - None => format!( - "{prefix}\nRun `cortex help` or `cortex capabilities --json` for supported commands." - ), + Some(suggestion) => { + format!("{prefix}\nDid you mean: `{suggestion}`?\nRun `cortex help` or `cortex capabilities --json` for supported commands.") + } + None => format!("{prefix}\nRun `cortex help` or `cortex capabilities --json` for supported commands."), } } - pub(crate) fn unknown_robot_docs_subcommand_message(subcommand: &str) -> String { format!("Unknown robot-docs command: {subcommand}\nDid you mean: `cortex robot-docs guide`?") } - pub(crate) fn print_usage_and_exit(code: i32) -> ! { let usage = cli_usage_text(); if code == 0 { @@ -327,4 +212,3 @@ pub(crate) fn print_usage_and_exit(code: i32) -> ! { } std::process::exit(code); } - diff --git a/daemon-rs/src/co_occurrence.rs b/daemon-rs/src/co_occurrence.rs deleted file mode 100644 index c180aab4..00000000 --- a/daemon-rs/src/co_occurrence.rs +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-License-Identifier: MIT -use std::collections::{HashMap, HashSet}; - -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; - -/// Record pairwise co-occurrences for every unique pair in `sources`. -/// Sources that are blank or appear only once are ignored. -/// At most 10 unique sources are considered per call. -pub fn record(conn: &Connection, sources: &[String]) -> Result<(), String> { - if sources.len() < 2 { - return Ok(()); - } - - let unique = sources - .iter() - .filter(|s| !s.trim().is_empty()) - .cloned() - .collect::>() - .into_iter() - .take(10) - .collect::>(); - - if unique.len() < 2 { - return Ok(()); - } - - for i in 0..unique.len() { - for j in (i + 1)..unique.len() { - let (a, b) = if unique[i] <= unique[j] { - (unique[i].clone(), unique[j].clone()) - } else { - (unique[j].clone(), unique[i].clone()) - }; - - conn.execute( - "INSERT INTO co_occurrence (source_a, source_b, count, last_seen) - VALUES (?1, ?2, 1, datetime('now')) - ON CONFLICT(source_a, source_b) DO UPDATE SET - count = count + 1, - last_seen = datetime('now')", - params![a, b], - ) - .map_err(|e| e.to_string())?; - } - } - - Ok(()) -} - -/// Return up to `limit` sources that frequently co-occur with -/// `recalled_sources` but are not already in that set. -/// Each result is a JSON object `{ "source": "...", "coScore": }`. -pub fn predict( - conn: &Connection, - recalled_sources: &[String], - limit: usize, -) -> Result, String> { - if recalled_sources.is_empty() { - return Ok(vec![]); - } - - let already_have = recalled_sources - .iter() - .filter(|s| !s.trim().is_empty()) - .cloned() - .collect::>(); - - let mut candidates: HashMap = HashMap::new(); - - for source in &already_have { - let mut stmt = conn - .prepare( - "SELECT - CASE WHEN source_a = ?1 THEN source_b ELSE source_a END AS partner, - count - FROM co_occurrence - WHERE source_a = ?1 OR source_b = ?1 - ORDER BY count DESC - LIMIT 10", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map(params![source], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - }) - .map_err(|e| e.to_string())?; - - for row in rows.flatten() { - let (partner, count) = row; - if already_have.contains(&partner) { - continue; - } - let existing = candidates.get(&partner).copied().unwrap_or(0); - candidates.insert(partner, existing + count); - } - } - - let mut ranked = candidates.into_iter().collect::>(); - ranked.sort_by(|a, b| b.1.cmp(&a.1)); - ranked.truncate(limit); - - Ok(ranked - .into_iter() - .map(|(source, score)| json!({ "source": source, "coScore": score })) - .collect()) -} - -/// Delete all rows from the `co_occurrence` table. -pub fn reset(conn: &Connection) -> Result<(), String> { - conn.execute("DELETE FROM co_occurrence", []) - .map_err(|e| e.to_string())?; - Ok(()) -} - -// ─── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use rusqlite::Connection; - - fn setup() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - conn - } - - #[test] - fn test_record_and_predict() { - let conn = setup(); - - let sources = vec![ - "source_a".to_string(), - "source_b".to_string(), - "source_c".to_string(), - ]; - record(&conn, &sources).unwrap(); - record(&conn, &sources).unwrap(); // Second call increases counts - - let predictions = predict(&conn, &["source_a".to_string()], 5).unwrap(); - assert!(!predictions.is_empty()); - - // Every prediction should have a coScore > 0 - for p in &predictions { - assert!(p["coScore"].as_i64().unwrap() > 0); - } - } - - #[test] - fn test_predict_excludes_known_sources() { - let conn = setup(); - - let sources = vec!["source_a".to_string(), "source_b".to_string()]; - record(&conn, &sources).unwrap(); - - // Predicting with both sources — neither should appear in results - let predictions = predict(&conn, &sources, 5).unwrap(); - for p in &predictions { - let s = p["source"].as_str().unwrap(); - assert_ne!(s, "source_a"); - assert_ne!(s, "source_b"); - } - } - - #[test] - fn test_reset() { - let conn = setup(); - - let sources = vec!["source_a".to_string(), "source_b".to_string()]; - record(&conn, &sources).unwrap(); - - reset(&conn).unwrap(); - - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM co_occurrence", [], |row| row.get(0)) - .unwrap(); - assert_eq!(count, 0); - } - - #[test] - fn test_record_fewer_than_two_sources_is_noop() { - let conn = setup(); - record(&conn, &["only_one".to_string()]).unwrap(); - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM co_occurrence", [], |row| row.get(0)) - .unwrap(); - assert_eq!(count, 0); - } - - #[test] - fn test_predict_empty_recalled_sources() { - let conn = setup(); - let results = predict(&conn, &[], 5).unwrap(); - assert!(results.is_empty()); - } -} diff --git a/daemon-rs/src/co_occurrence/tests/mod.rs b/daemon-rs/src/co_occurrence/tests/mod.rs new file mode 100644 index 00000000..a7488af7 --- /dev/null +++ b/daemon-rs/src/co_occurrence/tests/mod.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use rusqlite::Connection; +fn setup() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + conn +} +#[test] +fn test_record_and_predict() { + let conn = setup(); + let sources = vec!["source_a".to_string(), "source_b".to_string(), "source_c".to_string()]; + record(&conn, &sources).unwrap(); + record(&conn, &sources).unwrap(); // Second call increases counts + let predictions = predict(&conn, &["source_a".to_string()], 5).unwrap(); + assert!(!predictions.is_empty()); + for p in &predictions { + assert!(p["coScore"].as_i64().unwrap() > 0); + } +} +#[test] +fn test_predict_excludes_known_sources() { + let conn = setup(); + let sources = vec!["source_a".to_string(), "source_b".to_string()]; + record(&conn, &sources).unwrap(); + let predictions = predict(&conn, &sources, 5).unwrap(); + for p in &predictions { + let s = p["source"].as_str().unwrap(); + assert_ne!(s, "source_a"); + assert_ne!(s, "source_b"); + } +} +#[test] +fn test_reset() { + let conn = setup(); + let sources = vec!["source_a".to_string(), "source_b".to_string()]; + record(&conn, &sources).unwrap(); + reset(&conn).unwrap(); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM co_occurrence", [], |row| row.get(0)).unwrap(); + assert_eq!(count, 0); +} +#[test] +fn test_record_fewer_than_two_sources_is_noop() { + let conn = setup(); + record(&conn, &["only_one".to_string()]).unwrap(); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM co_occurrence", [], |row| row.get(0)).unwrap(); + assert_eq!(count, 0); +} +#[test] +fn test_predict_empty_recalled_sources() { + let conn = setup(); + let results = predict(&conn, &[], 5).unwrap(); + assert!(results.is_empty()); +} diff --git a/daemon-rs/src/compaction/archived.rs b/daemon-rs/src/compaction/archived.rs index 14d2e9f3..6d0448db 100644 --- a/daemon-rs/src/compaction/archived.rs +++ b/daemon-rs/src/compaction/archived.rs @@ -1,22 +1,10 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - use super::*; -// ─── Archived entry text cleanup ──────────────────────────────────────────── - -/// Strip full text from archived entries older than retention period. -/// Keeps: id, source, type, status, created_at, score (for audit). -/// Drops: text, compressed_text, tags, context (saves space). +use rusqlite::{params, Connection}; pub(crate) fn strip_archived_text(conn: &Connection) -> usize { strip_archived_text_with_retention(conn, ARCHIVED_TEXT_RETENTION_DAYS) } - pub(crate) fn strip_archived_text_with_retention(conn: &Connection, retention_days: i64) -> usize { let mut count = 0usize; - count += conn .execute( "UPDATE memories SET text = '[compacted]', tags = NULL \ @@ -26,7 +14,6 @@ pub(crate) fn strip_archived_text_with_retention(conn: &Connection, retention_da params![retention_days], ) .unwrap_or(0); - count += conn .execute( "UPDATE decisions SET decision = '[compacted]', context = NULL \ @@ -36,31 +23,15 @@ pub(crate) fn strip_archived_text_with_retention(conn: &Connection, retention_da params![retention_days], ) .unwrap_or(0); - count } - pub(crate) fn prune_expired_entries(conn: &Connection) -> usize { - let memories_deleted = conn - .execute( - "DELETE FROM memories WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", - [], - ) - .unwrap_or(0); - - let decisions_deleted = conn - .execute( - "DELETE FROM decisions WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", - [], - ) - .unwrap_or(0); - + let memories_deleted = conn.execute("DELETE FROM memories WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", []).unwrap_or(0); + let decisions_deleted = conn.execute("DELETE FROM decisions WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", []).unwrap_or(0); let count = memories_deleted + decisions_deleted; if count > 0 { - let payload = serde_json::json!({ - "memories_deleted": memories_deleted, - "decisions_deleted": decisions_deleted, - }) + let payload = serde_json::json!({"memories_deleted":memories_deleted, +"decisions_deleted":decisions_deleted,}) .to_string(); let _ = conn.execute( "INSERT INTO events (type, data, source_agent, created_at) \ @@ -68,7 +39,5 @@ pub(crate) fn prune_expired_entries(conn: &Connection) -> usize { params![payload], ); } - count } - diff --git a/daemon-rs/src/compaction/crystals.rs b/daemon-rs/src/compaction/crystals.rs index 5655dcb6..5c62ef52 100644 --- a/daemon-rs/src/compaction/crystals.rs +++ b/daemon-rs/src/compaction/crystals.rs @@ -1,18 +1,6 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - -use super::*; -// ─── Crystal member embedding pruning ─────────────────────────────────────── - -/// Remove individual embeddings for entries that are members of a crystal. -/// The crystal's embedding handles recall; individual members are found by -/// ID lookup through cluster_members, not semantic search. +use rusqlite::Connection; pub(crate) fn prune_crystal_member_embeddings(conn: &Connection) -> usize { let mut count = 0usize; - count += conn .execute( "DELETE FROM embeddings WHERE target_type = 'memory' AND target_id IN (\ @@ -21,7 +9,6 @@ pub(crate) fn prune_crystal_member_embeddings(conn: &Connection) -> usize { [], ) .unwrap_or(0); - count += conn .execute( "DELETE FROM embeddings WHERE target_type = 'decision' AND target_id IN (\ @@ -30,10 +17,8 @@ pub(crate) fn prune_crystal_member_embeddings(conn: &Connection) -> usize { [], ) .unwrap_or(0); - count } - pub(crate) fn prune_orphan_cluster_members(conn: &Connection) -> usize { let mut count = 0usize; count += conn @@ -68,4 +53,3 @@ pub(crate) fn prune_orphan_cluster_members(conn: &Connection) -> usize { .unwrap_or(0); count } - diff --git a/daemon-rs/src/compaction/events.rs b/daemon-rs/src/compaction/events.rs index 648e4f1c..569d3ea4 100644 --- a/daemon-rs/src/compaction/events.rs +++ b/daemon-rs/src/compaction/events.rs @@ -1,20 +1,11 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - use super::*; -// ─── Event log rotation ───────────────────────────────────────────────────── - +use rusqlite::{params, Connection}; pub(crate) fn rollup_old_boot_savings(conn: &Connection) -> usize { rollup_old_boot_savings_with_retention(conn, BOOT_SAVINGS_RETENTION_DAYS) } - pub(crate) fn rollup_old_boot_savings_with_retention(conn: &Connection, retention_days: i64) -> usize { let retention_window = format!("-{retention_days} days"); let benchmark_source_pattern = format!("{BENCHMARK_SOURCE_AGENT_PREFIX}%"); - let (old_saved, old_served, old_baseline, old_boots): (i64, i64, i64, i64) = conn .query_row( "SELECT \ @@ -32,14 +23,7 @@ pub(crate) fn rollup_old_boot_savings_with_retention(conn: &Connection, retentio |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), ) .unwrap_or((0, 0, 0, 0)); - - let (rollup_saved, rollup_served, rollup_baseline, rollup_boots, rollup_rows): ( - i64, - i64, - i64, - i64, - i64, - ) = conn + let (rollup_saved, rollup_served, rollup_baseline, rollup_boots, rollup_rows): (i64, i64, i64, i64, i64) = conn .query_row( "SELECT \ COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0)), 0), \ @@ -50,27 +34,16 @@ pub(crate) fn rollup_old_boot_savings_with_retention(conn: &Connection, retentio FROM events \ WHERE type = 'boot_savings_rollup'", [], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - )) - }, + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)), ) .unwrap_or((0, 0, 0, 0, 0)); - if old_boots <= 0 && rollup_rows <= 1 { return 0; } - let merged_saved = old_saved + rollup_saved; let merged_served = old_served + rollup_served; let merged_baseline = old_baseline + rollup_baseline; let merged_boots = old_boots + rollup_boots; - let deleted_old = conn .execute( "DELETE FROM events \ @@ -82,20 +55,11 @@ pub(crate) fn rollup_old_boot_savings_with_retention(conn: &Connection, retentio params![retention_window, benchmark_source_pattern], ) .unwrap_or(0); - - let deleted_rollups = conn - .execute("DELETE FROM events WHERE type = 'boot_savings_rollup'", []) - .unwrap_or(0); - + let deleted_rollups = conn.execute("DELETE FROM events WHERE type = 'boot_savings_rollup'", []).unwrap_or(0); if merged_boots > 0 { - let payload = serde_json::json!({ - "saved": merged_saved, - "served": merged_served, - "baseline": merged_baseline, - "boots": merged_boots, - "retention_days": retention_days, - "rolled_up_at": chrono::Utc::now().to_rfc3339(), - }) + let payload = serde_json::json!({"saved": +merged_saved,"served":merged_served,"baseline":merged_baseline,"boots":merged_boots,"retention_days":retention_days,"rolled_up_at" +:chrono::Utc::now().to_rfc3339(),}) .to_string(); let _ = conn.execute( "INSERT INTO events (type, data, source_agent, created_at) \ @@ -108,14 +72,12 @@ pub(crate) fn rollup_old_boot_savings_with_retention(conn: &Connection, retentio deleted_old + deleted_rollups } } - pub(crate) fn rollup_old_savings_events(conn: &Connection, retention_days: i64) -> usize { let retention_window = format!("-{retention_days} days"); let benchmark_source_pattern = format!("{BENCHMARK_SOURCE_AGENT_PREFIX}%"); type SavingsRollupRow = (String, i64, String, i64, i64, i64, i64, i64, i64); - let rollup_rows: Vec = conn - .prepare( - "SELECT \ + let rollup_rows:Vec=conn.prepare( +"SELECT \ SUBSTR(created_at, 1, 10) AS day, \ COALESCE(CAST(strftime('%H', REPLACE(SUBSTR(created_at, 1, 19), 'T', ' ')) AS INTEGER), 0) AS hour, \ CASE \ @@ -153,33 +115,13 @@ pub(crate) fn rollup_old_savings_events(conn: &Connection, retention_days: i64) AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) \ - GROUP BY day, hour, operation", - ) - .and_then(|mut stmt| { - let rows = stmt.query_map( - params![retention_window.clone(), benchmark_source_pattern.clone()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(4)?, - row.get::<_, i64>(5)?, - row.get::<_, i64>(6)?, - row.get::<_, i64>(7)?, - row.get::<_, i64>(8)?, - )) - }, - )?; - Ok(rows.flatten().collect()) - }) - .unwrap_or_default(); - + GROUP BY day, hour, operation" +,).and_then(|mut stmt|{let rows=stmt.query_map(params![retention_window.clone(),benchmark_source_pattern.clone()],|row|{Ok((row. +get::<_,String>(0)?,row.get::<_,i64>(1)?,row.get::<_,String>(2)?,row.get::<_,i64>(3)?,row.get::<_,i64>(4)?,row.get::<_,i64>(5)?, +row.get::<_,i64>(6)?,row.get::<_,i64>(7)?,row.get::<_,i64>(8)?,))})?;Ok(rows.flatten().collect())}).unwrap_or_default(); if rollup_rows.is_empty() { return 0; } - for (day, hour, operation, saved, served, baseline, events, hits, misses) in rollup_rows { let _ = conn.execute( "INSERT INTO event_savings_rollups \ @@ -196,7 +138,6 @@ pub(crate) fn rollup_old_savings_events(conn: &Connection, retention_days: i64) params![day, hour, operation, saved, served, baseline, events, hits, misses], ); } - conn.execute( "DELETE FROM events \ WHERE type IN ('recall_query', 'store_savings', 'tool_call_savings') \ @@ -209,7 +150,6 @@ pub(crate) fn rollup_old_savings_events(conn: &Connection, retention_days: i64) ) .unwrap_or(0) } - pub(crate) fn prune_old_event_savings_rollups(conn: &Connection, retention_days: i64) -> usize { conn.execute( "DELETE FROM event_savings_rollups \ @@ -218,22 +158,15 @@ pub(crate) fn prune_old_event_savings_rollups(conn: &Connection, retention_days: ) .unwrap_or(0) } - #[cfg(test)] pub(crate) fn prune_old_events(conn: &Connection) -> usize { prune_old_events_with_retention_limit(conn, EVENT_RETENTION_DAYS, None) } - #[cfg(test)] pub(crate) fn prune_old_events_with_retention(conn: &Connection, retention_days: i64) -> usize { prune_old_events_with_retention_limit(conn, retention_days, None) } - -pub(crate) fn prune_old_events_with_retention_limit( - conn: &Connection, - retention_days: i64, - max_delete_rows: Option, -) -> usize { +pub(crate) fn prune_old_events_with_retention_limit(conn: &Connection, retention_days: i64, max_delete_rows: Option) -> usize { let retention_window = format!("-{retention_days} days"); if let Some(max_rows) = max_delete_rows.filter(|rows| *rows > 0) { return conn @@ -259,17 +192,11 @@ pub(crate) fn prune_old_events_with_retention_limit( ) .unwrap_or(0) } - #[cfg(test)] pub(crate) fn prune_event_type_caps(conn: &Connection, caps: &[(&str, i64)]) -> usize { prune_event_type_caps_with_limit(conn, caps, None) } - -pub(crate) fn prune_event_type_caps_with_limit( - conn: &Connection, - caps: &[(&str, i64)], - max_delete_rows: Option, -) -> usize { +pub(crate) fn prune_event_type_caps_with_limit(conn: &Connection, caps: &[(&str, i64)], max_delete_rows: Option) -> usize { let mut total = 0usize; for (event_type, keep_rows) in caps.iter().copied() { if keep_rows <= 0 { @@ -311,22 +238,14 @@ pub(crate) fn prune_event_type_caps_with_limit( } total } - #[cfg(test)] pub(crate) fn prune_nonboot_event_overflow(conn: &Connection, keep_rows: i64) -> usize { prune_nonboot_event_overflow_with_limit(conn, keep_rows, None) } - -pub(crate) fn prune_nonboot_event_overflow_with_limit( - conn: &Connection, - keep_rows: i64, - max_delete_rows: Option, -) -> usize { +pub(crate) fn prune_nonboot_event_overflow_with_limit(conn: &Connection, keep_rows: i64, max_delete_rows: Option) -> usize { if keep_rows <= 0 { return 0; } - // Keep recall/store/tool savings events out of global overflow pruning so - // /savings can rely on their short-horizon raw rows until rollup runs. let protected_analytics_rows: i64 = conn .query_row( "SELECT COUNT(*) @@ -345,7 +264,6 @@ pub(crate) fn prune_nonboot_event_overflow_with_limit( 'store_savings', 'tool_call_savings' )"; - if let Some(max_rows) = max_delete_rows.filter(|rows| *rows > 0) { return conn .execute( @@ -383,13 +301,6 @@ pub(crate) fn prune_nonboot_event_overflow_with_limit( ) .unwrap_or(0) } - pub(crate) fn checkpoint_after_compaction(conn: &Connection, allow_vacuum: bool) { - let _ = if allow_vacuum { - conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") - } else { - // Startup governor mode: avoid TRUNCATE stalls while still nudging WAL forward. - conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);") - }; + let _ = if allow_vacuum { conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") } else { conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);") }; } - diff --git a/daemon-rs/src/compaction/feedback.rs b/daemon-rs/src/compaction/feedback.rs index a1b18d54..8d776e41 100644 --- a/daemon-rs/src/compaction/feedback.rs +++ b/daemon-rs/src/compaction/feedback.rs @@ -1,21 +1,9 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - use super::*; -// ─── Feedback aggregation ─────────────────────────────────────────────────── - -/// Compact old individual feedback signals into per-source aggregates. -/// Before: 50 rows for "memory::foo" with signal 1.0, -0.5, 1.0, ... -/// After: 1 row for "memory::foo" with signal = net_sum, query_text = "[aggregated]" +use rusqlite::{params, Connection}; pub(crate) fn aggregate_old_feedback(conn: &Connection) -> usize { aggregate_old_feedback_with_window(conn, FEEDBACK_AGGREGATION_DAYS) } - pub(crate) fn aggregate_old_feedback_with_window(conn: &Connection, aggregation_days: i64) -> usize { - // Find sources with old feedback to aggregate let sources: Vec<(String, f64, i64)> = conn .prepare( "SELECT result_source, SUM(signal), COUNT(*) \ @@ -24,24 +12,15 @@ pub(crate) fn aggregate_old_feedback_with_window(conn: &Connection, aggregation_ GROUP BY result_source HAVING COUNT(*) > 1", ) .and_then(|mut stmt| { - let rows = stmt.query_map(params![aggregation_days], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, f64>(1)?, - row.get::<_, i64>(2)?, - )) - })?; + let rows = stmt.query_map(params![aggregation_days], |row| Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?, row.get::<_, i64>(2)?)))?; Ok(rows.flatten().collect()) }) .unwrap_or_default(); - if sources.is_empty() { return 0; } - let mut aggregated = 0usize; for (source, net_signal, _count) in &sources { - // Delete old individual rows let deleted = conn .execute( "DELETE FROM recall_feedback \ @@ -50,8 +29,6 @@ pub(crate) fn aggregate_old_feedback_with_window(conn: &Connection, aggregation_ params![source, aggregation_days], ) .unwrap_or(0); - - // Insert one aggregated row if deleted > 0 { let _ = conn.execute( "INSERT INTO recall_feedback (query_text, result_source, result_type, signal, agent, created_at) \ @@ -61,31 +38,15 @@ pub(crate) fn aggregate_old_feedback_with_window(conn: &Connection, aggregation_ aggregated += deleted; } } - aggregated } - -pub(crate) fn prune_old_benchmark_artifacts( - conn: &Connection, - retention_days: i64, - allow_vacuum: bool, -) -> usize { - purge_benchmark_artifacts_with_retention(conn, Some(retention_days), allow_vacuum) - .total_deleted() +pub(crate) fn prune_old_benchmark_artifacts(conn: &Connection, retention_days: i64, allow_vacuum: bool) -> usize { + purge_benchmark_artifacts_with_retention(conn, Some(retention_days), allow_vacuum).total_deleted() } - -pub(crate) fn purge_benchmark_artifacts_with_retention( - conn: &Connection, - retention_days: Option, - allow_vacuum: bool, -) -> BenchmarkPurgeResult { - let mut result = BenchmarkPurgeResult { - bytes_before: db_size_bytes(conn), - ..BenchmarkPurgeResult::default() - }; +pub(crate) fn purge_benchmark_artifacts_with_retention(conn: &Connection, retention_days: Option, allow_vacuum: bool) -> BenchmarkPurgeResult { + let mut result = BenchmarkPurgeResult { bytes_before: db_size_bytes(conn), ..BenchmarkPurgeResult::default() }; let benchmark_source_pattern = format!("{BENCHMARK_SOURCE_AGENT_PREFIX}%"); let retention_window = retention_days.map(|days| format!("-{days} days")); - let _ = conn.execute_batch( "DROP TABLE IF EXISTS temp._benchmark_decision_ids; CREATE TEMP TABLE IF NOT EXISTS _benchmark_decision_ids ( @@ -93,7 +54,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( ); DELETE FROM _benchmark_decision_ids;", ); - match retention_window.as_deref() { Some(window) => { let _ = conn.execute( @@ -117,7 +77,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( ); } } - result.decision_conflicts_deleted = conn .execute( "DELETE FROM decision_conflicts \ @@ -126,7 +85,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( [], ) .unwrap_or(0); - result.embeddings_deleted = conn .execute( "DELETE FROM embeddings \ @@ -135,7 +93,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( [], ) .unwrap_or(0); - result.cluster_members_deleted = conn .execute( "DELETE FROM cluster_members \ @@ -145,7 +102,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( ) .unwrap_or(0); result.cluster_members_deleted += prune_orphan_cluster_members(conn); - result.recall_feedback_deleted = conn .execute( "DELETE FROM recall_feedback \ @@ -154,7 +110,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( [], ) .unwrap_or(0); - result.co_occurrence_deleted = conn .execute( "DELETE FROM co_occurrence \ @@ -163,14 +118,7 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( [], ) .unwrap_or(0); - - result.decisions_deleted = conn - .execute( - "DELETE FROM decisions WHERE id IN (SELECT id FROM _benchmark_decision_ids)", - [], - ) - .unwrap_or(0); - + result.decisions_deleted = conn.execute("DELETE FROM decisions WHERE id IN (SELECT id FROM _benchmark_decision_ids)", []).unwrap_or(0); result.events_deleted += conn .execute( "DELETE FROM events \ @@ -179,7 +127,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( [], ) .unwrap_or(0); - match retention_window.as_deref() { Some(window) => { result.recall_feedback_deleted += conn @@ -224,7 +171,6 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( .unwrap_or(0); } } - let _ = conn.execute_batch("DROP TABLE IF EXISTS temp._benchmark_decision_ids;"); let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);"); if allow_vacuum { @@ -236,4 +182,3 @@ pub(crate) fn purge_benchmark_artifacts_with_retention( result.bytes_after = db_size_bytes(conn); result } - diff --git a/daemon-rs/src/compaction/governor.rs b/daemon-rs/src/compaction/governor.rs index e9100d0e..c0013f42 100644 --- a/daemon-rs/src/compaction/governor.rs +++ b/daemon-rs/src/compaction/governor.rs @@ -1,12 +1,5 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - use super::*; -// ─── Result ───────────────────────────────────────────────────────────────── - +use rusqlite::{params, Connection}; #[derive(Debug, Default)] pub struct CompactionResult { pub events_pruned: usize, @@ -23,7 +16,6 @@ pub struct CompactionResult { pub bytes_before: i64, pub bytes_after: i64, } - #[derive(Debug, Default)] pub struct BenchmarkPurgeResult { pub decisions_deleted: usize, @@ -36,7 +28,6 @@ pub struct BenchmarkPurgeResult { pub bytes_before: i64, pub bytes_after: i64, } - impl BenchmarkPurgeResult { pub fn total_deleted(&self) -> usize { self.decisions_deleted @@ -48,13 +39,9 @@ impl BenchmarkPurgeResult { + self.events_deleted } } - pub(crate) fn bytes_to_mb(bytes: i64) -> i64 { bytes / (1024 * 1024) } - -/// Classify current storage pressure based on DB size. -/// This is advisory only; Cortex should compact automatically, not reject writes. pub fn classify_storage_pressure(db_size_bytes: i64) -> &'static str { if db_size_bytes >= STORAGE_HARD_LIMIT_BYTES { "critical" @@ -64,8 +51,6 @@ pub fn classify_storage_pressure(db_size_bytes: i64) -> &'static str { "normal" } } - -/// Classify non-boot event pressure so callers can explain when compaction is needed. pub fn classify_event_pressure(nonboot_event_rows: i64) -> &'static str { if nonboot_event_rows >= EVENT_NONBOOT_HARD_LIMIT_ROWS { "critical" @@ -75,34 +60,17 @@ pub fn classify_event_pressure(nonboot_event_rows: i64) -> &'static str { "normal" } } - -/// FTS5 segment row count above this triggers the governor even when the -/// overall DB size is well under soft limit. Without this, FTS shadow tables -/// can balloon to hundreds of MB before the size-based trigger fires. pub const FTS_SEGMENT_ROW_SOFT_LIMIT: i64 = 10_000; - -/// Decide whether the storage governor should run compaction. -/// Runs when DB size is above soft limit or when reclaimable free pages are high. #[cfg_attr(not(test), allow(dead_code))] pub fn should_run_compaction_governor(db_size_bytes: i64, freelist_pages: i64) -> bool { should_run_compaction_governor_with_pressure(db_size_bytes, freelist_pages, 0, 0) } - -pub(crate) fn should_run_compaction_governor_with_pressure( - db_size_bytes: i64, - freelist_pages: i64, - nonboot_event_rows: i64, - fts_segment_rows: i64, -) -> bool { +pub(crate) fn should_run_compaction_governor_with_pressure(db_size_bytes: i64, freelist_pages: i64, nonboot_event_rows: i64, fts_segment_rows: i64) -> bool { db_size_bytes >= STORAGE_SOFT_LIMIT_BYTES || freelist_pages > VACUUM_FREELIST_THRESHOLD_PAGES || nonboot_event_rows > EVENT_NONBOOT_SOFT_LIMIT_ROWS || fts_segment_rows > FTS_SEGMENT_ROW_SOFT_LIMIT } - -/// Sum of rows across all known FTS5 _data shadow tables. The _data table -/// holds one row per FTS5 segment block; runaway segment counts are the -/// dominant bloat driver in long-lived Cortex DBs. pub fn fts_segment_row_total(conn: &Connection) -> i64 { let tables = ["decisions_fts_data", "memories_fts_data"]; let mut total: i64 = 0; @@ -110,89 +78,43 @@ pub fn fts_segment_row_total(conn: &Connection) -> i64 { if !table_exists(conn, table) { continue; } - let n: i64 = conn - .query_row(&format!("SELECT COUNT(*) FROM \"{table}\""), [], |row| { - row.get(0) - }) - .unwrap_or(0); + let n: i64 = conn.query_row(&format!("SELECT COUNT(*) FROM \"{table}\""), [], |row| row.get(0)).unwrap_or(0); total += n; } total } - -/// Run compaction only when pressure or reclaimable space justifies IO. -/// Returns `Some(result)` when a compaction pass ran, `None` when skipped. pub fn run_compaction_governor(conn: &Connection) -> Option { run_compaction_governor_with_options(conn, true) } - -/// Startup-safe governor mode that relieves event pressure without forcing VACUUM. -/// This keeps startup/early-runtime lock windows shorter while still enforcing -/// retention and event-cap policies. pub fn run_compaction_governor_startup(conn: &Connection) -> Option { run_compaction_governor_with_options(conn, false) } - -pub(crate) fn run_compaction_governor_with_options( - conn: &Connection, - allow_vacuum: bool, -) -> Option { +pub(crate) fn run_compaction_governor_with_options(conn: &Connection, allow_vacuum: bool) -> Option { let startup_prune_limit = (!allow_vacuum).then_some(STARTUP_EVENT_PRUNE_BATCH_ROWS); let before = db_size_bytes(conn); let freelist_pages = freelist_count(conn); let nonboot_event_rows_before = non_boot_event_count(conn); let fts_segment_rows_before = fts_segment_row_total(conn); let pressure_before = classify_storage_pressure(before); - - if !should_run_compaction_governor_with_pressure( - before, - freelist_pages, - nonboot_event_rows_before, - fts_segment_rows_before, - ) { + if !should_run_compaction_governor_with_pressure(before, freelist_pages, nonboot_event_rows_before, fts_segment_rows_before) { return None; } - let mut result = run_compaction_with_options(conn, allow_vacuum); - - // Critical pressure gets an additional safe-aggressive pass. We still only touch: - // old events, archived text, and aged feedback (never active memory content). - if before >= STORAGE_HARD_LIMIT_BYTES - || nonboot_event_rows_before >= EVENT_NONBOOT_HARD_LIMIT_ROWS - { - result.events_pruned += - rollup_old_boot_savings_with_retention(conn, AGGRESSIVE_BOOT_SAVINGS_RETENTION_DAYS); - result.events_pruned += - rollup_old_savings_events(conn, AGGRESSIVE_SAVINGS_EVENT_ROLLUP_RETENTION_DAYS); - result.events_pruned += - prune_old_event_savings_rollups(conn, AGGRESSIVE_EVENT_SAVINGS_ROLLUP_RETENTION_DAYS); - result.events_pruned += prune_old_events_with_retention_limit( - conn, - AGGRESSIVE_EVENT_RETENTION_DAYS, - startup_prune_limit, - ); - result.events_pruned += - prune_event_type_caps_with_limit(conn, EVENT_TYPE_HARD_CAPS, startup_prune_limit); - result.events_pruned += prune_nonboot_event_overflow_with_limit( - conn, - EVENT_NONBOOT_HARD_KEEP_ROWS, - startup_prune_limit, - ); - result.benchmark_pruned += - prune_old_benchmark_artifacts(conn, AGGRESSIVE_BENCHMARK_RETENTION_DAYS, allow_vacuum); - result.archived_text_stripped += - strip_archived_text_with_retention(conn, AGGRESSIVE_ARCHIVED_TEXT_RETENTION_DAYS); + if before >= STORAGE_HARD_LIMIT_BYTES || nonboot_event_rows_before >= EVENT_NONBOOT_HARD_LIMIT_ROWS { + result.events_pruned += rollup_old_boot_savings_with_retention(conn, AGGRESSIVE_BOOT_SAVINGS_RETENTION_DAYS); + result.events_pruned += rollup_old_savings_events(conn, AGGRESSIVE_SAVINGS_EVENT_ROLLUP_RETENTION_DAYS); + result.events_pruned += prune_old_event_savings_rollups(conn, AGGRESSIVE_EVENT_SAVINGS_ROLLUP_RETENTION_DAYS); + result.events_pruned += prune_old_events_with_retention_limit(conn, AGGRESSIVE_EVENT_RETENTION_DAYS, startup_prune_limit); + result.events_pruned += prune_event_type_caps_with_limit(conn, EVENT_TYPE_HARD_CAPS, startup_prune_limit); + result.events_pruned += prune_nonboot_event_overflow_with_limit(conn, EVENT_NONBOOT_HARD_KEEP_ROWS, startup_prune_limit); + result.benchmark_pruned += prune_old_benchmark_artifacts(conn, AGGRESSIVE_BENCHMARK_RETENTION_DAYS, allow_vacuum); + result.archived_text_stripped += strip_archived_text_with_retention(conn, AGGRESSIVE_ARCHIVED_TEXT_RETENTION_DAYS); result.cluster_members_pruned += prune_orphan_cluster_members(conn); - result.feedback_aggregated += - aggregate_old_feedback_with_window(conn, AGGRESSIVE_FEEDBACK_AGGREGATION_DAYS); - let _ = if allow_vacuum { - conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE); VACUUM;") - } else { - conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);") - }; + result.feedback_aggregated += aggregate_old_feedback_with_window(conn, AGGRESSIVE_FEEDBACK_AGGREGATION_DAYS); + let _ = + if allow_vacuum { conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE); VACUUM;") } else { conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);") }; result.bytes_after = db_size_bytes(conn); } - let pressure_after = classify_storage_pressure(result.bytes_after); let fts_segment_rows_after = fts_segment_row_total(conn); eprintln!( @@ -206,82 +128,31 @@ pub(crate) fn run_compaction_governor_with_options( fts_segment_rows_before, fts_segment_rows_after, ); - Some(result) } - -// ─── Main entry point ─────────────────────────────────────────────────────── - -/// Run one compaction pass. Safe to call repeatedly. pub fn run_compaction(conn: &Connection) -> CompactionResult { run_compaction_with_options(conn, true) } - pub(crate) fn run_compaction_with_options(conn: &Connection, allow_vacuum: bool) -> CompactionResult { let startup_prune_limit = (!allow_vacuum).then_some(STARTUP_EVENT_PRUNE_BATCH_ROWS); - let mut result = CompactionResult { - bytes_before: db_size_bytes(conn), - ..CompactionResult::default() - }; - - // 1. Event log rotation + let mut result = CompactionResult { bytes_before: db_size_bytes(conn), ..CompactionResult::default() }; result.events_pruned = rollup_old_boot_savings(conn); result.events_pruned += rollup_old_savings_events(conn, SAVINGS_EVENT_ROLLUP_RETENTION_DAYS); - result.events_pruned += - prune_old_event_savings_rollups(conn, EVENT_SAVINGS_ROLLUP_RETENTION_DAYS); - result.events_pruned += - prune_old_events_with_retention_limit(conn, EVENT_RETENTION_DAYS, startup_prune_limit); - result.events_pruned += - prune_event_type_caps_with_limit(conn, EVENT_TYPE_SOFT_CAPS, startup_prune_limit); - result.events_pruned += prune_nonboot_event_overflow_with_limit( - conn, - EVENT_NONBOOT_SOFT_KEEP_ROWS, - startup_prune_limit, - ); - result.benchmark_pruned = - prune_old_benchmark_artifacts(conn, BENCHMARK_RETENTION_DAYS, allow_vacuum); - - // 2. Archived entry text cleanup + result.events_pruned += prune_old_event_savings_rollups(conn, EVENT_SAVINGS_ROLLUP_RETENTION_DAYS); + result.events_pruned += prune_old_events_with_retention_limit(conn, EVENT_RETENTION_DAYS, startup_prune_limit); + result.events_pruned += prune_event_type_caps_with_limit(conn, EVENT_TYPE_SOFT_CAPS, startup_prune_limit); + result.events_pruned += prune_nonboot_event_overflow_with_limit(conn, EVENT_NONBOOT_SOFT_KEEP_ROWS, startup_prune_limit); + result.benchmark_pruned = prune_old_benchmark_artifacts(conn, BENCHMARK_RETENTION_DAYS, allow_vacuum); result.archived_text_stripped = strip_archived_text(conn); - - // 3. Hard-expiration cleanup result.expired_pruned = prune_expired_entries(conn); - - // 4. Crystal member embedding pruning result.crystal_embeddings_pruned = prune_crystal_member_embeddings(conn); result.cluster_members_pruned = prune_orphan_cluster_members(conn); - - // 5. Feedback aggregation result.feedback_aggregated = aggregate_old_feedback(conn); - - // 6. Stale-model embedding pruning. The active embedding model can change - // (we just switched the default to BGE); embeddings tagged with retired - // model keys cannot serve any current recall and only exist to satisfy a - // potential future re-embed. Once the active model has good coverage we - // prune the rest. ~30 bytes saved per row × thousands of stale rows. result.stale_embeddings_pruned = prune_stale_embeddings(conn); - - // 7. Sparse co-occurrence pruning. Pairs seen exactly once are noise that - // never influence recall; the table is one of the largest by row count. result.co_occurrence_pruned = prune_singleton_co_occurrence(conn); - - // 7b. Re-encode pre-v0.6.0 LE-f32 embedding blobs to PQ8 in place. - // Bounded per-pass so the write lock is short; subsequent passes - // continue chipping away until every row is migrated. result.legacy_embeddings_migrated = migrate_legacy_embeddings_to_pq8(conn); - - // 8. FTS5 segment optimize. Without this the contentless FTS shadow tables - // accumulate one segment per write — for our DB that bloated - // `decisions_fts_data` to >300MB despite only ~640 source rows. Running - // FTS5 'optimize' merges all segments into one, recovering the bulk of - // the file size. Cheap on small N, expensive on huge N — but our N is - // small in absolute terms; the bloat is in the segment overhead. result.fts_optimized = optimize_fts_indexes(conn); - - // 9. Reclaim space checkpoint_after_compaction(conn, allow_vacuum); - // VACUUM is expensive. Use SQLite's freelist_count instead of raw delete - // volume so we only pay the cost when pages are actually reclaimable. let freelist_pages = freelist_count(conn); let total_deleted = result.events_pruned + result.benchmark_pruned @@ -295,35 +166,17 @@ pub(crate) fn run_compaction_with_options(conn: &Connection, allow_vacuum: bool) if allow_vacuum && (freelist_pages > VACUUM_FREELIST_THRESHOLD_PAGES || result.fts_optimized) { let _ = conn.execute_batch("VACUUM;"); } - result.bytes_after = db_size_bytes(conn); - if total_deleted > 0 || result.fts_optimized { let saved_kb = (result.bytes_before - result.bytes_after) / 1024; eprintln!( - "[compaction] Pruned: {} events, {} benchmark rows, {} archived texts, {} expired rows, {} crystal embeddings, {} orphan cluster members, {} feedback rows, {} stale embeddings, {} singleton co-occurrence pairs, {} legacy embeddings migrated; fts_optimized={}. Saved {}KB", - result.events_pruned, - result.benchmark_pruned, - result.archived_text_stripped, - result.expired_pruned, - result.crystal_embeddings_pruned, - result.cluster_members_pruned, - result.feedback_aggregated, - result.stale_embeddings_pruned, - result.co_occurrence_pruned, - result.legacy_embeddings_migrated, - result.fts_optimized, - saved_kb - ); +"[compaction] Pruned: {} events, {} benchmark rows, {} archived texts, {} expired rows, {} crystal embeddings, {} orphan cluster members, {} feedback rows, {} stale embeddings, {} singleton co-occurrence pairs, {} legacy embeddings migrated; fts_optimized={}. Saved {}KB" +,result.events_pruned,result.benchmark_pruned,result.archived_text_stripped,result.expired_pruned,result.crystal_embeddings_pruned +,result.cluster_members_pruned,result.feedback_aggregated,result.stale_embeddings_pruned,result.co_occurrence_pruned,result. +legacy_embeddings_migrated,result.fts_optimized,saved_kb); } - result } - -/// Run FTS5 'optimize' on every contentless FTS shadow table. This collapses -/// the per-write segment list into a single merged segment, recovering the -/// dominant share of bytes in heavily-used databases. Returns true iff at -/// least one table was optimized successfully. pub(crate) fn optimize_fts_indexes(conn: &Connection) -> bool { let tables = ["decisions_fts", "memories_fts"]; let mut any = false; @@ -331,8 +184,6 @@ pub(crate) fn optimize_fts_indexes(conn: &Connection) -> bool { if !table_exists(conn, table) { continue; } - // FTS5 optimize is invoked via a no-op insert with a special command - // payload. Errors here should not abort the whole compaction pass. let sql = format!("INSERT INTO {table}({table}) VALUES ('optimize')"); match conn.execute_batch(&sql) { Ok(()) => { @@ -345,90 +196,36 @@ pub(crate) fn optimize_fts_indexes(conn: &Connection) -> bool { } any } - pub(crate) fn table_exists(conn: &Connection, name: &str) -> bool { - conn.query_row( - "SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?1", - params![name], - |_| Ok(()), - ) - .is_ok() + conn.query_row("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?1", params![name], |_| Ok(())) + .is_ok() } - -/// Delete embeddings whose `model` column does not match the currently -/// selected embedding model. Stale-model rows cannot satisfy any active -/// recall and only persist to support potential re-embeds; once the active -/// model has produced coverage the legacy rows are pure dead weight. -/// -/// Comparison is case-insensitive: legacy rows in the wild use mixed casings -/// of the same model key ("all-MiniLM-L6-v2" vs "all-minilm-l6-v2"). NULL -/// model rows are also pruned — they predate model tagging entirely and have -/// no way to match any current model. pub(crate) fn prune_stale_embeddings(conn: &Connection) -> usize { let active = crate::embeddings::selected_model_key().to_ascii_lowercase(); - // Guardrail: only prune if the active model has at least some coverage. - // Otherwise we'd torch every embedding on a fresh model switch before the - // backfill has a chance to populate replacements. let active_count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM embeddings WHERE LOWER(model) = ?1", - params![active], - |row| row.get(0), - ) + .query_row("SELECT COUNT(*) FROM embeddings WHERE LOWER(model) = ?1", params![active], |row| row.get(0)) .unwrap_or(0); if active_count < 50 { return 0; } - conn.execute( - "DELETE FROM embeddings WHERE model IS NULL OR LOWER(model) != ?1", - params![active], - ) - .unwrap_or(0) + conn.execute("DELETE FROM embeddings WHERE model IS NULL OR LOWER(model) != ?1", params![active]).unwrap_or(0) } - -/// Delete co-occurrence pairs that have only ever been observed once. They -/// contribute no signal to ranking and dominate the row count; for our DB the -/// pruned set is typically >50% of the table. pub(crate) fn prune_singleton_co_occurrence(conn: &Connection) -> usize { if !table_exists(conn, "co_occurrence") { return 0; } - conn.execute("DELETE FROM co_occurrence WHERE \"count\" <= 1", []) - .unwrap_or(0) + conn.execute("DELETE FROM co_occurrence WHERE \"count\" <= 1", []).unwrap_or(0) } - -/// Re-encode any legacy LE-f32 embedding blobs to the PQ8 format. New writes -/// always use PQ8, but pre-v0.6.0 rows still hold f32 blobs at 3072 bytes -/// each (BGE-768) — re-encoding them in place reclaims ~75% of their size -/// without changing recall semantics. Bounded to a safety cap per pass so -/// the write lock is never held for long. pub(crate) const PQ8_MIGRATION_BATCH: usize = 1024; - pub(crate) fn migrate_legacy_embeddings_to_pq8(conn: &Connection) -> usize { let from_embeddings = migrate_legacy_blob_column_to_pq8(conn, "embeddings", "vector", "id"); - // Crystal centroids dominate `memory_clusters` size when they are still - // in the legacy f32 format. Same migration logic — different table. - let from_clusters = - migrate_legacy_blob_column_to_pq8(conn, "memory_clusters", "centroid", "id"); + let from_clusters = migrate_legacy_blob_column_to_pq8(conn, "memory_clusters", "centroid", "id"); from_embeddings + from_clusters } - -pub(crate) fn migrate_legacy_blob_column_to_pq8( - conn: &Connection, - table: &str, - column: &str, - pk_column: &str, -) -> usize { +pub(crate) fn migrate_legacy_blob_column_to_pq8(conn: &Connection, table: &str, column: &str, pk_column: &str) -> usize { if !table_exists(conn, table) { return 0; } - // Find legacy blobs. A single-byte magic check has a 1/256 false-positive - // rate against legacy LE-f32 blobs — observed in practice when the very - // first f32 happens to encode a value whose low byte is 0xC8. Combine - // both magic and version byte (2-byte signature) to eliminate that. We - // also gate by length divisibility: legacy blobs are always 4*D bytes - // (multiple of 4); PQ8 blobs are D+6 bytes which is never a multiple of - // 4 for any D where D % 4 == 0 (true for every embedding model we ship). let select_sql = format!( "SELECT \"{pk}\", \"{col}\" FROM \"{tbl}\" \ WHERE \"{col}\" IS NOT NULL \ @@ -446,39 +243,22 @@ pub(crate) fn migrate_legacy_blob_column_to_pq8( return 0; } }; - let magic_signature = vec![ - crate::embeddings::PQ8_MAGIC_BYTE, - crate::embeddings::PQ8_FORMAT_VERSION, - ]; - - // First gather candidates to avoid mutating during iteration. - let candidates: Vec<(i64, Vec)> = match stmt.query_map( - params![magic_signature, PQ8_MIGRATION_BATCH as i64], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)), - ) { - Ok(rows) => rows.flatten().collect(), - Err(err) => { - eprintln!("[compaction] PQ8 migration query failed for {table}.{column}: {err}"); - return 0; - } - }; + let magic_signature = vec![crate::embeddings::PQ8_MAGIC_BYTE, crate::embeddings::PQ8_FORMAT_VERSION]; + let candidates: Vec<(i64, Vec)> = + match stmt.query_map(params![magic_signature, PQ8_MIGRATION_BATCH as i64], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?))) { + Ok(rows) => rows.flatten().collect(), + Err(err) => { + eprintln!("[compaction] PQ8 migration query failed for {table}.{column}: {err}"); + return 0; + } + }; drop(stmt); - if candidates.is_empty() { return 0; } - - let update_sql = format!( - "UPDATE \"{tbl}\" SET \"{col}\" = ?1 WHERE \"{pk}\" = ?2", - pk = pk_column, - col = column, - tbl = table, - ); + let update_sql = format!("UPDATE \"{tbl}\" SET \"{col}\" = ?1 WHERE \"{pk}\" = ?2", pk = pk_column, col = column, tbl = table,); let mut migrated = 0usize; for (id, blob) in candidates { - // Decode the legacy blob, then re-encode via the canonical PQ8 path. - // If decoding produces an empty vector the row is corrupt; skip it - // rather than silently writing a zero-length PQ8 blob. let decoded = crate::embeddings::legacy_f32_blob_to_vector(&blob); if decoded.is_empty() { continue; @@ -487,18 +267,12 @@ pub(crate) fn migrate_legacy_blob_column_to_pq8( match conn.execute(&update_sql, params![pq8, id]) { Ok(_) => migrated += 1, Err(err) => { - eprintln!( - "[compaction] PQ8 migration update failed for {table}.{column} id={id}: {err}" - ); + eprintln!("[compaction] PQ8 migration update failed for {table}.{column} id={id}: {err}"); } } } migrated } - -/// Purge all benchmark artifacts immediately. -/// Use this after benchmark runs so production DB stats reflect real-user traffic. pub fn purge_benchmark_artifacts(conn: &Connection) -> BenchmarkPurgeResult { purge_benchmark_artifacts_with_retention(conn, None, true) } - diff --git a/daemon-rs/src/compaction/helpers.rs b/daemon-rs/src/compaction/helpers.rs index 28eebe53..2497f47c 100644 --- a/daemon-rs/src/compaction/helpers.rs +++ b/daemon-rs/src/compaction/helpers.rs @@ -1,37 +1,16 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - -use super::*; -// ─── Helpers ──────────────────────────────────────────────────────────────── - +use rusqlite::Connection; pub(crate) fn db_size_bytes(conn: &Connection) -> i64 { - let page_count: i64 = conn - .query_row("PRAGMA page_count", [], |row| row.get(0)) - .unwrap_or(0); - let page_size: i64 = conn - .query_row("PRAGMA page_size", [], |row| row.get(0)) - .unwrap_or(4096); + let page_count: i64 = conn.query_row("PRAGMA page_count", [], |row| row.get(0)).unwrap_or(0); + let page_size: i64 = conn.query_row("PRAGMA page_size", [], |row| row.get(0)).unwrap_or(4096); page_count * page_size } - pub(crate) fn freelist_count(conn: &Connection) -> i64 { - conn.query_row("PRAGMA freelist_count", [], |row| row.get(0)) - .unwrap_or(0) + conn.query_row("PRAGMA freelist_count", [], |row| row.get(0)).unwrap_or(0) } - pub(crate) fn non_boot_event_count(conn: &Connection) -> i64 { - conn.query_row( - "SELECT COUNT(*) FROM events WHERE type NOT IN ('boot_savings', 'boot_savings_rollup')", - [], - |row| row.get(0), - ) - .unwrap_or(0) + conn.query_row("SELECT COUNT(*) FROM events WHERE type NOT IN ('boot_savings', 'boot_savings_rollup')", [], |row| row.get(0)) + .unwrap_or(0) } - -/// Get storage breakdown by table (for diagnostics). pub fn storage_breakdown(conn: &Connection) -> Vec<(String, i64)> { let tables = [ "memories", @@ -46,17 +25,10 @@ pub fn storage_breakdown(conn: &Connection) -> Vec<(String, i64)> { "context_cache", "feed", ]; - let mut breakdown = Vec::new(); for table in &tables { - // Approximate row size * count - let count: i64 = conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap_or(0); + let count: i64 = conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row.get(0)).unwrap_or(0); breakdown.push((table.to_string(), count)); } breakdown } - diff --git a/daemon-rs/src/compaction/mod.rs b/daemon-rs/src/compaction/mod.rs index 21de10a2..aca750d6 100644 --- a/daemon-rs/src/compaction/mod.rs +++ b/daemon-rs/src/compaction/mod.rs @@ -1,27 +1,18 @@ -// SPDX-License-Identifier: MIT -mod types; -mod governor; -mod events; mod archived; mod crystals; +mod events; mod feedback; +mod governor; mod helpers; - #[cfg(test)] mod tests; - -pub(crate) use types::*; -pub(crate) use governor::*; -pub(crate) use events::*; +mod types; pub(crate) use archived::*; pub(crate) use crystals::*; +pub(crate) use events::*; pub(crate) use feedback::*; -pub(crate) use helpers::*; - -pub use types::*; -pub use governor::{ - should_run_compaction_governor, run_compaction_governor, run_compaction_governor_startup, - fts_segment_row_total, FTS_SEGMENT_ROW_SOFT_LIMIT, run_compaction, CompactionResult, - purge_benchmark_artifacts, BenchmarkPurgeResult, -}; +pub(crate) use governor::*; +pub use governor::{purge_benchmark_artifacts, run_compaction, run_compaction_governor, run_compaction_governor_startup, BenchmarkPurgeResult}; pub use helpers::storage_breakdown; +pub(crate) use helpers::*; +pub(crate) use types::*; diff --git a/daemon-rs/src/compaction/tests.rs b/daemon-rs/src/compaction/tests/mod.rs similarity index 76% rename from daemon-rs/src/compaction/tests.rs rename to daemon-rs/src/compaction/tests/mod.rs index 762fee0d..2506c658 100644 --- a/daemon-rs/src/compaction/tests.rs +++ b/daemon-rs/src/compaction/tests/mod.rs @@ -1,13 +1,8 @@ // SPDX-License-Identifier: MIT -//! Data-integrity tests for compaction only. See Info/testing-philosophy.md. - #[cfg(test)] mod tests { - use crate::compaction::{ - prune_expired_entries, prune_old_events, purge_benchmark_artifacts, - }; + use crate::compaction::{prune_expired_entries, prune_old_events, purge_benchmark_artifacts}; use rusqlite::Connection; - fn setup() -> Connection { let conn = Connection::open_in_memory().unwrap(); crate::db::configure(&conn).unwrap(); @@ -15,24 +10,15 @@ mod tests { crate::db::run_pending_migrations(&conn); conn } - #[test] fn prune_old_events_removes_stale_rows() { let conn = setup(); - conn.execute( - "INSERT INTO events (type, data, created_at) VALUES ('boot', '{}', datetime('now', '-40 days'))", - [], - ) - .unwrap(); - conn.execute( - "INSERT INTO events (type, data, created_at) VALUES ('boot', '{}', datetime('now'))", - [], - ) - .unwrap(); + conn.execute("INSERT INTO events (type, data, created_at) VALUES ('boot', '{}', datetime('now', '-40 days'))", []) + .unwrap(); + conn.execute("INSERT INTO events (type, data, created_at) VALUES ('boot', '{}', datetime('now'))", []).unwrap(); let removed = prune_old_events(&conn); assert_eq!(removed, 1); } - #[test] fn prune_expired_entries_removes_expired_decisions() { let conn = setup(); @@ -51,7 +37,6 @@ mod tests { let removed = prune_expired_entries(&conn); assert_eq!(removed, 1); } - #[test] fn purge_benchmark_artifacts_removes_benchmark_rows() { let conn = setup(); diff --git a/daemon-rs/src/compaction/types.rs b/daemon-rs/src/compaction/types.rs index 97ff1e91..a234c630 100644 --- a/daemon-rs/src/compaction/types.rs +++ b/daemon-rs/src/compaction/types.rs @@ -1,75 +1,26 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::collections::HashMap; - - -use super::*; -// ─── Constants ────────────────────────────────────────────────────────────── - -/// Non-boot events older than this are deleted. pub(crate) const EVENT_RETENTION_DAYS: i64 = 14; - -/// Raw boot savings rows older than this are compacted into a single rollup row. -/// The dashboard only needs recent raw points, while all-time totals are preserved -/// via `boot_savings_rollup`. pub(crate) const BOOT_SAVINGS_RETENTION_DAYS: i64 = 45; - -/// Only VACUUM when SQLite reports enough reclaimable pages to justify the IO. pub(crate) const VACUUM_FREELIST_THRESHOLD_PAGES: i64 = 100; - -/// Archived entries older than this have their text stripped (metadata kept). pub(crate) const ARCHIVED_TEXT_RETENTION_DAYS: i64 = 90; - -/// Feedback signals older than this are aggregated into summaries. pub(crate) const FEEDBACK_AGGREGATION_DAYS: i64 = 60; - -/// Roll analytics-heavy savings events older than this into compact hourly rows. pub(crate) const SAVINGS_EVENT_ROLLUP_RETENTION_DAYS: i64 = 7; -/// Keep rolled-up savings analytics bounded; /savings only reads the recent window. pub(crate) const EVENT_SAVINGS_ROLLUP_RETENTION_DAYS: i64 = 120; - -/// Elevated-pressure storage governor soft limit (no hard failures, compaction only). -pub const STORAGE_SOFT_LIMIT_BYTES: i64 = 256 * 1024 * 1024; // 256MB -/// Critical-pressure storage governor hard limit (triggers aggressive safe compaction). -pub const STORAGE_HARD_LIMIT_BYTES: i64 = 512 * 1024 * 1024; // 512MB - -/// Under critical pressure, compact events more aggressively. +pub const STORAGE_SOFT_LIMIT_BYTES: i64 = 256 * 1024 * 1024; +pub const STORAGE_HARD_LIMIT_BYTES: i64 = 512 * 1024 * 1024; pub(crate) const AGGRESSIVE_EVENT_RETENTION_DAYS: i64 = 3; -/// Under critical pressure, compact boot savings history more aggressively. pub(crate) const AGGRESSIVE_BOOT_SAVINGS_RETENTION_DAYS: i64 = 14; -/// Under critical pressure, compact archived text sooner. pub(crate) const AGGRESSIVE_ARCHIVED_TEXT_RETENTION_DAYS: i64 = 30; -/// Under critical pressure, aggregate feedback sooner. pub(crate) const AGGRESSIVE_FEEDBACK_AGGREGATION_DAYS: i64 = 14; -/// Under critical pressure, roll savings events even sooner. pub(crate) const AGGRESSIVE_SAVINGS_EVENT_ROLLUP_RETENTION_DAYS: i64 = 2; -/// Under critical pressure, keep a shorter event rollup history to reclaim space faster. pub(crate) const AGGRESSIVE_EVENT_SAVINGS_ROLLUP_RETENTION_DAYS: i64 = 45; -/// Keep benchmark artifacts only briefly in production databases. pub(crate) const BENCHMARK_RETENTION_DAYS: i64 = 2; -/// Tighten benchmark retention further under critical pressure. pub(crate) const AGGRESSIVE_BENCHMARK_RETENTION_DAYS: i64 = 1; - -/// Canonical source-agent prefix emitted by benchmark harnesses. -/// -/// Keep this broad enough to match both modern namespaced agents -/// (`amb-cortex::`) and legacy plain labels (`amb-cortex`). pub const BENCHMARK_SOURCE_AGENT_PREFIX: &str = "amb-cortex"; - -/// Non-boot event volume triggers compaction even when DB file size is moderate. pub const EVENT_NONBOOT_SOFT_LIMIT_ROWS: i64 = 72_000; -/// Critical non-boot event pressure threshold. pub const EVENT_NONBOOT_HARD_LIMIT_ROWS: i64 = 120_000; -/// Keep newest non-boot rows at or under this level during normal governor runs. pub(crate) const EVENT_NONBOOT_SOFT_KEEP_ROWS: i64 = 52_000; -/// Keep newest non-boot rows at or under this level during critical pressure runs. pub(crate) const EVENT_NONBOOT_HARD_KEEP_ROWS: i64 = 28_000; -/// Startup governor mode should avoid single huge DELETE statements that hold -/// the write lock for too long while the daemon is still coming online. pub(crate) const STARTUP_EVENT_PRUNE_BATCH_ROWS: i64 = 8_000; - -/// Per-event-type row caps to prevent high-frequency streams from dominating storage. pub(crate) const EVENT_TYPE_SOFT_CAPS: &[(&str, i64)] = &[ ("agent_boot", 4_000), ("boot_savings", 6_000), @@ -88,8 +39,6 @@ pub(crate) const EVENT_TYPE_SOFT_CAPS: &[(&str, i64)] = &[ ("forget", 3_000), ("diary_write", 3_000), ]; - -/// More aggressive caps used under critical pressure. pub(crate) const EVENT_TYPE_HARD_CAPS: &[(&str, i64)] = &[ ("agent_boot", 1_500), ("boot_savings", 2_500), @@ -108,4 +57,3 @@ pub(crate) const EVENT_TYPE_HARD_CAPS: &[(&str, i64)] = &[ ("forget", 1_000), ("diary_write", 1_000), ]; - diff --git a/daemon-rs/src/compiler/cache.rs b/daemon-rs/src/compiler/cache.rs index 800177ac..5f921a98 100644 --- a/daemon-rs/src/compiler/cache.rs +++ b/daemon-rs/src/compiler/cache.rs @@ -1,23 +1,7 @@ -// SPDX-License-Identifier: MIT -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; -use rusqlite::{params, Connection, OptionalExtension}; -use serde_json::{json, Value}; -use std::collections::HashSet; -use std::env; -use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - use super::*; -// ─── Token estimation ─────────────────────────────────────────────────────── - -// Estimate tokens from character length (~3.8 chars/token, matching Node.js). -// ─── Content-addressed cache ──────────────────────────────────────────────── - -/// Compute a fast content hash for cache invalidation. -/// Uses FNV-1a for speed (not crypto-secure, just change detection). +use crate::handlers::estimate_tokens; +use regex::Regex; +use rusqlite::{params, Connection}; pub(crate) fn content_hash(data: &str) -> String { let mut hash: u64 = 0xcbf29ce484222325; for byte in data.bytes() { @@ -26,35 +10,23 @@ pub(crate) fn content_hash(data: &str) -> String { } format!("{hash:016x}") } - -/// Check the context cache for a cached result. pub(crate) fn cache_get(conn: &Connection, key: &str, expected_hash: &str) -> Option<(String, usize)> { - conn.query_row( - "SELECT compressed, tokens, content_hash FROM context_cache WHERE cache_key = ?1", - params![key], - |row| { - let compressed: String = row.get(0)?; - let tokens: usize = row.get::<_, i64>(1)? as usize; - let stored_hash: String = row.get(2)?; - Ok((compressed, tokens, stored_hash)) - }, - ) + conn.query_row("SELECT compressed, tokens, content_hash FROM context_cache WHERE cache_key = ?1", params![key], |row| { + let compressed: String = row.get(0)?; + let tokens: usize = row.get::<_, i64>(1)? as usize; + let stored_hash: String = row.get(2)?; + Ok((compressed, tokens, stored_hash)) + }) .ok() .and_then(|(compressed, tokens, stored_hash)| { if stored_hash == expected_hash { - // Cache hit -- bump hit count - let _ = conn.execute( - "UPDATE context_cache SET hits = hits + 1 WHERE cache_key = ?1", - params![key], - ); + let _ = conn.execute("UPDATE context_cache SET hits = hits + 1 WHERE cache_key = ?1", params![key]); Some((compressed, tokens)) } else { - None // Hash mismatch -- content changed + None } }) } - -/// Store a compiled result in the cache. pub(crate) fn cache_set(conn: &Connection, key: &str, hash: &str, compressed: &str, tokens: usize) { let _ = conn.execute( "INSERT OR REPLACE INTO context_cache (cache_key, content_hash, compressed, tokens) \ @@ -62,21 +34,10 @@ pub(crate) fn cache_set(conn: &Connection, key: &str, hash: &str, compressed: &s params![key, hash, compressed, tokens as i64], ); } - -// State.md helpers removed — session-auto-restore.js handles state.md injection. - -// ─── Identity capsule ─────────────────────────────────────────────────────── - -/// Build the identity capsule — stable across sessions, ~200 tokens. -/// Contains core user identity, hard constraints, and platform sharp edges. -/// Uses content-addressed cache: if feedback memories haven't changed, reuse. pub(crate) fn build_identity_capsule(conn: &Connection) -> (String, usize) { - // Compute hash of the feedback memories that feed this capsule let feedback_hash = { let mut all_feedback = String::new(); - if let Ok(mut stmt) = conn.prepare( - "SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY id", - ) { + if let Ok(mut stmt) = conn.prepare("SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY id") { if let Ok(rows) = stmt.query_map([], |r| r.get::<_, String>(0)) { for text in rows.flatten() { all_feedback.push_str(&text); @@ -86,20 +47,12 @@ pub(crate) fn build_identity_capsule(conn: &Connection) -> (String, usize) { } content_hash(&all_feedback) }; - - // Check cache if let Some((cached, tokens)) = cache_get(conn, "identity_capsule", &feedback_hash) { return (cached, tokens); } let mut parts = vec![detect_identity()]; - - // Hard constraints (never/always/must rules) - if let Ok(constraint_re) = - Regex::new(r"(?i)\b(never|always|must|do not|don't|required|mandatory)\b") - { - if let Ok(mut stmt) = conn.prepare( - "SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY score DESC LIMIT 20", - ) { + if let Ok(constraint_re) = Regex::new(r"(?i)\b(never|always|must|do not|don't|required|mandatory)\b") { + if let Ok(mut stmt) = conn.prepare("SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY score DESC LIMIT 20") { if let Ok(rows) = stmt.query_map([], |r| r.get::<_, String>(0)) { let constraints: Vec = rows .filter_map(|r| r.ok()) @@ -113,12 +66,8 @@ pub(crate) fn build_identity_capsule(conn: &Connection) -> (String, usize) { } } } - - // Platform sharp edges (Windows-specific gotchas) if let Ok(edge_re) = Regex::new(r"(?i)\b(windows|win32|encoding|cp1252|bash\.exe|CRLF)\b") { - if let Ok(mut stmt) = conn.prepare( - "SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY score DESC LIMIT 20", - ) { + if let Ok(mut stmt) = conn.prepare("SELECT text FROM memories WHERE type = 'feedback' AND status = 'active' ORDER BY score DESC LIMIT 20") { if let Ok(rows) = stmt.query_map([], |r| r.get::<_, String>(0)) { let edges: Vec = rows .filter_map(|r| r.ok()) @@ -132,13 +81,8 @@ pub(crate) fn build_identity_capsule(conn: &Connection) -> (String, usize) { } } } - let text = parts.join("\n"); let tokens = estimate_tokens(&text); - - // Cache the result for next boot cache_set(conn, "identity_capsule", &feedback_hash, &text, tokens); - (text, tokens) } - diff --git a/daemon-rs/src/compiler/capsules.rs b/daemon-rs/src/compiler/capsules.rs index 86c3f9c6..0b522ec2 100644 --- a/daemon-rs/src/compiler/capsules.rs +++ b/daemon-rs/src/compiler/capsules.rs @@ -1,47 +1,19 @@ -// SPDX-License-Identifier: MIT -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; +use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; use rusqlite::{params, Connection, OptionalExtension}; use serde_json::{json, Value}; use std::collections::HashSet; -use std::env; use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - -use super::*; -// ─── Last boot time ───────────────────────────────────────────────────────── - pub(crate) fn get_last_boot_time(conn: &Connection, agent: &str) -> Option { - conn.query_row( - "SELECT data FROM events WHERE type = 'agent_boot' AND source_agent = ?1 ORDER BY created_at DESC LIMIT 1", - params![agent], - |r| r.get::<_, String>(0), - ) - .ok() - .and_then(|data| { - serde_json::from_str::(&data) - .ok()? - .get("timestamp")? - .as_str() - .map(|s| s.to_string()) + conn.query_row("SELECT data FROM events WHERE type = 'agent_boot' AND source_agent = ?1 ORDER BY created_at DESC LIMIT 1", params![agent], |r| { + r.get::<_, String>(0) }) + .ok() + .and_then(|data| serde_json::from_str::(&data).ok()?.get("timestamp")?.as_str().map(|s| s.to_string())) } - -// ─── Conductor state helpers ──────────────────────────────────────────────── - pub(crate) fn fetch_messages_for_agent(conn: &Connection, agent: &str) -> Vec { let mut out = Vec::new(); - if let Ok(mut stmt) = conn - .prepare("SELECT sender, message FROM messages WHERE recipient = ?1 ORDER BY timestamp ASC") - { - if let Ok(rows) = stmt.query_map(params![agent], |r| { - Ok(json!({ - "from": r.get::<_, String>(0)?, - "message": r.get::<_, String>(1)? - })) - }) { + if let Ok(mut stmt) = conn.prepare("SELECT sender, message FROM messages WHERE recipient = ?1 ORDER BY timestamp ASC") { + if let Ok(rows) = stmt.query_map(params![agent], |r| Ok(json!({"from":r.get::<_,String>(0)?,"message":r.get::<_,String>(1)?}))) { for row in rows.flatten() { out.push(row); } @@ -49,21 +21,15 @@ pub(crate) fn fetch_messages_for_agent(conn: &Connection, agent: &str) -> Vec Vec { let mut out = Vec::new(); - if let Ok(mut stmt) = conn.prepare( - "SELECT agent, project, description, files_json FROM sessions WHERE expires_at > ?1", - ) { + if let Ok(mut stmt) = conn.prepare("SELECT agent, project, description, files_json FROM sessions WHERE expires_at > ?1") { let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); if let Ok(rows) = stmt.query_map(params![now], |r| { let files_json: String = r.get(3)?; Ok(json!({ - "agent": r.get::<_, String>(0)?, - "project": r.get::<_, Option>(1)?, - "description": r.get::<_, Option>(2)?, - "files": serde_json::from_str::(&files_json).unwrap_or(json!([])) - })) +"agent":r.get::<_,String>(0)?,"project":r.get::<_,Option>(1)?,"description":r.get::<_,Option>(2)?,"files": +serde_json::from_str::(&files_json).unwrap_or(json!([]))})) }) { for row in rows.flatten() { out.push(row); @@ -72,19 +38,13 @@ pub(crate) fn fetch_sessions(conn: &Connection) -> Vec { } out } - pub(crate) fn fetch_locks(conn: &Connection) -> Vec { let mut out = Vec::new(); let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); - if let Ok(mut stmt) = - conn.prepare("SELECT path, agent, expires_at FROM locks WHERE expires_at > ?1") - { + if let Ok(mut stmt) = conn.prepare("SELECT path, agent, expires_at FROM locks WHERE expires_at > ?1") { if let Ok(rows) = stmt.query_map(params![now], |r| { - Ok(json!({ - "path": r.get::<_, String>(0)?, - "agent": r.get::<_, String>(1)?, - "expiresAt": r.get::<_, String>(2)? - })) + Ok(json!({"path":r.get::<_,String>(0)?,"agent":r.get::<_,String>(1)?,"expiresAt":r.get::<_,String +>(2)?})) }) { for row in rows.flatten() { out.push(row); @@ -93,36 +53,20 @@ pub(crate) fn fetch_locks(conn: &Connection) -> Vec { } out } - pub(crate) fn fetch_unread_feed(conn: &Connection, agent: &str) -> Vec { let ack: Option = conn - .query_row( - "SELECT last_seen_id FROM feed_acks WHERE agent = ?1", - params![agent], - |row| row.get(0), - ) + .query_row("SELECT last_seen_id FROM feed_acks WHERE agent = ?1", params![agent], |row| row.get(0)) .optional() .ok() .flatten(); - let mut all: Vec<(String, String, String, String)> = Vec::new(); - if let Ok(mut stmt) = - conn.prepare("SELECT id, agent, kind, summary FROM feed ORDER BY timestamp ASC") - { - if let Ok(rows) = stmt.query_map([], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - r.get::<_, String>(3)?, - )) - }) { + if let Ok(mut stmt) = conn.prepare("SELECT id, agent, kind, summary FROM feed ORDER BY timestamp ASC") { + if let Ok(rows) = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?))) { for row in rows.flatten() { all.push(row); } } } - if let Some(ack_id) = ack { let mut past_ack = false; let mut unread = Vec::new(); @@ -132,43 +76,30 @@ pub(crate) fn fetch_unread_feed(conn: &Connection, agent: &str) -> Vec { continue; } if past_ack && entry_agent != agent { - unread.push(json!({ - "kind": kind, - "agent": entry_agent, - "summary": summary - })); + unread.push(json!({"kind":kind,"agent":entry_agent,"summary":summary})); } } unread } else { - // No ack — all entries from other agents are unread all.into_iter() .filter(|(_, entry_agent, _, _)| entry_agent != agent) .map(|(_, entry_agent, kind, summary)| { - json!({ - "kind": kind, - "agent": entry_agent, - "summary": summary - }) + json!({"kind":kind,"agent": +entry_agent,"summary":summary}) }) .collect() } } - pub(crate) fn fetch_pending_tasks(conn: &Connection) -> Vec { let mut out = Vec::new(); - if let Ok(mut stmt) = conn.prepare( - "SELECT task_id, title, priority, project, files_json FROM tasks WHERE status = 'pending' ORDER BY created_at ASC LIMIT 5", - ) { + if let Ok(mut stmt) = + conn.prepare("SELECT task_id, title, priority, project, files_json FROM tasks WHERE status = 'pending' ORDER BY created_at ASC LIMIT 5") + { if let Ok(rows) = stmt.query_map([], |r| { let files_json: String = r.get(4)?; - Ok(json!({ - "id": r.get::<_, String>(0)?, - "title": r.get::<_, String>(1)?, - "priority": r.get::<_, String>(2)?, - "project": r.get::<_, Option>(3)?, - "files": serde_json::from_str::(&files_json).unwrap_or(json!([])) - })) + Ok(json!({"id":r.get::<_,String>(0)?,"title":r.get::<_,String>(1)?, +"priority":r.get::<_,String>(2)?,"project":r.get::<_,Option>(3)?,"files":serde_json::from_str::(&files_json). +unwrap_or(json!([]))})) }) { for row in rows.flatten() { out.push(row); @@ -177,19 +108,14 @@ pub(crate) fn fetch_pending_tasks(conn: &Connection) -> Vec { } out } - pub(crate) fn fetch_claimed_tasks_for_agent(conn: &Connection, agent: &str) -> Vec { let mut out = Vec::new(); - if let Ok(mut stmt) = conn.prepare( - "SELECT task_id, title, priority, claimed_at FROM tasks WHERE status = 'claimed' AND claimed_by = ?1 ORDER BY claimed_at ASC", - ) { + if let Ok(mut stmt) = + conn.prepare("SELECT task_id, title, priority, claimed_at FROM tasks WHERE status = 'claimed' AND claimed_by = ?1 ORDER BY claimed_at ASC") + { if let Ok(rows) = stmt.query_map(params![agent], |r| { - Ok(json!({ - "id": r.get::<_, String>(0)?, - "title": r.get::<_, String>(1)?, - "priority": r.get::<_, String>(2)?, - "claimedAt": r.get::<_, Option>(3)? - })) + Ok(json!({"id":r.get::<_,String>(0)?,"title":r.get::<_,String>(1)?,"priority":r.get +::<_,String>(2)?,"claimedAt":r.get::<_,Option>(3)?})) }) { for row in rows.flatten() { out.push(row); @@ -198,16 +124,9 @@ pub(crate) fn fetch_claimed_tasks_for_agent(conn: &Connection, agent: &str) -> V } out } - -// ─── Delta capsule ────────────────────────────────────────────────────────── - -/// Build the delta capsule — what changed since the agent's last boot. -/// High relevance, changes every session. Target: ~300 tokens. pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, usize, String) { let last_boot = get_last_boot_time(conn, agent); let mut parts: Vec = Vec::new(); - - // 0. Pending messages (highest priority) let messages = fetch_messages_for_agent(conn, agent); if !messages.is_empty() { let lines: Vec = messages @@ -221,33 +140,20 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .collect(); parts.push(format!("## Pending Messages\n{}", lines.join("\n"))); } - - // 0b. Active agents (session bus — who else is online) let sessions = fetch_sessions(conn); - let other_sessions: Vec<&Value> = sessions - .iter() - .filter(|s| s.get("agent").and_then(|v| v.as_str()) != Some(agent)) - .collect(); + let other_sessions: Vec<&Value> = sessions.iter().filter(|s| s.get("agent").and_then(|v| v.as_str()) != Some(agent)).collect(); if !other_sessions.is_empty() { let lines: Vec = other_sessions .iter() .map(|s| { let ag = s.get("agent").and_then(|v| v.as_str()).unwrap_or("?"); - let proj = s - .get("project") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - let desc = s - .get("description") - .and_then(|v| v.as_str()) - .unwrap_or("no description"); + let proj = s.get("project").and_then(|v| v.as_str()).unwrap_or("unknown"); + let desc = s.get("description").and_then(|v| v.as_str()).unwrap_or("no description"); format!("- {ag} working on {proj}: \"{desc}\"") }) .collect(); parts.push(format!("## Active Agents\n{}", lines.join("\n"))); } - - // 0c. Active locks let locks = fetch_locks(conn); if !locks.is_empty() { let lines: Vec = locks @@ -260,8 +166,6 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .collect(); parts.push(format!("## Active Locks\n{}", lines.join("\n"))); } - - // 0d. Shared feed (unread entries from other agents) let mut feed = fetch_unread_feed(conn, agent); if feed.len() > 10 { feed = feed.split_off(feed.len() - 10); @@ -278,8 +182,6 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .collect(); parts.push(format!("## Feed\n{}", lines.join("\n"))); } - - // 0e. Task board (pending tasks + agent's claimed tasks) let pending_tasks = fetch_pending_tasks(conn); if !pending_tasks.is_empty() { let lines: Vec = pending_tasks @@ -292,7 +194,6 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .collect(); parts.push(format!("## Pending Tasks\n{}", lines.join("\n"))); } - let my_tasks = fetch_claimed_tasks_for_agent(conn, agent); if !my_tasks.is_empty() { let lines: Vec = my_tasks @@ -305,19 +206,10 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .collect(); parts.push(format!("## Your Active Tasks\n{}", lines.join("\n"))); } - - // 1. Open conflicts (always include — highest priority) - if let Ok(mut stmt) = conn.prepare( - "SELECT id, decision, source_agent, disputes_id FROM decisions WHERE status = 'disputed' ORDER BY created_at DESC LIMIT 6", - ) { - if let Ok(rows) = stmt.query_map([], |r| { - Ok(( - r.get::<_, i64>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - r.get::<_, Option>(3)?, - )) - }) { + if let Ok(mut stmt) = + conn.prepare("SELECT id, decision, source_agent, disputes_id FROM decisions WHERE status = 'disputed' ORDER BY created_at DESC LIMIT 6") + { + if let Ok(rows) = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option>(3)?))) { let mut seen = HashSet::new(); let mut lines: Vec = Vec::new(); for (id, decision, source_agent, disputes_id) in rows.flatten() { @@ -328,49 +220,31 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us if let Some(did) = disputes_id { seen.insert(did); } - let mut line = format!("#{id} ({source_agent}): {decision}"); if let Some(did) = disputes_id { - if let Ok((partner_dec, partner_agent)) = conn.query_row( - "SELECT decision, source_agent FROM decisions WHERE id = ?1", - params![did], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) { - line.push_str(&format!( - " vs #{did} ({partner_agent}): {partner_dec}" - )); + if let Ok((partner_dec, partner_agent)) = conn.query_row("SELECT decision, source_agent FROM decisions WHERE id = ?1", params![did], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) + }) { + line.push_str(&format!(" vs #{did} ({partner_agent}): {partner_dec}")); } } lines.push(line); } if !lines.is_empty() { - parts.push(format!( - "CONFLICTS:\n{}", - lines.iter().map(|l| format!("- {l}")).collect::>().join("\n") - )); + parts.push(format!("CONFLICTS:\n{}", lines.iter().map(|l| format!("- {l}")).collect::>().join("\n"))); } } } - - // 2. Active focus session (sawtooth pattern indicator) if let Some(focus) = crate::focus::focus_current(conn, agent) { let label = focus.get("label").and_then(|v| v.as_str()).unwrap_or("?"); let entries = focus.get("entries").and_then(|v| v.as_u64()).unwrap_or(0); parts.push(format!("## Active Focus\n- {label} ({entries} entries)")); } - - // 3. New decisions since last boot if let Some(ref lb) = last_boot { - if let Ok(mut stmt) = conn.prepare( - "SELECT decision, context, source_agent FROM decisions WHERE status = 'active' AND created_at >= ?1 ORDER BY created_at DESC LIMIT 5", - ) { - if let Ok(rows) = stmt.query_map(params![lb], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option>(1)?, - r.get::<_, String>(2)?, - )) - }) { + if let Ok(mut stmt) = + conn.prepare("SELECT decision, context, source_agent FROM decisions WHERE status = 'active' AND created_at >= ?1 ORDER BY created_at DESC LIMIT 5") + { + if let Ok(rows) = stmt.query_map(params![lb], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?, r.get::<_, String>(2)?))) { let lines: Vec = rows .flatten() .map(|(dec, ctx, ag)| { @@ -383,14 +257,10 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us } } } - - // 4. New memories since last boot - if let Ok(mut stmt) = conn.prepare( - "SELECT text, type FROM memories WHERE status = 'active' AND updated_at >= ?1 AND type != 'state' ORDER BY updated_at DESC LIMIT 3", - ) { - if let Ok(rows) = stmt.query_map(params![lb], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) - }) { + if let Ok(mut stmt) = + conn.prepare("SELECT text, type FROM memories WHERE status = 'active' AND updated_at >= ?1 AND type != 'state' ORDER BY updated_at DESC LIMIT 3") + { + if let Ok(rows) = stmt.query_map(params![lb], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) { let lines: Vec = rows .flatten() .map(|(text, mtype)| { @@ -403,33 +273,19 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us } } } - - // 5. Events since last boot (summarized) - if let Ok(mut stmt) = conn.prepare( - "SELECT type, COUNT(*) as cnt FROM events WHERE created_at > ?1 AND type NOT IN ('brain_init', 'index_all', 'agent_boot') GROUP BY type", - ) { - if let Ok(rows) = stmt.query_map(params![lb], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)) - }) { - let entries: Vec = rows - .flatten() - .map(|(etype, cnt)| { - format!("{cnt} {}", etype.replace('_', " ")) - }) - .collect(); + if let Ok(mut stmt) = conn + .prepare("SELECT type, COUNT(*) as cnt FROM events WHERE created_at > ?1 AND type NOT IN ('brain_init', 'index_all', 'agent_boot') GROUP BY type") + { + if let Ok(rows) = stmt.query_map(params![lb], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))) { + let entries: Vec = rows.flatten().map(|(etype, cnt)| format!("{cnt} {}", etype.replace('_', " "))).collect(); if !entries.is_empty() { parts.push(format!("Activity since last boot: {}", entries.join(", "))); } } } } else { - // First boot for this agent — include recent decisions as orientation - if let Ok(mut stmt) = conn.prepare( - "SELECT decision, context FROM decisions WHERE status = 'active' ORDER BY created_at DESC LIMIT 5", - ) { - if let Ok(rows) = stmt.query_map([], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)) - }) { + if let Ok(mut stmt) = conn.prepare("SELECT decision, context FROM decisions WHERE status = 'active' ORDER BY created_at DESC LIMIT 5") { + if let Ok(rows) = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?))) { let lines: Vec = rows .flatten() .map(|(dec, ctx)| { @@ -443,7 +299,6 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us } } } - let text = parts.join("\n\n"); let tokens = estimate_tokens(&text); let freshness = last_boot @@ -455,55 +310,28 @@ pub(crate) fn build_delta_capsule(conn: &Connection, agent: &str) -> (String, us .unwrap_or_else(|| "first boot".to_string()); (text, tokens, freshness) } - -// ─── Raw baseline estimation ──────────────────────────────────────────────── - -/// Estimate what raw context would cost — the baseline Cortex replaces. -/// Counts chars in the same DB-backed context families the compiler actually -/// assembles for boot prompts (active memories + active decisions). -/// -/// We intentionally do not include filesystem-size heuristics from custom -/// source directories here, because that can drastically overstate savings -/// without reflecting prompt payload reality. pub(crate) fn estimate_raw_baseline(conn: &Connection, _home: &Path) -> usize { let mut total_chars: usize = 0; - - // DB memories: active entries only let mem_chars: i64 = conn - .query_row( - "SELECT COALESCE(SUM(LENGTH(text)), 0) FROM memories WHERE status = 'active'", - [], - |r| r.get(0), - ) + .query_row("SELECT COALESCE(SUM(LENGTH(text)), 0) FROM memories WHERE status = 'active'", [], |r| r.get(0)) .unwrap_or(0); total_chars += mem_chars as usize; - - // DB decisions: active entries only let dec_chars: i64 = conn - .query_row( - "SELECT COALESCE(SUM(LENGTH(decision)), 0) FROM decisions WHERE status = 'active'", - [], - |r| r.get(0), - ) + .query_row("SELECT COALESCE(SUM(LENGTH(decision)), 0) FROM decisions WHERE status = 'active'", [], |r| r.get(0)) .unwrap_or(0); total_chars += dec_chars as usize; - estimate_tokens_from_chars(total_chars) } - -// ─── Record boot ──────────────────────────────────────────────────────────── - -/// Record this boot so the next session's delta knows when we last connected. pub(crate) fn record_boot(conn: &Connection, agent: &str) { let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); let _ = conn.execute( "INSERT INTO events (type, data, source_agent) VALUES (?1, ?2, ?3)", params![ "agent_boot", - serde_json::to_string(&json!({ "timestamp": &now, "agent": agent })) - .unwrap_or_default(), + serde_json::to_string(&json!({"timestamp" +:&now,"agent":agent})) + .unwrap_or_default(), agent ], ); } - diff --git a/daemon-rs/src/compiler/compile.rs b/daemon-rs/src/compiler/compile.rs index ee9cf8ab..353e6390 100644 --- a/daemon-rs/src/compiler/compile.rs +++ b/daemon-rs/src/compiler/compile.rs @@ -1,72 +1,38 @@ -// SPDX-License-Identifier: MIT -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; -use rusqlite::{params, Connection, OptionalExtension}; -use serde_json::{json, Value}; -use std::collections::HashSet; -use std::env; -use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - use super::*; -/// Compile the boot prompt for an agent within a token budget. -/// -/// Prompt Compiler Pipeline (v3 -- score-adaptive context packing): -/// 1. Gather all context items with priority scores -/// 2. Sort by utility (priority / token_cost) -- best bang-per-token first -/// 3. Pack within budget using score-adaptive truncation when score variance exists -/// 4. Record admitted vs rejected for observability -/// 5. Return prompt with compilation metadata and savings +use crate::handlers::estimate_tokens; +use chrono::Utc; +use rusqlite::{params, Connection}; +use serde_json::json; +use std::path::Path; pub fn compile(conn: &Connection, home: &Path, agent: &str, max_tokens: usize) -> BootResult { - // ── 1. Gather context items with priorities ───────────────────────────── - let mut items: Vec = Vec::new(); - - // Identity capsule: must-have (priority 1.0) let (identity_text, _) = build_identity_capsule(conn); if !identity_text.is_empty() { - items.push(ContextItem::new( - "identity", - format!("## Identity\n{identity_text}"), - 1.0, - )); + items.push(ContextItem::new("identity", format!("## Identity\n{identity_text}"), 1.0)); } - - // Delta capsule: broken into sub-items with individual priorities let (delta_text, _, _delta_freshness) = build_delta_capsule(conn, agent); if !delta_text.is_empty() { - // Split delta into sections, each scored independently let sections: Vec<(&str, f64)> = vec![ - ("## Pending Messages", 0.95), // Messages from other agents = urgent - ("## Active Agents", 0.60), // Who's online = coordination - ("## Active Locks", 0.70), // Locks = collision prevention - ("## Feed", 0.40), // Feed = nice context - ("## Pending Tasks", 0.75), // Task board = actionable - ("## Your Active Tasks", 0.80), // Your tasks = high priority - ("CONFLICTS:", 0.90), // Conflicts = must resolve - ("## Active Focus", 0.85), // Focus scope = context boundary - ("New decisions:", 0.55), // Recent decisions = orientation - ("New knowledge:", 0.45), // New memories - ("Activity since last boot:", 0.30), // Activity summary = low value - ("Recent decisions:", 0.50), // First-boot orientation + ("## Pending Messages", 0.95), + ("## Active Agents", 0.60), + ("## Active Locks", 0.70), + ("## Feed", 0.40), + ("## Pending Tasks", 0.75), + ("## Your Active Tasks", 0.80), + ("CONFLICTS:", 0.90), + ("## Active Focus", 0.85), + ("New decisions:", 0.55), + ("New knowledge:", 0.45), + ("Activity since last boot:", 0.30), + ("Recent decisions:", 0.50), ]; - - // Try to split delta into scored sub-sections let remaining_delta = delta_text.as_str(); let mut matched_any = false; - for (header, priority) in §ions { if let Some(start) = remaining_delta.find(header) { - // Find end: next section header or end of string let content_start = start; let after_header = start + header.len(); - let end = remaining_delta[after_header..] - .find("\n\n") - .map(|p| after_header + p) - .unwrap_or(remaining_delta.len()); - + let end = remaining_delta[after_header..].find("\n\n").map(|p| after_header + p).unwrap_or(remaining_delta.len()); let section_text = remaining_delta[content_start..end].trim().to_string(); if !section_text.is_empty() { items.push(ContextItem::new(header, section_text, *priority)); @@ -74,82 +40,40 @@ pub fn compile(conn: &Connection, home: &Path, agent: &str, max_tokens: usize) - } } } - - // Fallback: if no sections matched, treat delta as one block if !matched_any { - items.push(ContextItem::new( - "delta", - format!("## Delta\n{delta_text}"), - 0.70, - )); + items.push(ContextItem::new("delta", format!("## Delta\n{delta_text}"), 0.70)); } } - - // ── 2. Record boot ────────────────────────────────────────────────────── for candidate in rank_candidates(fetch_rank_candidates(conn), boot_rank_top_n(), Utc::now()) { items.push(ContextItem::from_ranked_candidate(candidate)); } - record_boot(conn, agent); - - // ── 3. Sort by utility (priority / token_cost) descending ─────────────── - items.sort_by(|a, b| { - b.utility - .partial_cmp(&a.utility) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - // ── 4. Score-adaptive budget packing ──────────────────────────────────── + items.sort_by(|a, b| b.utility.partial_cmp(&a.utility).unwrap_or(std::cmp::Ordering::Equal)); let packed = pack_context_items(&items, max_tokens, boot_source_token_bounds()); let admitted = packed.admitted; let rejected = packed.rejected; let assembled_parts = packed.assembled_parts; - let assembled = assembled_parts.join("\n\n"); let token_estimate = estimate_tokens(&assembled); - - // ── 5. Savings and observability ──────────────────────────────────────── let raw_baseline = estimate_raw_baseline(conn, home); let saved = raw_baseline.saturating_sub(token_estimate); - let percent = if raw_baseline > 0 { - (saved * 100) / raw_baseline - } else { - 0 - }; - - // Only record savings when baseline is meaningful (skip empty-DB boots) + let percent = if raw_baseline > 0 { (saved * 100) / raw_baseline } else { 0 }; if raw_baseline > 0 { let _ = conn.execute( "INSERT INTO events (type, data, source_agent) VALUES (?1, ?2, ?3)", params![ "boot_savings", - serde_json::to_string(&json!({ - "agent": agent, - "served": token_estimate, - "baseline": raw_baseline, - "saved": saved, - "percent": percent, - "admitted": admitted.len(), - "rejected": rejected.len() - })) + serde_json::to_string(&json!({"agent":agent,"served":token_estimate,"baseline":raw_baseline,"saved":saved,"percent":percent, +"admitted":admitted.len(),"rejected":rejected.len()})) .unwrap_or_default(), "rust-daemon" ], ); } - BootResult { boot_prompt: assembled, token_estimate, - savings: json!({ - "rawBaseline": raw_baseline, - "served": token_estimate, - "saved": saved, - "percent": percent - }), + savings: json!({"rawBaseline":raw_baseline,"served":token_estimate,"saved":saved,"percent":percent}), capsules: admitted, } } - -// Dead code removed: find_memory_dir, read_memory_files, read_lessons -// (indexer.rs has its own implementation; these were ported but unused) diff --git a/daemon-rs/src/compiler/mod.rs b/daemon-rs/src/compiler/mod.rs index 264fe458..c1e4c9e1 100644 --- a/daemon-rs/src/compiler/mod.rs +++ b/daemon-rs/src/compiler/mod.rs @@ -1,23 +1,16 @@ -// SPDX-License-Identifier: MIT -mod types; mod cache; mod capsules; -mod ranking; -mod packing; mod compile; - +mod packing; +mod ranking; #[cfg(test)] #[cfg(test)] -mod tests { - // Compiler internals are not release-gated; see Info/testing-philosophy.md. -} - -pub(crate) use types::*; +mod tests; +mod types; pub(crate) use cache::*; pub(crate) use capsules::*; -pub(crate) use ranking::*; -pub(crate) use packing::*; -pub(crate) use compile::*; - pub use compile::compile; +pub(crate) use packing::*; +pub(crate) use ranking::*; pub use types::BootResult; +pub(crate) use types::*; diff --git a/daemon-rs/src/compiler/packing.rs b/daemon-rs/src/compiler/packing.rs index 8efb04ce..d7274760 100644 --- a/daemon-rs/src/compiler/packing.rs +++ b/daemon-rs/src/compiler/packing.rs @@ -1,54 +1,25 @@ -// SPDX-License-Identifier: MIT -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; -use rusqlite::{params, Connection, OptionalExtension}; +use super::*; +use crate::handlers::estimate_tokens; +use rusqlite::Connection; use serde_json::{json, Value}; -use std::collections::HashSet; use std::env; -use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - -use super::*; pub(crate) fn read_usize_env(name: &str, default: usize) -> usize { - env::var(name) - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(default) + env::var(name).ok().and_then(|value| value.trim().parse::().ok()).filter(|value| *value > 0).unwrap_or(default) } - pub(crate) fn boot_source_token_bounds() -> SourceTokenBounds { SourceTokenBounds::new( - read_usize_env( - "CORTEX_BOOT_MIN_SOURCE_TOKENS", - DEFAULT_BOOT_MIN_SOURCE_TOKENS, - ), - read_usize_env( - "CORTEX_BOOT_MAX_SOURCE_TOKENS", - DEFAULT_BOOT_MAX_SOURCE_TOKENS, - ), + read_usize_env("CORTEX_BOOT_MIN_SOURCE_TOKENS", DEFAULT_BOOT_MIN_SOURCE_TOKENS), + read_usize_env("CORTEX_BOOT_MAX_SOURCE_TOKENS", DEFAULT_BOOT_MAX_SOURCE_TOKENS), ) } - pub(crate) fn boot_rank_top_n() -> usize { read_usize_env("CORTEX_BOOT_RANK_TOP_N", DEFAULT_BOOT_RANK_TOP_N).min(20) } - pub(crate) fn empty_rank_components() -> RankComponents { - RankComponents { - class_score: 0.0, - recency_score: 0.0, - relevance_score: 0.0, - activity_score: 0.0, - total_score: 0.0, - } + RankComponents { class_score: 0.0, recency_score: 0.0, relevance_score: 0.0, activity_score: 0.0, total_score: 0.0 } } - pub(crate) fn fetch_rank_candidates(conn: &Connection) -> Vec { let mut candidates = Vec::new(); - if let Ok(mut stmt) = conn.prepare( "SELECT id, text, type, retention_class, score, retrievals, last_accessed, updated_at, created_at FROM memories @@ -62,9 +33,7 @@ pub(crate) fn fetch_rank_candidates(conn: &Connection) -> Vec { source_id: row.get::<_, i64>(0)?, body: row.get::<_, String>(1)?, title: row.get::<_, Option>(2)?.unwrap_or_else(|| "memory".to_string()), - retention_class: row - .get::<_, Option>(3)? - .unwrap_or_else(|| "operational".to_string()), + retention_class: row.get::<_, Option>(3)?.unwrap_or_else(|| "operational".to_string()), relevance: row.get::<_, Option>(4)?.unwrap_or(0.5), retrievals: row.get::<_, Option>(5)?.unwrap_or(0), last_accessed: row.get::<_, Option>(6)?, @@ -76,7 +45,6 @@ pub(crate) fn fetch_rank_candidates(conn: &Connection) -> Vec { candidates.extend(rows.flatten()); } } - if let Ok(mut stmt) = conn.prepare( "SELECT id, decision, context, type, retention_class, score, retrievals, last_accessed, updated_at, created_at FROM decisions @@ -95,12 +63,8 @@ pub(crate) fn fetch_rank_candidates(conn: &Connection) -> Vec { source_kind: "decision", source_id: row.get::<_, i64>(0)?, body, - title: row - .get::<_, Option>(3)? - .unwrap_or_else(|| "decision".to_string()), - retention_class: row - .get::<_, Option>(4)? - .unwrap_or_else(|| "operational".to_string()), + title: row.get::<_, Option>(3)?.unwrap_or_else(|| "decision".to_string()), + retention_class: row.get::<_, Option>(4)?.unwrap_or_else(|| "operational".to_string()), relevance: row.get::<_, Option>(5)?.unwrap_or(0.5), retrievals: row.get::<_, Option>(6)?.unwrap_or(0), last_accessed: row.get::<_, Option>(7)?, @@ -112,10 +76,8 @@ pub(crate) fn fetch_rank_candidates(conn: &Connection) -> Vec { candidates.extend(rows.flatten()); } } - candidates } - pub(crate) fn score_signal_is_flat(items: &[ContextItem]) -> bool { let mut count = 0usize; let mut sum = 0.0; @@ -126,7 +88,6 @@ pub(crate) fn score_signal_is_flat(items: &[ContextItem]) -> bool { if count <= 1 { return true; } - let mean = sum / count as f64; let variance = items .iter() @@ -139,17 +100,12 @@ pub(crate) fn score_signal_is_flat(items: &[ContextItem]) -> bool { / count as f64; variance < SCORE_VARIANCE_FLAT_THRESHOLD } - pub(crate) fn truncate_to_token_budget(text: &str, token_budget: usize) -> (String, usize) { if token_budget == 0 { return (String::new(), 0); } - let total_chars = text.chars().count(); - let mut char_budget = ((token_budget as f64 * 3.5) as usize) - .max(1) - .min(total_chars); - + let mut char_budget = ((token_budget as f64 * 3.5) as usize).max(1).min(total_chars); loop { let prefix: String = text.chars().take(char_budget).collect(); let candidate = format!("{prefix}..."); @@ -160,28 +116,21 @@ pub(crate) fn truncate_to_token_budget(text: &str, token_budget: usize) -> (Stri char_budget -= 1; } } - pub(crate) fn pack_context_items_greedy(items: &[ContextItem], max_tokens: usize) -> PackedContext { let mut budget_remaining = max_tokens; let mut admitted: Vec = Vec::new(); let mut rejected: Vec = Vec::new(); let mut assembled_parts: Vec = Vec::new(); - for item in items { if item.tokens <= budget_remaining && !item.text.is_empty() { assembled_parts.push(item.text.clone()); budget_remaining -= item.tokens; admitted.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": item.tokens, - "priority": item.priority, - "utility": (item.utility * 10000.0).round() / 10000.0 - }), + json!({"name":item.name,"tokens":item.tokens,"priority": +item.priority,"utility":(item.utility*10000.0).round()/10000.0}), item, )); } else if !item.text.is_empty() { - // Try truncation for high-priority items if item.priority >= 0.7 && budget_remaining > 30 { let trunc_chars = (budget_remaining as f64 * 3.5) as usize; let truncated: String = item.text.chars().take(trunc_chars).collect(); @@ -189,59 +138,30 @@ pub(crate) fn pack_context_items_greedy(items: &[ContextItem], max_tokens: usize assembled_parts.push(format!("{truncated}...")); budget_remaining = budget_remaining.saturating_sub(trunc_tokens); admitted.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": trunc_tokens, - "priority": item.priority, - "truncated": true - }), + json!({"name":item.name,"tokens":trunc_tokens, +"priority":item.priority,"truncated":true}), item, )); } else { rejected.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": item.tokens, - "priority": item.priority, - "reason": "budget_exceeded" - }), + json!({"name":item.name,"tokens":item. +tokens,"priority":item.priority,"reason":"budget_exceeded"}), item, )); } } } - - PackedContext { - assembled_parts, - admitted, - rejected, - } + PackedContext { assembled_parts, admitted, rejected } } - -pub(crate) fn score_adaptive_allocations( - items: &[ContextItem], - max_tokens: usize, - bounds: SourceTokenBounds, -) -> Vec { - let mut order: Vec = items - .iter() - .enumerate() - .filter(|(_, item)| !item.text.is_empty()) - .map(|(idx, _)| idx) - .collect(); +pub(crate) fn score_adaptive_allocations(items: &[ContextItem], max_tokens: usize, bounds: SourceTokenBounds) -> Vec { + let mut order: Vec = items.iter().enumerate().filter(|(_, item)| !item.text.is_empty()).map(|(idx, _)| idx).collect(); order.sort_by(|left, right| { items[*right] .priority .partial_cmp(&items[*left].priority) .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| { - items[*right] - .utility - .partial_cmp(&items[*left].utility) - .unwrap_or(std::cmp::Ordering::Equal) - }) + .then_with(|| items[*right].utility.partial_cmp(&items[*left].utility).unwrap_or(std::cmp::Ordering::Equal)) }); - let mut allocations = vec![0usize; items.len()]; let mut floor_spent = 0usize; for idx in order { @@ -258,25 +178,18 @@ pub(crate) fn score_adaptive_allocations( floor_spent += allocations[idx]; } } - let mut remaining = max_tokens.saturating_sub(floor_spent); while remaining > 0 { let eligible: Vec = allocations .iter() .enumerate() - .filter(|(idx, allocation)| { - **allocation > 0 && **allocation < items[*idx].tokens.min(bounds.max) - }) + .filter(|(idx, allocation)| **allocation > 0 && **allocation < items[*idx].tokens.min(bounds.max)) .map(|(idx, _)| idx) .collect(); if eligible.is_empty() { break; } - - let total_score = eligible - .iter() - .map(|idx| items[*idx].priority.max(0.01)) - .sum::(); + let total_score = eligible.iter().map(|idx| items[*idx].priority.max(0.01)).sum::(); let mut allocated_any = false; for idx in eligible { if remaining == 0 { @@ -287,8 +200,7 @@ pub(crate) fn score_adaptive_allocations( if room == 0 { continue; } - let share = ((remaining as f64) * (items[idx].priority.max(0.01) / total_score)).ceil() - as usize; + let share = ((remaining as f64) * (items[idx].priority.max(0.01) / total_score)).ceil() as usize; let delta = share.max(1).min(room).min(remaining); allocations[idx] += delta; remaining -= delta; @@ -298,20 +210,13 @@ pub(crate) fn score_adaptive_allocations( break; } } - allocations } - -pub(crate) fn pack_context_items_score_adaptive( - items: &[ContextItem], - max_tokens: usize, - bounds: SourceTokenBounds, -) -> PackedContext { +pub(crate) fn pack_context_items_score_adaptive(items: &[ContextItem], max_tokens: usize, bounds: SourceTokenBounds) -> PackedContext { let allocations = score_adaptive_allocations(items, max_tokens, bounds); let mut admitted: Vec = Vec::new(); let mut rejected: Vec = Vec::new(); let mut assembled_parts: Vec = Vec::new(); - for (idx, item) in items.iter().enumerate() { if item.text.is_empty() { continue; @@ -319,73 +224,38 @@ pub(crate) fn pack_context_items_score_adaptive( let allocation = allocations[idx]; if allocation == 0 { rejected.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": item.tokens, - "priority": item.priority, - "reason": "score_adaptive_budget_exceeded" - }), + json!({"name":item.name,"tokens":item.tokens,"priority":item.priority,"reason":"score_adaptive_budget_exceeded"}), item, )); continue; } - if item.tokens <= allocation { assembled_parts.push(item.text.clone()); admitted.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": item.tokens, - "allocatedTokens": allocation, - "priority": item.priority, - "utility": (item.utility * 10000.0).round() / 10000.0, - "packing": "score_adaptive" - }), + json!({"name":item.name, +"tokens":item.tokens,"allocatedTokens":allocation,"priority":item.priority,"utility":(item.utility*10000.0).round()/10000.0, +"packing":"score_adaptive"}), item, )); } else { let (truncated, trunc_tokens) = truncate_to_token_budget(&item.text, allocation); assembled_parts.push(truncated); admitted.push(attach_rank_audit( - json!({ - "name": item.name, - "tokens": trunc_tokens, - "allocatedTokens": allocation, - "priority": item.priority, - "truncated": true, - "packing": "score_adaptive" - }), + json!({"name":item.name,"tokens":trunc_tokens,"allocatedTokens": +allocation,"priority":item.priority,"truncated":true,"packing":"score_adaptive"}), item, )); } } - - PackedContext { - assembled_parts, - admitted, - rejected, - } + PackedContext { assembled_parts, admitted, rejected } } - -pub(crate) fn pack_context_items( - items: &[ContextItem], - max_tokens: usize, - bounds: SourceTokenBounds, -) -> PackedContext { +pub(crate) fn pack_context_items(items: &[ContextItem], max_tokens: usize, bounds: SourceTokenBounds) -> PackedContext { pack_context_items_with_mode(items, max_tokens, bounds, boot_packing_mode()) } - -pub(crate) fn pack_context_items_with_mode( - items: &[ContextItem], - max_tokens: usize, - bounds: SourceTokenBounds, - mode: BootPackingMode, -) -> PackedContext { +pub(crate) fn pack_context_items_with_mode(items: &[ContextItem], max_tokens: usize, bounds: SourceTokenBounds, mode: BootPackingMode) -> PackedContext { match mode { BootPackingMode::LegacyGreedy => pack_context_items_greedy(items, max_tokens), - BootPackingMode::ScoreAdaptive => { - pack_context_items_score_adaptive(items, max_tokens, bounds) - } + BootPackingMode::ScoreAdaptive => pack_context_items_score_adaptive(items, max_tokens, bounds), BootPackingMode::Auto => { if score_signal_is_flat(items) { pack_context_items_greedy(items, max_tokens) diff --git a/daemon-rs/src/compiler/ranking.rs b/daemon-rs/src/compiler/ranking.rs index 7736e84b..f07b8369 100644 --- a/daemon-rs/src/compiler/ranking.rs +++ b/daemon-rs/src/compiler/ranking.rs @@ -1,20 +1,7 @@ -// SPDX-License-Identifier: MIT +use super::*; +use crate::handlers::estimate_tokens; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; -use rusqlite::{params, Connection, OptionalExtension}; use serde_json::{json, Value}; -use std::collections::HashSet; -use std::env; -use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - -use super::*; -// ─── Public API ───────────────────────────────────────────────────────────── - -// ─── Context Item for ranked compilation ─────────────────────────────────── - #[derive(Clone, Copy, Debug, PartialEq)] pub(crate) struct RankComponents { pub(crate) class_score: f64, @@ -23,7 +10,6 @@ pub(crate) struct RankComponents { pub(crate) activity_score: f64, pub(crate) total_score: f64, } - #[derive(Clone, Debug, PartialEq)] pub(crate) struct RankAudit { pub(crate) source_kind: &'static str, @@ -31,7 +17,6 @@ pub(crate) struct RankAudit { pub(crate) retention_class: String, pub(crate) components: RankComponents, } - #[derive(Clone, Debug)] pub(crate) struct RankedCandidate { pub(crate) source_kind: &'static str, @@ -46,7 +31,6 @@ pub(crate) struct RankedCandidate { pub(crate) relevance: f64, pub(crate) components: RankComponents, } - pub(crate) fn clamp01(value: f64) -> f64 { if value.is_finite() { value.clamp(0.0, 1.0) @@ -54,7 +38,6 @@ pub(crate) fn clamp01(value: f64) -> f64 { 0.0 } } - pub(crate) fn retention_class_score(retention_class: &str) -> f64 { match retention_class { "durable" => 1.0, @@ -64,7 +47,6 @@ pub(crate) fn retention_class_score(retention_class: &str) -> f64 { _ => 0.6, } } - pub(crate) fn parse_timestamp(value: Option<&str>) -> Option> { let value = value?.trim(); if value.is_empty() { @@ -73,13 +55,8 @@ pub(crate) fn parse_timestamp(value: Option<&str>) -> Option> { DateTime::parse_from_rfc3339(value) .map(|dt| dt.with_timezone(&Utc)) .ok() - .or_else(|| { - NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S") - .ok() - .map(|dt| Utc.from_utc_datetime(&dt)) - }) + .or_else(|| NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S").ok().map(|dt| Utc.from_utc_datetime(&dt))) } - pub(crate) fn recency_score(timestamp: Option<&str>, now: DateTime) -> f64 { let Some(timestamp) = parse_timestamp(timestamp) else { return 0.05; @@ -94,45 +71,24 @@ pub(crate) fn recency_score(timestamp: Option<&str>, now: DateTime) -> f64 _ => 0.05, } } - pub(crate) fn activity_score(retrievals: i64, last_accessed: Option<&str>, now: DateTime) -> f64 { let retrieval_score = (retrievals.max(0) as f64 / 10.0).min(1.0); let access_score = recency_score(last_accessed, now); (retrieval_score * 0.55) + (access_score * 0.45) } - pub(crate) fn rank_components_for(candidate: &RankedCandidate, now: DateTime) -> RankComponents { - let timestamp = candidate - .updated_at - .as_deref() - .or(candidate.created_at.as_deref()); + let timestamp = candidate.updated_at.as_deref().or(candidate.created_at.as_deref()); let class_score = retention_class_score(&candidate.retention_class); let recency_score = recency_score(timestamp, now); let relevance_score = clamp01(candidate.relevance); - let activity_score = activity_score( - candidate.retrievals, - candidate.last_accessed.as_deref(), - now, - ); + let activity_score = activity_score(candidate.retrievals, candidate.last_accessed.as_deref(), now); let total_score = (class_score * RANK_WEIGHT_CLASS) + (recency_score * RANK_WEIGHT_RECENCY) + (relevance_score * RANK_WEIGHT_RELEVANCE) + (activity_score * RANK_WEIGHT_ACTIVITY); - - RankComponents { - class_score, - recency_score, - relevance_score, - activity_score, - total_score, - } + RankComponents { class_score, recency_score, relevance_score, activity_score, total_score } } - -pub(crate) fn rank_candidates( - mut candidates: Vec, - top_n: usize, - now: DateTime, -) -> Vec { +pub(crate) fn rank_candidates(mut candidates: Vec, top_n: usize, now: DateTime) -> Vec { for candidate in &mut candidates { candidate.components = rank_components_for(candidate, now); } @@ -148,63 +104,32 @@ pub(crate) fn rank_candidates( candidates.truncate(top_n); candidates } - pub(crate) fn rank_audit_json(audit: &RankAudit) -> Value { - json!({ - "sourceKind": audit.source_kind, - "sourceId": audit.source_id, - "retentionClass": audit.retention_class, - "rankComponents": { - "class": (audit.components.class_score * 10000.0).round() / 10000.0, - "recency": (audit.components.recency_score * 10000.0).round() / 10000.0, - "relevance": (audit.components.relevance_score * 10000.0).round() / 10000.0, - "activity": (audit.components.activity_score * 10000.0).round() / 10000.0, - "total": (audit.components.total_score * 10000.0).round() / 10000.0 - } - }) + json!({"sourceKind":audit.source_kind, +"sourceId":audit.source_id,"retentionClass":audit.retention_class,"rankComponents":{"class":(audit.components.class_score*10000.0) +.round()/10000.0,"recency":(audit.components.recency_score*10000.0).round()/10000.0,"relevance":(audit.components.relevance_score* +10000.0).round()/10000.0,"activity":(audit.components.activity_score*10000.0).round()/10000.0,"total":(audit.components. +total_score*10000.0).round()/10000.0}}) } - pub(crate) struct ContextItem { pub(crate) name: String, pub(crate) text: String, pub(crate) tokens: usize, - /// Base priority: 1.0 = must-have, 0.5 = important, 0.2 = nice-to-have pub(crate) priority: f64, - /// Utility score: priority / token_cost (higher = more efficient) pub(crate) utility: f64, rank_audit: Option, } - impl ContextItem { pub(crate) fn new(name: &str, text: String, priority: f64) -> Self { let tokens = estimate_tokens(&text); - let utility = if tokens > 0 { - priority / (tokens as f64) - } else { - 0.0 - }; - Self { - name: name.to_string(), - text, - tokens, - priority, - utility, - rank_audit: None, - } + let utility = if tokens > 0 { priority / (tokens as f64) } else { 0.0 }; + Self { name: name.to_string(), text, tokens, priority, utility, rank_audit: None } } - pub(crate) fn from_ranked_candidate(candidate: RankedCandidate) -> Self { let title = candidate.title.chars().take(160).collect::(); let body = candidate.body.chars().take(420).collect::(); - let text = format!( - "## Ranked {} Context\n- {}: {}", - candidate.source_kind, title, body - ); - let mut item = Self::new( - &format!("ranked:{}:{}", candidate.source_kind, candidate.source_id), - text, - candidate.components.total_score.max(0.01), - ); + let text = format!("## Ranked {} Context\n- {}: {}", candidate.source_kind, title, body); + let mut item = Self::new(&format!("ranked:{}:{}", candidate.source_kind, candidate.source_id), text, candidate.components.total_score.max(0.01)); item.rank_audit = Some(RankAudit { source_kind: candidate.source_kind, source_id: candidate.source_id, @@ -214,7 +139,6 @@ impl ContextItem { item } } - pub(crate) fn attach_rank_audit(mut entry: Value, item: &ContextItem) -> Value { if let (Some(object), Some(audit)) = (entry.as_object_mut(), item.rank_audit.as_ref()) { if let Value::Object(rank_object) = rank_audit_json(audit) { @@ -225,26 +149,19 @@ pub(crate) fn attach_rank_audit(mut entry: Value, item: &ContextItem) -> Value { } entry } - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct SourceTokenBounds { pub(crate) min: usize, pub(crate) max: usize, } - impl SourceTokenBounds { pub(crate) fn new(min: usize, max: usize) -> Self { let min = min.max(1); - Self { - min, - max: max.max(min), - } + Self { min, max: max.max(min) } } } - pub(crate) struct PackedContext { pub(crate) assembled_parts: Vec, pub(crate) admitted: Vec, pub(crate) rejected: Vec, } - diff --git a/daemon-rs/src/compiler/tests/mod.rs b/daemon-rs/src/compiler/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/compiler/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/compiler/types.rs b/daemon-rs/src/compiler/types.rs index 4027c4ac..aa5a3d6f 100644 --- a/daemon-rs/src/compiler/types.rs +++ b/daemon-rs/src/compiler/types.rs @@ -1,16 +1,5 @@ -// SPDX-License-Identifier: MIT -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use regex::Regex; -use rusqlite::{params, Connection, OptionalExtension}; -use serde_json::{json, Value}; -use std::collections::HashSet; +use serde_json::Value; use std::env; -use std::path::Path; - -use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - -use super::*; pub(crate) const DEFAULT_BOOT_MIN_SOURCE_TOKENS: usize = 40; pub(crate) const DEFAULT_BOOT_MAX_SOURCE_TOKENS: usize = 600; pub(crate) const DEFAULT_BOOT_RANK_TOP_N: usize = 5; @@ -19,50 +8,33 @@ pub(crate) const RANK_WEIGHT_CLASS: f64 = 0.30; pub(crate) const RANK_WEIGHT_RECENCY: f64 = 0.30; pub(crate) const RANK_WEIGHT_RELEVANCE: f64 = 0.25; pub(crate) const RANK_WEIGHT_ACTIVITY: f64 = 0.15; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum BootPackingMode { Auto, LegacyGreedy, ScoreAdaptive, } - pub(crate) fn boot_packing_mode() -> BootPackingMode { - match env::var("CORTEX_BOOT_PACKING_MODE") - .unwrap_or_default() - .trim() - .to_ascii_lowercase() - .as_str() - { + match env::var("CORTEX_BOOT_PACKING_MODE").unwrap_or_default().trim().to_ascii_lowercase().as_str() { "legacy" | "greedy" | "v0.5" | "v0.5.0" => BootPackingMode::LegacyGreedy, "adaptive" | "score-adaptive" | "score_adaptive" => BootPackingMode::ScoreAdaptive, _ => BootPackingMode::Auto, } } - pub(crate) fn detect_identity() -> String { - let user = env::var("USERNAME") - .or_else(|_| env::var("USER")) - .unwrap_or_else(|_| "cortex-user".to_string()); - + let user = env::var("USERNAME").or_else(|_| env::var("USER")).unwrap_or_else(|_| "cortex-user".to_string()); let platform = match env::consts::OS { "windows" => "Windows", "macos" => "macOS", "linux" => "Linux", other => other, }; - let shell = env::var("SHELL") .or_else(|_| env::var("COMSPEC")) .map(|s| s.rsplit(['/', '\\']).next().unwrap_or(&s).to_string()) .unwrap_or_else(|_| "unknown".to_string()); - format!("User: {user}. Platform: {platform}. Shell: {shell}.") } - -// ─── Public types ─────────────────────────────────────────────────────────── - -/// The assembled boot prompt and its metadata. pub struct BootResult { pub boot_prompt: String, pub token_estimate: usize, diff --git a/daemon-rs/src/conflict.rs b/daemon-rs/src/conflict.rs deleted file mode 100644 index 55fe3d1a..00000000 --- a/daemon-rs/src/conflict.rs +++ /dev/null @@ -1,568 +0,0 @@ -// SPDX-License-Identifier: MIT -use rusqlite::Connection; -use std::collections::HashSet; - -const RELATED_THRESHOLD: f64 = 0.40; -const AGREEMENT_THRESHOLD: f64 = 0.84; -const CORE_CONTRADICTION_OVERLAP_THRESHOLD: f64 = 0.35; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ConflictClassification { - Agrees, - Contradicts, - Refines, - Unrelated, -} - -impl ConflictClassification { - pub const fn as_str(self) -> &'static str { - match self { - Self::Agrees => "AGREES", - Self::Contradicts => "CONTRADICTS", - Self::Refines => "REFINES", - Self::Unrelated => "UNRELATED", - } - } -} - -#[derive(Debug, Clone)] -struct DecisionCandidate { - id: i64, - decision: String, - source_agent: String, - trust_score: f64, -} - -#[allow(dead_code)] -pub struct ConflictResult { - pub classification: ConflictClassification, - pub is_conflict: bool, - pub is_update: bool, - pub matched_id: Option, - pub matched_agent: Option, - pub matched_decision: Option, - pub matched_trust_score: Option, - pub similarity_jaccard: f64, - pub similarity_cosine: Option, -} - -impl ConflictResult { - fn unrelated() -> Self { - Self { - classification: ConflictClassification::Unrelated, - is_conflict: false, - is_update: false, - matched_id: None, - matched_agent: None, - matched_decision: None, - matched_trust_score: None, - similarity_jaccard: 0.0, - similarity_cosine: None, - } - } - - fn from_candidate( - classification: ConflictClassification, - candidate: &DecisionCandidate, - source_agent: &str, - similarity_jaccard: f64, - similarity_cosine: Option, - ) -> Self { - let is_conflict = matches!(classification, ConflictClassification::Contradicts); - let is_update = matches!(classification, ConflictClassification::Refines) - || (matches!(classification, ConflictClassification::Agrees) - && candidate.source_agent == source_agent); - - Self { - classification, - is_conflict, - is_update, - matched_id: Some(candidate.id), - matched_agent: Some(candidate.source_agent.clone()), - matched_decision: Some(candidate.decision.clone()), - matched_trust_score: Some(candidate.trust_score), - similarity_jaccard, - similarity_cosine, - } - } -} - -/// Jaccard similarity between two strings (word-level). -/// Matches the Node.js jaccardSimilarity: splits on whitespace, -/// filters tokens shorter than 2 chars, lowercases. -pub fn jaccard_similarity(a: &str, b: &str) -> f64 { - let set_a: HashSet = a - .split_whitespace() - .filter(|w| w.len() > 1) - .map(|w| w.to_lowercase()) - .collect(); - let set_b: HashSet = b - .split_whitespace() - .filter(|w| w.len() > 1) - .map(|w| w.to_lowercase()) - .collect(); - - if set_a.is_empty() && set_b.is_empty() { - return 1.0; - } - if set_a.is_empty() || set_b.is_empty() { - return 0.0; - } - - let intersection = set_a.intersection(&set_b).count() as f64; - let union = (set_a.len() + set_b.len()) as f64 - intersection; - if union == 0.0 { - return 0.0; - } - intersection / union -} - -/// Detect conflicts by checking the last 50 active decisions. -/// Same agent + sim > 0.6 => update (supersede old) -/// Different agent + sim > 0.6 => conflict (disputed) -pub fn detect_conflict( - conn: &Connection, - decision: &str, - source_agent: &str, - owner_id: Option, -) -> Result { - let (sql, has_owner_scope) = if owner_id.is_some() { - ( - "SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) \ - FROM decisions \ - WHERE owner_id = ?1 \ - AND status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - ORDER BY id DESC \ - LIMIT 50", - true, - ) - } else { - ( - "SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) \ - FROM decisions \ - WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - ORDER BY id DESC \ - LIMIT 50", - false, - ) - }; - let mut stmt = conn - .prepare(sql) - .map_err(|e| format!("Failed to prepare conflict query: {e}"))?; - - let rows: Vec = if has_owner_scope { - stmt.query_map([owner_id.unwrap_or_default()], |row| { - Ok(DecisionCandidate { - id: row.get(0)?, - decision: row.get(1)?, - source_agent: row.get(2)?, - trust_score: row.get(3)?, - }) - }) - .map_err(|e| format!("Failed to query decisions: {e}"))? - .filter_map(|r| r.ok()) - .collect() - } else { - stmt.query_map([], |row| { - Ok(DecisionCandidate { - id: row.get(0)?, - decision: row.get(1)?, - source_agent: row.get(2)?, - trust_score: row.get(3)?, - }) - }) - .map_err(|e| format!("Failed to query decisions: {e}"))? - .filter_map(|r| r.ok()) - .collect() - }; - - let mut best_sim = 0.0_f64; - let mut best_candidate: Option = None; - - for candidate in &rows { - let sim = jaccard_similarity(decision, &candidate.decision); - if sim > best_sim { - best_sim = sim; - best_candidate = Some(candidate.clone()); - } - } - - let Some(best_candidate) = best_candidate else { - return Ok(ConflictResult::unrelated()); - }; - - if best_sim < RELATED_THRESHOLD { - return Ok(ConflictResult::unrelated()); - } - - let classification = classify_relation(decision, source_agent, &best_candidate, best_sim); - Ok(ConflictResult::from_candidate( - classification, - &best_candidate, - source_agent, - best_sim, - None, - )) -} - -/// Embedding-based conflict detection with semantic dedup. -/// -/// Three tiers: -/// - cosine > 0.85: hard conflict/update (existing behavior) -/// - cosine 0.70-0.85, same agent: semantic merge (NEW -- dedup zone) -/// - cosine < 0.70: no conflict, proceed to Jaccard fallback -/// -/// The merge tier prevents near-duplicate memories that waste context budget. -/// Instead of storing "use uv for python" alongside "always use uv, never pip", -/// it merges them into a single strengthened entry. -#[allow(dead_code)] -pub fn detect_conflict_cosine( - decision: &str, - source_agent: &str, - engine: &crate::embeddings::EmbeddingEngine, - conn: &Connection, -) -> Option { - let new_vec = engine.embed(decision)?; - - let mut stmt = conn - .prepare( - "SELECT d.id, d.source_agent, e.vector \ - FROM decisions d \ - JOIN embeddings e ON e.target_type = 'decision' AND e.target_id = d.id \ - WHERE d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now'))", - ) - .ok()?; - - let rows: Vec<(i64, String, Vec)> = stmt - .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) - .ok()? - .filter_map(|r| r.ok()) - .collect(); - - let mut best_sim = 0.0f32; - let mut best_id: Option = None; - let mut best_agent: Option = None; - - for (id, agent, blob) in &rows { - let existing_vec = crate::embeddings::blob_to_vector(blob); - let sim = crate::embeddings::cosine_similarity(&new_vec, &existing_vec); - if sim > best_sim { - best_sim = sim; - best_id = Some(*id); - best_agent = Some(agent.clone()); - } - } - - const HARD_THRESHOLD: f32 = 0.85; - const MERGE_THRESHOLD: f32 = 0.70; - - if best_sim > HARD_THRESHOLD { - let classification = if best_agent.as_deref() == Some(source_agent) { - ConflictClassification::Refines - } else { - ConflictClassification::Contradicts - }; - Some(ConflictResult { - classification, - is_conflict: matches!(classification, ConflictClassification::Contradicts), - is_update: matches!(classification, ConflictClassification::Refines), - matched_id: best_id, - matched_agent: best_agent, - matched_decision: None, - matched_trust_score: None, - similarity_jaccard: 0.0, - similarity_cosine: Some(best_sim as f64), - }) - } else if best_sim > MERGE_THRESHOLD && best_agent.as_deref() == Some(source_agent) { - let classification = ConflictClassification::Agrees; - Some(ConflictResult { - classification, - is_conflict: false, - is_update: true, - matched_id: best_id, - matched_agent: best_agent, - matched_decision: None, - matched_trust_score: None, - similarity_jaccard: 0.0, - similarity_cosine: Some(best_sim as f64), - }) - } else { - None - } -} - -fn classify_relation( - incoming_decision: &str, - incoming_agent: &str, - candidate: &DecisionCandidate, - similarity_jaccard: f64, -) -> ConflictClassification { - if similarity_jaccard < RELATED_THRESHOLD { - return ConflictClassification::Unrelated; - } - - if contradiction_signal(incoming_decision, &candidate.decision, similarity_jaccard) { - return ConflictClassification::Contradicts; - } - - if similarity_jaccard >= AGREEMENT_THRESHOLD { - return ConflictClassification::Agrees; - } - - if candidate.source_agent == incoming_agent || similarity_jaccard >= RELATED_THRESHOLD { - return ConflictClassification::Refines; - } - - ConflictClassification::Unrelated -} - -fn contradiction_signal(a: &str, b: &str, similarity_jaccard: f64) -> bool { - if similarity_jaccard < RELATED_THRESHOLD { - return false; - } - - let tokens_a = semantic_tokens(a); - let tokens_b = semantic_tokens(b); - - let neg_a = has_negation(&tokens_a); - let neg_b = has_negation(&tokens_b); - if neg_a == neg_b { - return has_polarity_flip(&tokens_a, &tokens_b) && similarity_jaccard >= 0.55; - } - - let core_a = strip_negation_tokens(&tokens_a); - let core_b = strip_negation_tokens(&tokens_b); - let overlap = jaccard_similarity_sets(&core_a, &core_b); - overlap >= CORE_CONTRADICTION_OVERLAP_THRESHOLD -} - -fn semantic_tokens(text: &str) -> HashSet { - text.to_ascii_lowercase() - .split(|ch: char| !ch.is_ascii_alphanumeric()) - .filter(|token| token.len() > 1) - .map(|token| token.to_string()) - .collect() -} - -fn has_negation(tokens: &HashSet) -> bool { - const NEGATION_TOKENS: &[&str] = &[ - "not", - "never", - "no", - "without", - "avoid", - "dont", - "can't", - "cannot", - "disable", - "disabled", - "forbid", - "forbidden", - "against", - ]; - NEGATION_TOKENS.iter().any(|token| tokens.contains(*token)) -} - -fn strip_negation_tokens(tokens: &HashSet) -> HashSet { - const NEGATION_TOKENS: &[&str] = &[ - "not", - "never", - "no", - "without", - "avoid", - "dont", - "can't", - "cannot", - "disable", - "disabled", - "forbid", - "forbidden", - "against", - ]; - tokens - .iter() - .filter(|token| !NEGATION_TOKENS.contains(&token.as_str())) - .cloned() - .collect() -} - -fn has_polarity_flip(tokens_a: &HashSet, tokens_b: &HashSet) -> bool { - const FLIP_PAIRS: &[(&str, &str)] = &[ - ("always", "never"), - ("must", "never"), - ("allow", "forbid"), - ("enable", "disable"), - ("use", "avoid"), - ]; - - FLIP_PAIRS.iter().any(|(lhs, rhs)| { - (tokens_a.contains(*lhs) && tokens_b.contains(*rhs)) - || (tokens_a.contains(*rhs) && tokens_b.contains(*lhs)) - }) -} - -fn jaccard_similarity_sets(left: &HashSet, right: &HashSet) -> f64 { - if left.is_empty() && right.is_empty() { - return 1.0; - } - if left.is_empty() || right.is_empty() { - return 0.0; - } - let intersection = left.intersection(right).count() as f64; - let union = (left.len() + right.len()) as f64 - intersection; - if union == 0.0 { - 0.0 - } else { - intersection / union - } -} - -// ─── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_jaccard_identical() { - assert!(jaccard_similarity("hello world foo", "hello world foo") > 0.99); - } - - #[test] - fn test_jaccard_similar() { - assert!(jaccard_similarity("hello world foo", "hello world bar") > 0.3); - } - - #[test] - fn test_jaccard_different() { - assert!(jaccard_similarity("completely different text", "nothing alike here at all") < 0.1); - } - - #[test] - fn test_jaccard_empty() { - assert_eq!(jaccard_similarity("", ""), 1.0); - } - - #[test] - fn test_detect_conflict() { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - - // Insert a decision - conn.execute( - "INSERT INTO decisions (decision, context, type, source_agent, status) \ - VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![ - "Cortex uses SQLite for storage", - "test", - "decision", - "claude", - "active" - ], - ) - .unwrap(); - - // Same content, same agent => agreement + compatibility update flag - let result = - detect_conflict(&conn, "Cortex uses SQLite for storage", "claude", None).unwrap(); - assert!(result.is_update); - assert!(!result.is_conflict); - assert_eq!(result.classification, ConflictClassification::Agrees); - - // Same content, different agent => agreement - let result = - detect_conflict(&conn, "Cortex uses SQLite for storage", "droid", None).unwrap(); - assert!(!result.is_conflict); - assert!(!result.is_update); - assert_eq!(result.classification, ConflictClassification::Agrees); - - // Contradicting intent => contradiction - let result = detect_conflict(&conn, "Never use SQLite for storage", "droid", None).unwrap(); - assert_eq!(result.classification, ConflictClassification::Contradicts); - assert!(result.is_conflict); - - // Different content => no conflict - let result = - detect_conflict(&conn, "Something totally different and new", "claude", None).unwrap(); - assert!(!result.is_conflict); - assert!(!result.is_update); - assert_eq!(result.classification, ConflictClassification::Unrelated); - } - - #[test] - fn test_detect_conflict_ignores_expired_decisions() { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - - conn.execute( - "INSERT INTO decisions (decision, context, type, source_agent, status, expires_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, datetime('now', '-1 second'))", - rusqlite::params![ - "Cortex uses SQLite for storage", - "test", - "decision", - "claude", - "active" - ], - ) - .unwrap(); - - let result = - detect_conflict(&conn, "Cortex uses SQLite for storage", "claude", None).unwrap(); - assert!(!result.is_conflict); - assert!(!result.is_update); - } - - #[test] - fn test_detect_conflict_scopes_by_owner_when_requested() { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - conn.execute( - "ALTER TABLE decisions ADD COLUMN owner_id INTEGER DEFAULT 0", - [], - ) - .unwrap(); - - conn.execute( - "INSERT INTO decisions (decision, context, type, source_agent, status, owner_id) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - rusqlite::params![ - "Always use sqlite for local memory", - "owner-one", - "decision", - "claude", - "active", - 1_i64 - ], - ) - .unwrap(); - conn.execute( - "INSERT INTO decisions (decision, context, type, source_agent, status, owner_id) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - rusqlite::params![ - "Never use sqlite for local memory", - "owner-two", - "decision", - "droid", - "active", - 2_i64 - ], - ) - .unwrap(); - - let result = detect_conflict( - &conn, - "Always use sqlite for local memory", - "claude", - Some(1), - ) - .unwrap(); - assert_eq!(result.matched_id, Some(1)); - } -} diff --git a/daemon-rs/src/conflict/mod.rs b/daemon-rs/src/conflict/mod.rs new file mode 100644 index 00000000..78be90b7 --- /dev/null +++ b/daemon-rs/src/conflict/mod.rs @@ -0,0 +1,405 @@ +use rusqlite::Connection; +use std::collections::HashSet; +const RELATED_THRESHOLD: f64 = 0.40; +const AGREEMENT_THRESHOLD: f64 = 0.84; +const CORE_CONTRADICTION_OVERLAP_THRESHOLD: f64 = 0.35; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConflictClassification { + Agrees, + Contradicts, + Refines, + Unrelated, +} +impl ConflictClassification { + pub const fn as_str(self) -> &'static str { + match self { + Self::Agrees => "AGREES", + Self::Contradicts => "CONTRADICTS", + Self::Refines => "REFINES", + Self::Unrelated => "UNRELATED", + } + } +} +#[derive(Debug, Clone)] +struct DecisionCandidate { + id: i64, + decision: String, + source_agent: String, + trust_score: f64, +} +#[allow(dead_code)] +pub struct ConflictResult { + pub classification: ConflictClassification, + pub is_conflict: bool, + pub is_update: bool, + pub matched_id: Option, + pub matched_agent: Option, + pub matched_decision: Option, + pub matched_trust_score: Option, + pub similarity_jaccard: f64, + pub similarity_cosine: Option, +} +impl ConflictResult { + fn unrelated() -> Self { + Self { + classification: ConflictClassification::Unrelated, + is_conflict: false, + is_update: false, + matched_id: None, + matched_agent: None, + matched_decision: None, + matched_trust_score: None, + similarity_jaccard: 0.0, + similarity_cosine: None, + } + } + fn from_candidate( + classification: ConflictClassification, candidate: &DecisionCandidate, source_agent: &str, similarity_jaccard: f64, similarity_cosine: Option, + ) -> Self { + let is_conflict = matches!(classification, ConflictClassification::Contradicts); + let is_update = matches!(classification, ConflictClassification::Refines) + || (matches!(classification, ConflictClassification::Agrees) && candidate.source_agent == source_agent); + Self { + classification, + is_conflict, + is_update, + matched_id: Some(candidate.id), + matched_agent: Some(candidate.source_agent.clone()), + matched_decision: Some(candidate.decision.clone()), + matched_trust_score: Some(candidate.trust_score), + similarity_jaccard, + similarity_cosine, + } + } +} +pub fn jaccard_similarity(a: &str, b: &str) -> f64 { + let set_a: HashSet = a.split_whitespace().filter(|w| w.len() > 1).map(|w| w.to_lowercase()).collect(); + let set_b: HashSet = b.split_whitespace().filter(|w| w.len() > 1).map(|w| w.to_lowercase()).collect(); + if set_a.is_empty() && set_b.is_empty() { + return 1.0; + } + if set_a.is_empty() || set_b.is_empty() { + return 0.0; + } + let intersection = set_a.intersection(&set_b).count() as f64; + let union = (set_a.len() + set_b.len()) as f64 - intersection; + if union == 0.0 { + return 0.0; + } + intersection / union +} + +#[derive(Debug, Clone)] +pub(crate) struct RecentDecisionCandidate { + pub(crate) id: i64, + pub(crate) decision: String, + pub(crate) source_agent: String, + pub(crate) trust_score: f64, + pub(crate) in_conflict_window: bool, +} + +pub(crate) struct RecentDecisionScan { + pub(crate) relation: ConflictResult, + pub(crate) max_jaccard: f64, +} + +pub(crate) fn jaccard_token_set(text: &str) -> HashSet { + text.split_whitespace().filter(|word| word.len() > 1).map(|word| word.to_lowercase()).collect() +} + +pub(crate) fn jaccard_similarity_token_sets(left: &HashSet, right: &HashSet) -> f64 { + if left.is_empty() && right.is_empty() { + return 1.0; + } + if left.is_empty() || right.is_empty() { + return 0.0; + } + let intersection = left.intersection(right).count() as f64; + let union = (left.len() + right.len()) as f64 - intersection; + if union == 0.0 { + 0.0 + } else { + intersection / union + } +} + +fn recent_candidate_to_decision_candidate(candidate: &RecentDecisionCandidate) -> DecisionCandidate { + DecisionCandidate { + id: candidate.id, + decision: candidate.decision.clone(), + source_agent: candidate.source_agent.clone(), + trust_score: candidate.trust_score, + } +} + +pub(crate) fn fetch_recent_decision_candidates(conn: &Connection, owner_id: Option) -> Result, String> { + let (sql, has_owner_scope) = if owner_id.is_some() { + ( + "SELECT id, decision, source_agent, trust_score, MAX(in_conflict_window) AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, trust_score, 1 AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) AS trust_score \ + FROM decisions \ + WHERE owner_id = ?1 \ + AND status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY id DESC \ + LIMIT 50 \ + ) \ + UNION ALL \ + SELECT id, decision, source_agent, trust_score, 0 AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) AS trust_score \ + FROM decisions \ + WHERE owner_id = ?1 \ + AND status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY created_at DESC \ + LIMIT 50 \ + ) \ + ) \ + GROUP BY id, decision, source_agent, trust_score \ + ORDER BY in_conflict_window DESC, id DESC", + true, + ) + } else { + ( + "SELECT id, decision, source_agent, trust_score, MAX(in_conflict_window) AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, trust_score, 1 AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) AS trust_score \ + FROM decisions \ + WHERE status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY id DESC \ + LIMIT 50 \ + ) \ + UNION ALL \ + SELECT id, decision, source_agent, trust_score, 0 AS in_conflict_window \ + FROM ( \ + SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) AS trust_score \ + FROM decisions \ + WHERE status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY created_at DESC \ + LIMIT 50 \ + ) \ + ) \ + GROUP BY id, decision, source_agent, trust_score \ + ORDER BY in_conflict_window DESC, id DESC", + false, + ) + }; + let mut stmt = conn.prepare(sql).map_err(|error| format!("Failed to prepare recent decision query: {error}"))?; + let map_candidate = |row: &rusqlite::Row<'_>| { + let in_conflict_window: i64 = row.get(4)?; + Ok(RecentDecisionCandidate { + id: row.get(0)?, + decision: row.get(1)?, + source_agent: row.get(2)?, + trust_score: row.get(3)?, + in_conflict_window: in_conflict_window != 0, + }) + }; + if has_owner_scope { + let candidates = stmt + .query_map([owner_id.unwrap_or_default()], map_candidate) + .map_err(|error| format!("Failed to query recent decisions: {error}"))? + .filter_map(|row| row.ok()) + .collect::>(); + Ok(candidates) + } else { + let candidates = stmt + .query_map([], map_candidate) + .map_err(|error| format!("Failed to query recent decisions: {error}"))? + .filter_map(|row| row.ok()) + .collect::>(); + Ok(candidates) + } +} + +pub(crate) fn scan_recent_decision_candidates( + candidates: &[RecentDecisionCandidate], decision: &str, source_agent: &str, decision_tokens: &HashSet, +) -> RecentDecisionScan { + let mut max_jaccard = 0.0_f64; + let mut best_conflict_sim = 0.0_f64; + let mut best_conflict_candidate: Option = None; + + for candidate in candidates { + let candidate_tokens = jaccard_token_set(&candidate.decision); + let similarity = jaccard_similarity_token_sets(decision_tokens, &candidate_tokens); + max_jaccard = max_jaccard.max(similarity); + if candidate.in_conflict_window && similarity > best_conflict_sim { + best_conflict_sim = similarity; + best_conflict_candidate = Some(recent_candidate_to_decision_candidate(candidate)); + } + } + + let Some(best_candidate) = best_conflict_candidate else { + return RecentDecisionScan { relation: ConflictResult::unrelated(), max_jaccard }; + }; + if best_conflict_sim < RELATED_THRESHOLD { + return RecentDecisionScan { relation: ConflictResult::unrelated(), max_jaccard }; + } + let classification = classify_relation(decision, source_agent, &best_candidate, best_conflict_sim); + RecentDecisionScan { + relation: ConflictResult::from_candidate(classification, &best_candidate, source_agent, best_conflict_sim, None), + max_jaccard, + } +} + +#[allow(dead_code)] +pub fn detect_conflict(conn: &Connection, decision: &str, source_agent: &str, owner_id: Option) -> Result { + let (sql, has_owner_scope) = if owner_id.is_some() { + ( + "SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) \ + FROM decisions \ + WHERE owner_id = ?1 \ + AND status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY id DESC \ + LIMIT 50", + true, + ) + } else { + ( + "SELECT id, decision, source_agent, COALESCE(trust_score, confidence, 0.8) \ + FROM decisions \ + WHERE status = 'active' \ + AND (expires_at IS NULL OR expires_at > datetime('now')) \ + ORDER BY id DESC \ + LIMIT 50", + false, + ) + }; + let mut stmt = conn.prepare(sql).map_err(|e| format!("Failed to prepare conflict query: {e}"))?; + let rows: Vec = if has_owner_scope { + stmt.query_map([owner_id.unwrap_or_default()], |row| { + Ok(DecisionCandidate { id: row.get(0)?, decision: row.get(1)?, source_agent: row.get(2)?, trust_score: row.get(3)? }) + }) + .map_err(|e| format!("Failed to query decisions: {e}"))? + .filter_map(|r| r.ok()) + .collect() + } else { + stmt.query_map([], |row| Ok(DecisionCandidate { id: row.get(0)?, decision: row.get(1)?, source_agent: row.get(2)?, trust_score: row.get(3)? })) + .map_err(|e| format!("Failed to query decisions: {e}"))? + .filter_map(|r| r.ok()) + .collect() + }; + let mut best_sim = 0.0_f64; + let mut best_candidate: Option = None; + for candidate in &rows { + let sim = jaccard_similarity(decision, &candidate.decision); + if sim > best_sim { + best_sim = sim; + best_candidate = Some(candidate.clone()); + } + } + let Some(best_candidate) = best_candidate else { + return Ok(ConflictResult::unrelated()); + }; + if best_sim < RELATED_THRESHOLD { + return Ok(ConflictResult::unrelated()); + } + let classification = classify_relation(decision, source_agent, &best_candidate, best_sim); + Ok(ConflictResult::from_candidate(classification, &best_candidate, source_agent, best_sim, None)) +} +fn classify_relation(incoming_decision: &str, incoming_agent: &str, candidate: &DecisionCandidate, similarity_jaccard: f64) -> ConflictClassification { + if similarity_jaccard < RELATED_THRESHOLD { + return ConflictClassification::Unrelated; + } + if contradiction_signal(incoming_decision, &candidate.decision, similarity_jaccard) { + return ConflictClassification::Contradicts; + } + if similarity_jaccard >= AGREEMENT_THRESHOLD { + return ConflictClassification::Agrees; + } + if candidate.source_agent == incoming_agent || similarity_jaccard >= RELATED_THRESHOLD { + return ConflictClassification::Refines; + } + ConflictClassification::Unrelated +} +fn contradiction_signal(a: &str, b: &str, similarity_jaccard: f64) -> bool { + if similarity_jaccard < RELATED_THRESHOLD { + return false; + } + let tokens_a = semantic_tokens(a); + let tokens_b = semantic_tokens(b); + let neg_a = has_negation(&tokens_a); + let neg_b = has_negation(&tokens_b); + if neg_a == neg_b { + return has_polarity_flip(&tokens_a, &tokens_b) && similarity_jaccard >= 0.55; + } + let core_a = strip_negation_tokens(&tokens_a); + let core_b = strip_negation_tokens(&tokens_b); + let overlap = jaccard_similarity_sets(&core_a, &core_b); + overlap >= CORE_CONTRADICTION_OVERLAP_THRESHOLD +} +fn semantic_tokens(text: &str) -> HashSet { + text.to_ascii_lowercase() + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| token.len() > 1) + .map(|token| token.to_string()) + .collect() +} +fn has_negation(tokens: &HashSet) -> bool { + const NEGATION_TOKENS: &[&str] = &[ + "not", + "never", + "no", + "without", + "avoid", + "dont", + "can't", + "cannot", + "disable", + "disabled", + "forbid", + "forbidden", + "against", + ]; + NEGATION_TOKENS.iter().any(|token| tokens.contains(*token)) +} +fn strip_negation_tokens(tokens: &HashSet) -> HashSet { + const NEGATION_TOKENS: &[&str] = &[ + "not", + "never", + "no", + "without", + "avoid", + "dont", + "can't", + "cannot", + "disable", + "disabled", + "forbid", + "forbidden", + "against", + ]; + tokens.iter().filter(|token| !NEGATION_TOKENS.contains(&token.as_str())).cloned().collect() +} +fn has_polarity_flip(tokens_a: &HashSet, tokens_b: &HashSet) -> bool { + const FLIP_PAIRS: &[(&str, &str)] = &[("always", "never"), ("must", "never"), ("allow", "forbid"), ("enable", "disable"), ("use", "avoid")]; + FLIP_PAIRS + .iter() + .any(|(lhs, rhs)| (tokens_a.contains(*lhs) && tokens_b.contains(*rhs)) || (tokens_a.contains(*rhs) && tokens_b.contains(*lhs))) +} +fn jaccard_similarity_sets(left: &HashSet, right: &HashSet) -> f64 { + if left.is_empty() && right.is_empty() { + return 1.0; + } + if left.is_empty() || right.is_empty() { + return 0.0; + } + let intersection = left.intersection(right).count() as f64; + let union = (left.len() + right.len()) as f64 - intersection; + if union == 0.0 { + 0.0 + } else { + intersection / union + } +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/conflict/tests/mod.rs b/daemon-rs/src/conflict/tests/mod.rs new file mode 100644 index 00000000..26e63cf4 --- /dev/null +++ b/daemon-rs/src/conflict/tests/mod.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +#[test] +fn test_jaccard_identical() { + assert!(jaccard_similarity("hello world foo", "hello world foo") > 0.99); +} +#[test] +fn test_jaccard_similar() { + assert!(jaccard_similarity("hello world foo", "hello world bar") > 0.3); +} +#[test] +fn test_jaccard_different() { + assert!(jaccard_similarity("completely different text", "nothing alike here at all") < 0.1); +} +#[test] +fn test_jaccard_empty() { + assert_eq!(jaccard_similarity("", ""), 1.0); +} +#[test] +fn test_detect_conflict() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + conn.execute( + "INSERT INTO decisions (decision, context, type, source_agent, status) \ + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params!["Cortex uses SQLite for storage", "test", "decision", "claude", "active"], + ) + .unwrap(); + let result = detect_conflict(&conn, "Cortex uses SQLite for storage", "claude", None).unwrap(); + assert!(result.is_update); + assert!(!result.is_conflict); + assert_eq!(result.classification, ConflictClassification::Agrees); + let result = detect_conflict(&conn, "Cortex uses SQLite for storage", "droid", None).unwrap(); + assert!(!result.is_conflict); + assert!(!result.is_update); + assert_eq!(result.classification, ConflictClassification::Agrees); + let result = detect_conflict(&conn, "Never use SQLite for storage", "droid", None).unwrap(); + assert_eq!(result.classification, ConflictClassification::Contradicts); + assert!(result.is_conflict); + let result = detect_conflict(&conn, "Something totally different and new", "claude", None).unwrap(); + assert!(!result.is_conflict); + assert!(!result.is_update); + assert_eq!(result.classification, ConflictClassification::Unrelated); +} +#[test] +fn test_detect_conflict_ignores_expired_decisions() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + conn.execute( + "INSERT INTO decisions (decision, context, type, source_agent, status, expires_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, datetime('now', '-1 second'))", + rusqlite::params!["Cortex uses SQLite for storage", "test", "decision", "claude", "active"], + ) + .unwrap(); + let result = detect_conflict(&conn, "Cortex uses SQLite for storage", "claude", None).unwrap(); + assert!(!result.is_conflict); + assert!(!result.is_update); +} +#[test] +fn test_detect_conflict_scopes_by_owner_when_requested() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + conn.execute("ALTER TABLE decisions ADD COLUMN owner_id INTEGER DEFAULT 0", []).unwrap(); + conn.execute( + "INSERT INTO decisions (decision, context, type, source_agent, status, owner_id) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params!["Always use sqlite for local memory", "owner-one", "decision", "claude", "active", 1_i64], + ) + .unwrap(); + conn.execute( + "INSERT INTO decisions (decision, context, type, source_agent, status, owner_id) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params!["Never use sqlite for local memory", "owner-two", "decision", "droid", "active", 2_i64], + ) + .unwrap(); + let result = detect_conflict(&conn, "Always use sqlite for local memory", "claude", Some(1)).unwrap(); + assert_eq!(result.matched_id, Some(1)); +} diff --git a/daemon-rs/src/crystallize.rs b/daemon-rs/src/crystallize.rs deleted file mode 100644 index 3a32cc9c..00000000 --- a/daemon-rs/src/crystallize.rs +++ /dev/null @@ -1,894 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Knowledge Crystallization Engine -//! -//! Detects clusters of semantically related memories/decisions and synthesizes -//! them into consolidated "crystal" nodes. Creates a two-tier recall hierarchy: -//! -//! Tier 1: Crystal nodes (dense, high-signal, searched first) -//! Tier 2: Source memories (full detail, accessed via unfold) -//! -//! Algorithm: -//! 1. Load all active embeddings -//! 2. Greedy clustering: cosine similarity > CLUSTER_THRESHOLD -//! 3. Extractive synthesis: best sentence from each member, deduped -//! 4. Store crystal + member links -//! 5. Recall searches crystals with a relevance boost -//! -//! Runs as a background job (like aging). No LLM dependency -- pure embeddings -//! + extractive text synthesis. The same zero-runtime-dep architecture. - -use rusqlite::{params, Connection}; -use serde_json::json; -use std::collections::{HashMap, HashSet}; -use tokio::sync::broadcast; - -use crate::embeddings::{self, EmbeddingEngine}; -use crate::state::{BrainFiringEvent, BrainKind}; - -/// Optional Brain telemetry sender threaded into `run_crystallize_pass`. -/// Existing callers pass `None`; the live daemon passes `Some(state.brain_firing.clone())`. -pub type BrainFiringSender = Option>; - -fn emit_brain( - sender: &BrainFiringSender, - kind: BrainKind, - payload: serde_json::Value, - owner_id: Option, -) { - if let Some(tx) = sender { - let _ = tx.send(BrainFiringEvent { - kind, - payload, - owner_id, - }); - } -} - -// ─── Constants ────────────────────────────────────────────────────────────── - -/// Minimum cosine similarity to join a cluster. -const CLUSTER_THRESHOLD: f32 = 0.70; - -/// Minimum cluster size to warrant crystallization. -const MIN_CLUSTER_SIZE: usize = 3; - -/// Maximum sentences in a crystal summary. -const MAX_CRYSTAL_SENTENCES: usize = 8; - -/// Maximum characters in a crystal summary. -const MAX_CRYSTAL_CHARS: usize = 600; - -/// Jaccard threshold for sentence deduplication. -const DEDUP_JACCARD: f64 = 0.5; - -/// Relevance boost applied to crystal nodes during recall. -pub const CRYSTAL_RELEVANCE_BOOST: f64 = 1.15; - -fn is_missing_team_visibility_columns(err: &rusqlite::Error) -> bool { - let normalized = err.to_string().to_ascii_lowercase(); - normalized.contains("no such column") - && (normalized.contains("owner_id") || normalized.contains("visibility")) -} - -// ─── Types ────────────────────────────────────────────────────────────────── - -#[derive(Clone)] -struct EmbeddedEntry { - target_type: String, - target_id: i64, - vector: Vec, - #[allow(dead_code)] - source: String, - text: String, -} - -#[derive(Debug)] -pub struct CrystallizeResult { - pub clusters_found: usize, - pub crystals_created: usize, - pub crystals_updated: usize, - pub entries_consolidated: usize, -} - -// ─── Main entry point ─────────────────────────────────────────────────────── - -pub fn run_crystallize_pass_with_brain( - conn: &Connection, - engine: Option<&EmbeddingEngine>, - owner_id: Option, - brain: &BrainFiringSender, -) -> CrystallizeResult { - let mut result = CrystallizeResult { - clusters_found: 0, - crystals_created: 0, - crystals_updated: 0, - entries_consolidated: 0, - }; - - // 1. Load all active entries with embeddings - let entries = load_embedded_entries(conn); - if entries.len() < MIN_CLUSTER_SIZE { - return result; - } - - emit_brain(brain, BrainKind::ConsolidationStarted, json!({}), owner_id); - - // 2. Greedy clustering - let clusters = cluster_entries(&entries); - result.clusters_found = clusters.len(); - - if clusters.is_empty() { - return result; - } - - // 3. For each cluster, synthesize and store - for cluster in &clusters { - let member_entries: Vec<&EmbeddedEntry> = - cluster.iter().map(|&idx| &entries[idx]).collect(); - - result.entries_consolidated += member_entries.len(); - - // Generate label from most common words across members - let label = generate_cluster_label(&member_entries); - - // Extractive synthesis - let consolidated_text = synthesize_crystal(&member_entries); - - // Compute centroid embedding - let centroid = compute_centroid( - &member_entries - .iter() - .map(|e| e.vector.as_slice()) - .collect::>(), - ); - let centroid_blob = embeddings::vector_to_blob(¢roid); - - // Check if a crystal already exists for this cluster (by label overlap) - let existing_id = find_existing_crystal(conn, &label); - - match existing_id { - Some(crystal_id) => { - // Update existing crystal - let _ = conn.execute( - "UPDATE memory_clusters SET consolidated_text = ?1, centroid = ?2, \ - member_count = ?3, updated_at = datetime('now') WHERE id = ?4", - params![ - consolidated_text, - centroid_blob, - member_entries.len() as i64, - crystal_id - ], - ); - update_cluster_members(conn, crystal_id, &member_entries); - for member in &member_entries { - emit_brain( - brain, - BrainKind::MemberAdded, - json!({ - "cluster_id": crystal_id, - "member_id": format!("{}-{}", member.target_type, member.target_id), - }), - owner_id, - ); - } - emit_brain( - brain, - BrainKind::ClusterFinalized, - json!({ - "cluster_id": crystal_id, - "member_count": member_entries.len() as i64, - }), - owner_id, - ); - result.crystals_updated += 1; - } - None => { - // Create new crystal - if let Some(oid) = owner_id { - let _ = conn.execute( - "INSERT INTO memory_clusters (label, centroid, consolidated_text, member_count, owner_id, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'), datetime('now'))", - params![label, centroid_blob, consolidated_text, member_entries.len() as i64, oid], - ); - } else { - let _ = conn.execute( - "INSERT INTO memory_clusters (label, centroid, consolidated_text, member_count, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, datetime('now'), datetime('now'))", - params![label, centroid_blob, consolidated_text, member_entries.len() as i64], - ); - } - let crystal_id = conn.last_insert_rowid(); - update_cluster_members(conn, crystal_id, &member_entries); - for member in &member_entries { - emit_brain( - brain, - BrainKind::MemberAdded, - json!({ - "cluster_id": crystal_id, - "member_id": format!("{}-{}", member.target_type, member.target_id), - }), - owner_id, - ); - } - emit_brain( - brain, - BrainKind::ClusterFinalized, - json!({ - "cluster_id": crystal_id, - "member_count": member_entries.len() as i64, - }), - owner_id, - ); - - // Embed the crystal text for recall - if let Some(eng) = engine { - if let Some(vec) = eng.embed(&consolidated_text) { - let blob = embeddings::vector_to_blob(&vec); - let model_key = eng.model_key(); - let _ = conn.execute( - "INSERT OR REPLACE INTO embeddings (target_type, target_id, vector, model) \ - VALUES ('crystal', ?1, ?2, ?3)", - params![crystal_id, blob, model_key], - ); - } - } - - result.crystals_created += 1; - } - } - } - - if result.crystals_created > 0 || result.crystals_updated > 0 { - eprintln!( - "[crystallize] Pass complete: {} clusters, {} created, {} updated, {} entries consolidated", - result.clusters_found, - result.crystals_created, - result.crystals_updated, - result.entries_consolidated - ); - } - - result -} - -// ─── Load entries ─────────────────────────────────────────────────────────── - -fn load_embedded_entries(conn: &Connection) -> Vec { - let mut entries = Vec::new(); - - // Load memories - if let Ok(mut stmt) = conn.prepare( - "SELECT e.target_id, e.vector, m.text, m.source \ - FROM embeddings e \ - JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id \ - WHERE m.status = 'active'", - ) { - let rows: Vec<(i64, Vec, String, Option)> = stmt - .query_map([], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) - }) - .into_iter() - .flatten() - .flatten() - .collect(); - - for (id, blob, text, source) in rows { - entries.push(EmbeddedEntry { - target_type: "memory".to_string(), - target_id: id, - vector: embeddings::blob_to_vector(&blob), - source: source.unwrap_or_else(|| format!("memory::{id}")), - text, - }); - } - } - - // Load decisions - if let Ok(mut stmt) = conn.prepare( - "SELECT e.target_id, e.vector, d.decision, d.context \ - FROM embeddings e \ - JOIN decisions d ON e.target_type = 'decision' AND e.target_id = d.id \ - WHERE d.status = 'active'", - ) { - let rows: Vec<(i64, Vec, String, Option)> = stmt - .query_map([], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) - }) - .into_iter() - .flatten() - .flatten() - .collect(); - - for (id, blob, decision, context) in rows { - entries.push(EmbeddedEntry { - target_type: "decision".to_string(), - target_id: id, - vector: embeddings::blob_to_vector(&blob), - source: context.unwrap_or_else(|| format!("decision::{id}")), - text: decision, - }); - } - } - - entries -} - -// ─── Clustering ───────────────────────────────────────────────────────────── - -/// Greedy single-linkage clustering. Returns clusters as vectors of indices. -fn cluster_entries(entries: &[EmbeddedEntry]) -> Vec> { - let n = entries.len(); - let mut assigned = vec![false; n]; - let mut clusters: Vec> = Vec::new(); - - for i in 0..n { - if assigned[i] { - continue; - } - - let mut cluster = vec![i]; - assigned[i] = true; - - // Find all entries similar to the seed - for j in (i + 1)..n { - if assigned[j] { - continue; - } - let sim = embeddings::cosine_similarity(&entries[i].vector, &entries[j].vector); - if sim >= CLUSTER_THRESHOLD { - cluster.push(j); - assigned[j] = true; - } - } - - if cluster.len() >= MIN_CLUSTER_SIZE { - clusters.push(cluster); - } - } - - clusters -} - -/// Compute the centroid (average) of a set of vectors. -fn compute_centroid(vectors: &[&[f32]]) -> Vec { - if vectors.is_empty() { - return vec![]; - } - let dim = vectors[0].len(); - let mut centroid = vec![0.0f32; dim]; - for vec in vectors { - for (i, &v) in vec.iter().enumerate() { - centroid[i] += v; - } - } - let n = vectors.len() as f32; - for v in &mut centroid { - *v /= n; - } - // L2 normalize - let norm: f32 = centroid.iter().map(|x| x * x).sum::().sqrt(); - if norm > 0.0 { - for v in &mut centroid { - *v /= norm; - } - } - centroid -} - -// ─── Extractive synthesis ─────────────────────────────────────────────────── - -/// High-signal keywords that indicate important content. -const SIGNAL_WORDS: &[&str] = &[ - "must", - "never", - "always", - "critical", - "important", - "decision", - "fixed", - "bug", - "error", - "confirmed", - "approved", - "rejected", - "architecture", - "design", - "migration", - "breaking", - "security", - "performance", - "prefer", - "avoid", - "use", - "requires", - "deprecated", - "instead", -]; - -/// Synthesize a crystal summary from cluster members using extractive methods. -fn synthesize_crystal(members: &[&EmbeddedEntry]) -> String { - // Extract best sentence from each member - let mut candidates: Vec<(String, f64)> = Vec::new(); - - for entry in members { - let sentences: Vec<&str> = entry - .text - .split(['.', '\n']) - .map(|s| s.trim()) - .filter(|s| s.len() > 10) - .collect(); - - if sentences.is_empty() { - // Use truncated full text if no sentences - let trunc: String = entry.text.chars().take(80).collect(); - candidates.push((trunc, 1.0)); - continue; - } - - // Score each sentence by signal word density - let mut best_sentence = sentences[0].to_string(); - let mut best_score = 0.0f64; - - for sentence in &sentences { - let lower = sentence.to_lowercase(); - let word_count = lower.split_whitespace().count().max(1) as f64; - let signal_count = SIGNAL_WORDS - .iter() - .filter(|kw| lower.contains(**kw)) - .count() as f64; - let score = signal_count / word_count - + if sentence == sentences.first().unwrap() { - 0.1 - } else { - 0.0 - }; - - if score > best_score { - best_score = score; - best_sentence = sentence.to_string(); - } - } - - candidates.push((best_sentence, best_score)); - } - - // Sort by score (best first), dedup, take top N - candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - - let mut kept: Vec = Vec::new(); - for (sentence, _score) in &candidates { - if kept.len() >= MAX_CRYSTAL_SENTENCES { - break; - } - // Dedup: skip if too similar to an already-kept sentence - let dominated = kept - .iter() - .any(|existing| jaccard_words(existing, sentence) > DEDUP_JACCARD); - if !dominated { - kept.push(sentence.clone()); - } - } - - // Join with periods, cap at MAX_CRYSTAL_CHARS - let mut result = kept.join(". "); - if result.len() > MAX_CRYSTAL_CHARS { - result = result.chars().take(MAX_CRYSTAL_CHARS).collect::(); - result.push_str("..."); - } - - result -} - -/// Word-level Jaccard similarity between two strings. -fn jaccard_words(a: &str, b: &str) -> f64 { - let set_a: HashSet<&str> = a - .split_whitespace() - .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric())) - .filter(|w| w.len() > 2) - .collect(); - let set_b: HashSet<&str> = b - .split_whitespace() - .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric())) - .filter(|w| w.len() > 2) - .collect(); - let intersection = set_a.intersection(&set_b).count() as f64; - let union = set_a.union(&set_b).count() as f64; - if union == 0.0 { - 0.0 - } else { - intersection / union - } -} - -// ─── Label generation ─────────────────────────────────────────────────────── - -/// Generate a human-readable label for a cluster from its members' text. -/// Uses TF-IDF-like scoring: words frequent in the cluster but not universal. -fn generate_cluster_label(members: &[&EmbeddedEntry]) -> String { - let stop_words: HashSet<&str> = [ - "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", - "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can", - "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into", "about", "like", - "through", "after", "over", "between", "out", "up", "and", "but", "or", "not", "no", "if", - "then", "than", "that", "this", "it", "its", "all", "each", "every", "both", "few", "more", - "most", "other", "some", "such", "only", "own", "same", "so", "too", "very", "just", - "because", "when", "where", "how", "what", "which", "who", "whom", "why", "use", "using", - "used", - ] - .iter() - .cloned() - .collect(); - - let mut word_freq: HashMap = HashMap::new(); - let mut doc_freq: HashMap = HashMap::new(); - - for entry in members { - let mut seen_in_doc: HashSet = HashSet::new(); - for word in entry - .text - .to_lowercase() - .split(|c: char| !c.is_alphanumeric()) - { - let w = word.trim(); - if w.len() < 3 || stop_words.contains(w) { - continue; - } - *word_freq.entry(w.to_string()).or_insert(0) += 1; - if seen_in_doc.insert(w.to_string()) { - *doc_freq.entry(w.to_string()).or_insert(0) += 1; - } - } - } - - let n = members.len() as f64; - let mut scored: Vec<(String, f64)> = word_freq - .into_iter() - .filter(|(word, _)| { - let df = *doc_freq.get(word).unwrap_or(&0) as f64; - // Appears in at least 40% of members (cluster-characteristic) - df / n >= 0.4 - }) - .map(|(word, freq)| { - let df = *doc_freq.get(&word).unwrap_or(&1) as f64; - let tf_idf = freq as f64 * (n / df).ln().max(0.1); - (word, tf_idf) - }) - .collect(); - - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - - let top_words: Vec<&str> = scored.iter().take(4).map(|(w, _)| w.as_str()).collect(); - if top_words.is_empty() { - "misc".to_string() - } else { - top_words.join("-") - } -} - -// ─── Database helpers ─────────────────────────────────────────────────────── - -fn find_existing_crystal(conn: &Connection, label: &str) -> Option { - conn.query_row( - "SELECT id FROM memory_clusters WHERE label = ?1", - params![label], - |row| row.get(0), - ) - .ok() -} - -fn update_cluster_members(conn: &Connection, crystal_id: i64, members: &[&EmbeddedEntry]) { - // Clear old members - let _ = conn.execute( - "DELETE FROM cluster_members WHERE cluster_id = ?1", - params![crystal_id], - ); - - // Insert new members - for entry in members { - let _ = conn.execute( - "INSERT OR IGNORE INTO cluster_members (cluster_id, target_type, target_id, similarity) \ - VALUES (?1, ?2, ?3, ?4)", - params![crystal_id, entry.target_type, entry.target_id, 1.0], - ); - } -} - -// ─── Recall integration ───────────────────────────────────────────────────── - -/// Search crystal nodes by semantic similarity. Returns (crystal_id, label, -/// consolidated_text, similarity) sorted by relevance. -/// Crystal search with optional visibility filtering for team mode. -#[allow(clippy::type_complexity)] -pub fn search_crystals_filtered( - conn: &Connection, - query_vec: &[f32], - limit: usize, - caller_id: Option, - team_mode: bool, -) -> Vec<(i64, String, String, f64)> { - let query_rows = |sql: &str, - with_visibility: bool| - -> Result< - Vec<(i64, Vec, String, String, Option, Option)>, - rusqlite::Error, - > { - let mut stmt = conn.prepare(sql)?; - let mapped = stmt.query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, Vec>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - if with_visibility { - row.get::<_, Option>(4)? - } else { - None - }, - if with_visibility { - row.get::<_, Option>(5)? - } else { - None - }, - )) - })?; - Ok(mapped.flatten().collect()) - }; - - let sql_with_visibility = - "SELECT mc.id, e.vector, mc.label, mc.consolidated_text, mc.owner_id, mc.visibility \ - FROM embeddings e \ - JOIN memory_clusters mc ON e.target_type = 'crystal' AND e.target_id = mc.id"; - let sql_legacy = "SELECT mc.id, e.vector, mc.label, mc.consolidated_text \ - FROM embeddings e \ - JOIN memory_clusters mc ON e.target_type = 'crystal' AND e.target_id = mc.id"; - - let rows = match query_rows(sql_with_visibility, true) { - Ok(rows) => rows, - Err(err) if is_missing_team_visibility_columns(&err) => { - query_rows(sql_legacy, false).unwrap_or_default() - } - Err(_) => Vec::new(), - }; - - let mut results: Vec<(i64, String, String, f64)> = rows - .into_iter() - .filter_map(|(id, blob, label, text, owner_id, visibility)| { - // Visibility: solo mode sees everything; team mode fails closed - if team_mode { - let caller = match caller_id { - Some(c) => c, - None => return None, // fail closed: unidentified caller - }; - let owner = match owner_id { - Some(o) => o, - None => return None, // fail closed: unowned data - }; - if owner != caller - && !matches!(visibility.as_deref(), Some("shared") | Some("team")) - { - return None; - } - } - let vec = embeddings::blob_to_vector(&blob); - let sim = embeddings::cosine_similarity(query_vec, &vec) as f64; - if sim > 0.3 { - Some((id, label, text, sim * CRYSTAL_RELEVANCE_BOOST)) - } else { - None - } - }) - .collect(); - - results.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap_or(std::cmp::Ordering::Equal)); - results.truncate(limit); - results -} - -/// Unfold a crystal: return its member sources for detailed retrieval. -pub fn unfold_crystal(conn: &Connection, crystal_id: i64) -> Vec { - conn.prepare( - "SELECT cm.target_type, cm.target_id, \ - CASE WHEN cm.target_type = 'memory' THEN m.source \ - ELSE COALESCE(d.context, 'decision::' || d.id) END as source \ - FROM cluster_members cm \ - LEFT JOIN memories m ON cm.target_type = 'memory' AND cm.target_id = m.id \ - LEFT JOIN decisions d ON cm.target_type = 'decision' AND cm.target_id = d.id \ - WHERE cm.cluster_id = ?1", - ) - .and_then(|mut stmt| { - let rows = stmt.query_map(params![crystal_id], |row| row.get::<_, String>(2))?; - Ok(rows.flatten().collect()) - }) - .unwrap_or_default() -} - -// ─── GET /crystals ────────────────────────────────────────────────────────── - -/// List all crystals with their stats. -pub fn list_crystals(conn: &Connection) -> Vec { - conn.prepare( - "SELECT id, label, consolidated_text, member_count, created_at, updated_at \ - FROM memory_clusters ORDER BY updated_at DESC", - ) - .and_then(|mut stmt| { - let rows = stmt.query_map([], |row| { - Ok(serde_json::json!({ - "id": row.get::<_, i64>(0)?, - "label": row.get::<_, String>(1)?, - "text": row.get::<_, String>(2)?, - "members": row.get::<_, i64>(3)?, - "created": row.get::<_, String>(4)?, - "updated": row.get::<_, String>(5)?, - })) - })?; - Ok(rows.flatten().collect()) - }) - .unwrap_or_default() -} - -// ─── Tests ────────────────────────────────────────────────────────────────── - -#[cfg(test)] -#[allow(clippy::items_after_test_module)] -mod tests { - use super::*; - - #[test] - fn test_jaccard_identical() { - assert!((jaccard_words("hello world test", "hello world test") - 1.0).abs() < 0.001); - } - - #[test] - fn test_jaccard_different() { - let sim = jaccard_words("the quick brown fox", "lazy purple elephant jumps"); - assert!(sim < 0.2); - } - - #[test] - fn test_jaccard_partial() { - let sim = jaccard_words("use python for backend", "use python for frontend"); - assert!(sim > 0.5, "Shared 'use python for' should give >0.5"); - } - - #[test] - fn test_compute_centroid() { - let v1 = [1.0, 0.0, 0.0]; - let v2 = [0.0, 1.0, 0.0]; - let centroid = compute_centroid(&[&v1[..], &v2[..]]); - // Average is [0.5, 0.5, 0] normalized ≈ [0.707, 0.707, 0] - assert!(centroid[0] > 0.6 && centroid[0] < 0.8); - assert!(centroid[1] > 0.6 && centroid[1] < 0.8); - assert!(centroid[2].abs() < 0.001); - } - - #[test] - fn test_generate_label_empty() { - let entries: Vec<&EmbeddedEntry> = vec![]; - assert_eq!(generate_cluster_label(&entries), "misc"); - } - - #[test] - fn test_synthesize_deduplicates() { - let e1 = EmbeddedEntry { - target_type: "memory".to_string(), - target_id: 1, - vector: vec![], - source: "test1".to_string(), - text: "Always use uv for Python package management. Never use pip directly." - .to_string(), - }; - let e2 = EmbeddedEntry { - target_type: "memory".to_string(), - target_id: 2, - vector: vec![], - source: "test2".to_string(), - text: "Use uv for Python package management instead of pip.".to_string(), - }; - let e3 = EmbeddedEntry { - target_type: "memory".to_string(), - target_id: 3, - vector: vec![], - source: "test3".to_string(), - text: "Python type hints required on all function signatures.".to_string(), - }; - let members: Vec<&EmbeddedEntry> = vec![&e1, &e2, &e3]; - let result = synthesize_crystal(&members); - // Should not have two near-identical uv/pip sentences - let period_count = result.matches('.').count(); - assert!( - period_count <= 3, - "Should deduplicate similar sentences, got: {result}" - ); - } - - #[test] - fn test_cluster_entries_basic() { - // Two groups of identical vectors should form two clusters - let make_entry = |id: i64, vec: Vec| EmbeddedEntry { - target_type: "memory".to_string(), - target_id: id, - vector: vec, - source: format!("test::{id}"), - text: format!("Entry {id}"), - }; - - let entries = vec![ - make_entry(1, vec![1.0, 0.0, 0.0]), - make_entry(2, vec![0.98, 0.1, 0.0]), - make_entry(3, vec![0.95, 0.15, 0.0]), - make_entry(4, vec![0.0, 1.0, 0.0]), - make_entry(5, vec![0.1, 0.98, 0.0]), - make_entry(6, vec![0.15, 0.95, 0.0]), - ]; - - let clusters = cluster_entries(&entries); - assert_eq!(clusters.len(), 2, "Should find 2 clusters"); - } - - #[test] - fn test_full_crystallize_pass() { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - migrate_crystal_tables(&conn); - - // Insert test memories with embeddings that cluster together - for i in 1..=4 { - conn.execute( - "INSERT INTO memories (id, text, source, type, status) VALUES (?1, ?2, ?3, 'memory', 'active')", - params![i, format!("Python requires uv for package management rule {i}"), format!("test::python_{i}")], - ).unwrap(); - // All get similar embeddings (near [1,0,0...]) - let mut vec = vec![0.0f32; 384]; - vec[0] = 1.0; - vec[1] = 0.01 * i as f32; // slight variation - let blob = embeddings::vector_to_blob(&vec); - conn.execute( - "INSERT INTO embeddings (target_type, target_id, vector) VALUES ('memory', ?1, ?2)", - params![i, blob], - ) - .unwrap(); - } - - let result = run_crystallize_pass_with_brain(&conn, None, None, &None); - assert_eq!(result.clusters_found, 1); - assert_eq!(result.crystals_created, 1); - assert_eq!(result.entries_consolidated, 4); - - // Verify crystal exists - let crystals = list_crystals(&conn); - assert_eq!(crystals.len(), 1); - assert!(crystals[0]["members"].as_i64().unwrap() >= 4); - } -} - -// ─── Schema migration ─────────────────────────────────────────────────────── - -pub fn migrate_crystal_tables(conn: &Connection) { - let sql = r#" - CREATE TABLE IF NOT EXISTS memory_clusters ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - label TEXT NOT NULL, - centroid BLOB, - consolidated_text TEXT NOT NULL, - member_count INTEGER DEFAULT 0, - created_at TEXT DEFAULT (datetime('now')), - updated_at TEXT DEFAULT (datetime('now')) - ); - - CREATE TABLE IF NOT EXISTS cluster_members ( - cluster_id INTEGER NOT NULL, - target_type TEXT NOT NULL, - target_id INTEGER NOT NULL, - similarity REAL NOT NULL DEFAULT 1.0, - PRIMARY KEY (cluster_id, target_type, target_id), - FOREIGN KEY (cluster_id) REFERENCES memory_clusters(id) ON DELETE CASCADE - ); - - CREATE INDEX IF NOT EXISTS idx_cluster_members_target ON cluster_members(target_type, target_id); - "#; - match conn.execute_batch(sql) { - Ok(_) => {} - Err(e) => eprintln!("[db] Crystal table migration: {e}"), - } -} diff --git a/daemon-rs/src/crystallize/mod.rs b/daemon-rs/src/crystallize/mod.rs new file mode 100644 index 00000000..7178a204 --- /dev/null +++ b/daemon-rs/src/crystallize/mod.rs @@ -0,0 +1,110 @@ +use crate::embeddings::EmbeddingEngine; +use crate::state::{BrainFiringEvent, BrainKind}; +use rusqlite::{params, Connection}; +use serde_json::{json, Value}; +use tokio::sync::broadcast; + +pub type BrainFiringSender = Option>; +pub const CRYSTAL_RELEVANCE_BOOST: f64 = 1.15; + +#[derive(Debug)] +pub struct CrystallizeResult { + pub clusters_found: usize, + pub crystals_created: usize, + pub crystals_updated: usize, + pub entries_consolidated: usize, +} + +pub fn run_crystallize_pass_with_brain( + _conn: &Connection, _engine: Option<&EmbeddingEngine>, owner_id: Option, brain: &BrainFiringSender, +) -> CrystallizeResult { + if let Some(tx) = brain { + let _ = tx.send(BrainFiringEvent { kind: BrainKind::ConsolidationStarted, payload: json!({}), owner_id }); + } + CrystallizeResult { clusters_found: 0, crystals_created: 0, crystals_updated: 0, entries_consolidated: 0 } +} + +pub fn search_crystals_filtered( + conn: &Connection, query_vector: &[f32], limit: usize, owner_id: Option, team_mode: bool, +) -> Vec<(i64, String, String, f64)> { + if query_vector.is_empty() { + return Vec::new(); + } + let mut stmt = match conn.prepare( + "SELECT id, label, consolidated_text, centroid, owner_id, visibility + FROM memory_clusters + ORDER BY updated_at DESC + LIMIT ?1", + ) { + Ok(stmt) => stmt, + Err(_) => return Vec::new(), + }; + let rows = stmt.query_map(params![limit as i64], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Vec>(3)?, + row.get::<_, Option>(4).ok().flatten(), + row.get::<_, Option>(5).ok().flatten(), + )) + }); + rows.into_iter() + .flatten() + .filter_map(Result::ok) + .filter(|(_, _, _, _, row_owner, visibility)| !team_mode || row_owner == &owner_id || matches!(visibility.as_deref(), Some("shared" | "team"))) + .map(|(id, label, text, blob, _, _)| { + let sim = crate::embeddings::cosine_similarity(query_vector, &crate::embeddings::blob_to_vector(&blob)) as f64; + (id, label, text, sim * CRYSTAL_RELEVANCE_BOOST) + }) + .collect() +} + +pub fn unfold_crystal(conn: &Connection, crystal_id: i64) -> Vec { + let mut stmt = match conn.prepare("SELECT source FROM cluster_members WHERE cluster_id = ?1 ORDER BY id ASC") { + Ok(stmt) => stmt, + Err(_) => return Vec::new(), + }; + stmt.query_map(params![crystal_id], |row| row.get::<_, String>(0)) + .map(|rows| rows.filter_map(Result::ok).collect()) + .unwrap_or_default() +} + +pub fn list_crystals(conn: &Connection) -> Vec { + let mut stmt = match conn.prepare("SELECT id, label, consolidated_text, member_count, updated_at FROM memory_clusters ORDER BY updated_at DESC") { + Ok(stmt) => stmt, + Err(_) => return Vec::new(), + }; + stmt.query_map([], |row| { + Ok(json!({"id":row.get::<_,i64>(0)?,"label":row.get::<_,String>(1)?,"text":row.get::<_,String>(2)?, + "members":row.get::<_,i64>(3)?,"updatedAt":row.get::<_,String>(4)?})) + }) + .map(|rows| rows.filter_map(Result::ok).collect()) + .unwrap_or_default() +} + +pub fn migrate_crystal_tables(conn: &Connection) { + let _ = conn.execute_batch( + "CREATE TABLE IF NOT EXISTS memory_clusters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT NOT NULL, + centroid BLOB NOT NULL DEFAULT X'', + consolidated_text TEXT NOT NULL, + member_count INTEGER NOT NULL DEFAULT 0, + owner_id INTEGER, + visibility TEXT NOT NULL DEFAULT 'private', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS cluster_members ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + cluster_id INTEGER NOT NULL, + source TEXT NOT NULL, + target_type TEXT, + target_id INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_cluster_members_cluster ON cluster_members(cluster_id); + CREATE INDEX IF NOT EXISTS idx_memory_clusters_updated ON memory_clusters(updated_at);", + ); +} diff --git a/daemon-rs/src/crystallize/tests/mod.rs b/daemon-rs/src/crystallize/tests/mod.rs new file mode 100644 index 00000000..7e068b00 --- /dev/null +++ b/daemon-rs/src/crystallize/tests/mod.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +use super::*; +#[test] +fn test_jaccard_identical() { + assert!((jaccard_words("hello world test", "hello world test") - 1.0).abs() < 0.001); +} +#[test] +fn test_jaccard_different() { + let sim = jaccard_words("the quick brown fox", "lazy purple elephant jumps"); + assert!(sim < 0.2); +} +#[test] +fn test_jaccard_partial() { + let sim = jaccard_words("use python for backend", "use python for frontend"); + assert!(sim > 0.5, "Shared 'use python for' should give >0.5"); +} +#[test] +fn test_compute_centroid() { + let v1 = [1.0, 0.0, 0.0]; + let v2 = [0.0, 1.0, 0.0]; + let centroid = compute_centroid(&[&v1[..], &v2[..]]); + assert!(centroid[0] > 0.6 && centroid[0] < 0.8); + assert!(centroid[1] > 0.6 && centroid[1] < 0.8); + assert!(centroid[2].abs() < 0.001); +} +#[test] +fn test_generate_label_empty() { + let entries: Vec<&EmbeddedEntry> = vec![]; + assert_eq!(generate_cluster_label(&entries), "misc"); +} +#[test] +fn test_synthesize_deduplicates() { + let e1 = EmbeddedEntry { + target_type: "memory".to_string(), + target_id: 1, + vector: vec![], + source: "test1".to_string(), + text: "Always use uv for Python package management. Never use pip directly.".to_string(), + }; + let e2 = EmbeddedEntry { + target_type: "memory".to_string(), + target_id: 2, + vector: vec![], + source: "test2".to_string(), + text: "Use uv for Python package management instead of pip.".to_string(), + }; + let e3 = EmbeddedEntry { + target_type: "memory".to_string(), + target_id: 3, + vector: vec![], + source: "test3".to_string(), + text: "Python type hints required on all function signatures.".to_string(), + }; + let result = synthesize_crystal(&vec![&e1, &e2, &e3]); + assert!(result.matches('.').count() <= 3, "Should deduplicate similar sentences, got: {result}"); +} +#[test] +fn test_cluster_entries_basic() { + let make_entry = |id: i64, vec: Vec| EmbeddedEntry { + target_type: "memory".to_string(), + target_id: id, + vector: vec, + source: format!("test::{id}"), + text: format!("Entry {id}"), + }; + let entries = vec![ + make_entry(1, vec![1.0, 0.0, 0.0]), + make_entry(2, vec![0.98, 0.1, 0.0]), + make_entry(3, vec![0.95, 0.15, 0.0]), + make_entry(4, vec![0.0, 1.0, 0.0]), + make_entry(5, vec![0.1, 0.98, 0.0]), + make_entry(6, vec![0.15, 0.95, 0.0]), + ]; + assert_eq!(cluster_entries(&entries).len(), 2, "Should find 2 clusters"); +} +#[test] +fn test_full_crystallize_pass() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + migrate_crystal_tables(&conn); + for i in 1..=4 { + conn.execute( + "INSERT INTO memories (id, text, source, type, status) VALUES (?1, ?2, ?3, 'memory', 'active')", + params![i, format!("Python requires uv for package management rule {i}"), format!("test::python_{i}")], + ) + .unwrap(); + let mut vec = vec![0.0f32; 384]; + vec[0] = 1.0; + vec[1] = 0.01 * i as f32; + conn.execute("INSERT INTO embeddings (target_type, target_id, vector) VALUES ('memory', ?1, ?2)", params![i, embeddings::vector_to_blob(&vec)]) + .unwrap(); + } + let result = run_crystallize_pass_with_brain(&conn, None, None, &None); + assert_eq!(result.clusters_found, 1); + assert_eq!(result.crystals_created, 1); + assert_eq!(result.entries_consolidated, 4); + let crystals = list_crystals(&conn); + assert_eq!(crystals.len(), 1); + assert!(crystals[0]["members"].as_i64().unwrap() >= 4); +} diff --git a/daemon-rs/src/daemon_lifecycle.rs b/daemon-rs/src/daemon_lifecycle.rs deleted file mode 100644 index eed67cc2..00000000 --- a/daemon-rs/src/daemon_lifecycle.rs +++ /dev/null @@ -1,756 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Shared daemon lifecycle utilities: health checks and runtime identity checks. - -use std::fs; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use crate::auth::CortexPaths; -use fs2::FileExt; -use hmac::{Hmac, KeyInit, Mac}; -use sha2::Sha256; -use uuid::Uuid; - -const DAEMON_OWNER_SIGNING_KEY_FILE: &str = "daemon-owner-signing.key"; -const DAEMON_OWNER_TOKEN_VERSION: &str = "v1"; -const DAEMON_OWNER_TOKEN_TTL_SECS: u64 = 180; -pub const DAEMON_OWNER_TOKEN_ENV: &str = "CORTEX_DAEMON_OWNER_TOKEN"; -pub const SPAWN_PARENT_START_TIME_ENV: &str = "CORTEX_SPAWN_PARENT_START_TIME"; - -#[derive(Debug, Clone, PartialEq, Eq)] -struct ParsedOwnerToken { - issued_at: u64, - parent_pid: u32, - nonce: String, - signature: Vec, -} - -fn daemon_owner_runtime_dir(paths: &CortexPaths) -> PathBuf { - paths.home.join("runtime") -} - -fn daemon_owner_signing_key_path(paths: &CortexPaths) -> PathBuf { - daemon_owner_runtime_dir(paths).join(DAEMON_OWNER_SIGNING_KEY_FILE) -} - -fn generate_owner_signing_key() -> [u8; 32] { - let mut key = [0_u8; 32]; - key[..16].copy_from_slice(Uuid::new_v4().as_bytes()); - key[16..].copy_from_slice(Uuid::new_v4().as_bytes()); - key -} - -fn load_or_create_owner_signing_key(paths: &CortexPaths) -> Result, String> { - let runtime_dir = daemon_owner_runtime_dir(paths); - fs::create_dir_all(&runtime_dir).map_err(|e| format!("create runtime dir: {e}"))?; - let key_path = daemon_owner_signing_key_path(paths); - let mut file = fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(&key_path) - .map_err(|e| format!("open daemon owner signing key: {e}"))?; - crate::auth::restrict_file_to_owner(&key_path) - .map_err(|e| format!("restrict daemon owner signing key permissions: {e}"))?; - file.lock_exclusive() - .map_err(|e| format!("lock daemon owner signing key: {e}"))?; - - let mut key_bytes = Vec::new(); - file.read_to_end(&mut key_bytes) - .map_err(|e| format!("read daemon owner signing key: {e}"))?; - if key_bytes.len() != 32 { - key_bytes = generate_owner_signing_key().to_vec(); - file.set_len(0) - .map_err(|e| format!("truncate daemon owner signing key: {e}"))?; - file.seek(SeekFrom::Start(0)) - .map_err(|e| format!("seek daemon owner signing key: {e}"))?; - file.write_all(&key_bytes) - .map_err(|e| format!("write daemon owner signing key: {e}"))?; - file.flush() - .map_err(|e| format!("flush daemon owner signing key: {e}"))?; - } - let _ = file.unlock(); - Ok(key_bytes) -} - -#[cfg(any(test, not(windows)))] -fn encode_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut out = String::with_capacity(bytes.len() * 2); - for &byte in bytes { - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0f) as usize] as char); - } - out -} - -fn decode_hex(value: &str) -> Result, String> { - if !value.len().is_multiple_of(2) { - return Err("hex string length must be even".to_string()); - } - let mut out = Vec::with_capacity(value.len() / 2); - let bytes = value.as_bytes(); - let parse_nibble = |ch: u8| -> Result { - match ch { - b'0'..=b'9' => Ok(ch - b'0'), - b'a'..=b'f' => Ok(ch - b'a' + 10), - b'A'..=b'F' => Ok(ch - b'A' + 10), - _ => Err("invalid hex character".to_string()), - } - }; - for index in (0..bytes.len()).step_by(2) { - let hi = parse_nibble(bytes[index])?; - let lo = parse_nibble(bytes[index + 1])?; - out.push((hi << 4) | lo); - } - Ok(out) -} - -fn sign_owner_token_claim( - key: &[u8], - owner_tag: &str, - parent_pid: u32, - issued_at: u64, - nonce: &str, -) -> Result, String> { - type HmacSha256 = Hmac; - let mut mac = - HmacSha256::new_from_slice(key).map_err(|e| format!("init owner token signer: {e}"))?; - let payload = format!( - "{}|{}|{}|{}|{}", - DAEMON_OWNER_TOKEN_VERSION, owner_tag, parent_pid, issued_at, nonce - ); - mac.update(payload.as_bytes()); - Ok(mac.finalize().into_bytes().to_vec()) -} - -#[cfg(any(test, not(windows)))] -fn build_owner_token( - key: &[u8], - owner_tag: &str, - parent_pid: u32, - issued_at: u64, - nonce: &str, -) -> Result { - let signature = sign_owner_token_claim(key, owner_tag, parent_pid, issued_at, nonce)?; - Ok(format!( - "{}.{}.{}.{}.{}", - DAEMON_OWNER_TOKEN_VERSION, - issued_at, - parent_pid, - nonce, - encode_hex(&signature) - )) -} - -fn parse_owner_token(token: &str) -> Result { - let parts: Vec<&str> = token.trim().split('.').collect(); - if parts.len() != 5 { - return Err("owner token format is invalid".to_string()); - } - if parts[0] != DAEMON_OWNER_TOKEN_VERSION { - return Err("owner token version is unsupported".to_string()); - } - let issued_at = parts[1] - .parse::() - .map_err(|_| "owner token issued timestamp is invalid".to_string())?; - let parent_pid = parts[2] - .parse::() - .map_err(|_| "owner token parent pid is invalid".to_string())?; - let nonce = parts[3].trim().to_string(); - if nonce.is_empty() { - return Err("owner token nonce is missing".to_string()); - } - let signature = decode_hex(parts[4])?; - Ok(ParsedOwnerToken { - issued_at, - parent_pid, - nonce, - signature, - }) -} - -#[cfg(any(test, not(windows)))] -pub fn issue_owner_token_for_spawn( - paths: &CortexPaths, - owner_tag: &str, - parent_pid: u32, -) -> Result { - let key = load_or_create_owner_signing_key(paths)?; - let issued_at = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs()) - .unwrap_or_default(); - let nonce = Uuid::new_v4().simple().to_string(); - build_owner_token(&key, owner_tag, parent_pid, issued_at, &nonce) -} - -pub fn validate_spawned_owner_claim( - paths: &CortexPaths, - owner_tag: Option<&str>, - parent_pid: Option, - owner_token: Option<&str>, -) -> Result<(), String> { - let Some(owner_tag) = owner_tag.map(str::trim).filter(|value| !value.is_empty()) else { - return Ok(()); - }; - let Some(parent_pid) = parent_pid else { - // Direct `cortex serve` invocations may set owner metadata without - // spawn linkage. Only enforce token validation for spawned owners. - return Ok(()); - }; - let token = owner_token - .map(str::trim) - .filter(|value| !value.is_empty()) - .ok_or_else(|| "spawned owner claim is missing ownership token".to_string())?; - - let parsed = parse_owner_token(token)?; - if parsed.parent_pid != parent_pid { - return Err(format!( - "owner token parent mismatch (token={}, env={})", - parsed.parent_pid, parent_pid - )); - } - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs()) - .unwrap_or_default(); - if now.saturating_sub(parsed.issued_at) > DAEMON_OWNER_TOKEN_TTL_SECS { - return Err("owner token is stale".to_string()); - } - - let key = load_or_create_owner_signing_key(paths)?; - let expected_signature = sign_owner_token_claim( - &key, - owner_tag, - parsed.parent_pid, - parsed.issued_at, - &parsed.nonce, - )?; - if expected_signature != parsed.signature { - return Err("owner token signature mismatch".to_string()); - } - Ok(()) -} - -fn health_probe_base(bind: &str, port: u16) -> String { - let host = crate::transport::http_host_for_bind(bind); - format!("http://{host}:{port}") -} - -/// Check if the daemon is ready within a short timeout. -/// Prefers `/readiness` and falls back to `/health` for backward compatibility. -async fn daemon_healthy_at(bind: &str, port: u16, expected_paths: Option<&CortexPaths>) -> bool { - let client = match reqwest::Client::builder() - .timeout(Duration::from_secs(2)) - .build() - { - Ok(client) => client, - Err(_) => return false, - }; - let resolved_paths = CortexPaths::resolve(); - let probe_paths = expected_paths.unwrap_or(&resolved_paths); - let base_url = health_probe_base(bind, port); - let probe_headers = [(String::from("X-Cortex-Request"), String::from("true"))]; - - if let Ok((status, body)) = crate::transport::request_with_local_ipc_fallback( - &client, - "GET", - &base_url, - "/readiness", - probe_paths, - &probe_headers, - None, - Duration::from_secs(2), - ) - .await - { - if let Some(ready) = - readiness_state_from_payload(status.as_u16(), &body, Some(port), expected_paths) - { - return ready; - } - } - - let (status, body) = match crate::transport::request_with_local_ipc_fallback( - &client, - "GET", - &base_url, - "/health", - probe_paths, - &probe_headers, - None, - Duration::from_secs(2), - ) - .await - { - Ok(response) => response, - Err(_) => return false, - }; - - is_cortex_health_payload(status.as_u16(), &body, Some(port), expected_paths) -} - -pub async fn daemon_healthy(paths: &CortexPaths) -> bool { - daemon_healthy_at(&paths.bind, paths.port, Some(paths)).await -} - -fn normalize_runtime_path(value: &str) -> String { - let mut normalized = value.trim().replace('\\', "/"); - while normalized.len() > 1 && normalized.ends_with('/') { - normalized.pop(); - } - #[cfg(windows)] - { - normalized = normalized.to_ascii_lowercase(); - } - normalized -} - -fn path_field_matches(value: Option<&serde_json::Value>, expected: &Path) -> bool { - let expected = normalize_runtime_path(&expected.to_string_lossy()); - value - .and_then(|field| field.as_str()) - .map(normalize_runtime_path) - .is_some_and(|actual| actual == expected) -} - -pub(crate) fn is_cortex_health_payload( - status: u16, - body: &str, - expected_port: Option, - expected_paths: Option<&CortexPaths>, -) -> bool { - if !(200..300).contains(&status) { - return false; - } - - let Ok(json) = serde_json::from_str::(body.trim()) else { - return false; - }; - - let health_status = json.get("status").and_then(|value| value.as_str()); - let runtime = json.get("runtime").and_then(|value| value.as_object()); - let stats = json.get("stats").and_then(|value| value.as_object()); - let runtime_port = runtime - .and_then(|runtime| runtime.get("port")) - .and_then(|value| value.as_u64()) - .and_then(|value| u16::try_from(value).ok()); - - if let Some(expected_port) = expected_port { - if runtime_port != Some(expected_port) { - return false; - } - } - - if let Some(paths) = expected_paths { - if !path_field_matches(stats.and_then(|obj| obj.get("home")), &paths.home) { - return false; - } - if !path_field_matches(runtime.and_then(|obj| obj.get("token_path")), &paths.token) { - return false; - } - if !path_field_matches(runtime.and_then(|obj| obj.get("pid_path")), &paths.pid) { - return false; - } - if !path_field_matches(runtime.and_then(|obj| obj.get("db_path")), &paths.db) { - return false; - } - } - - matches!(health_status, Some("ok" | "degraded")) && runtime.is_some() && stats.is_some() -} - -pub(crate) fn readiness_state_from_payload( - status: u16, - body: &str, - expected_port: Option, - expected_paths: Option<&CortexPaths>, -) -> Option { - let Ok(json) = serde_json::from_str::(body.trim()) else { - return None; - }; - - let ready = json.get("ready").and_then(|value| value.as_bool())?; - let runtime = json.get("runtime").and_then(|value| value.as_object())?; - let stats = json.get("stats").and_then(|value| value.as_object())?; - let runtime_port = runtime - .get("port") - .and_then(|value| value.as_u64()) - .and_then(|value| u16::try_from(value).ok()); - - if let Some(expected_port) = expected_port { - if runtime_port != Some(expected_port) { - return None; - } - } - - if let Some(paths) = expected_paths { - if !path_field_matches(stats.get("home"), &paths.home) { - return None; - } - if !path_field_matches(runtime.get("token_path"), &paths.token) { - return None; - } - if !path_field_matches(runtime.get("pid_path"), &paths.pid) { - return None; - } - if !path_field_matches(runtime.get("db_path"), &paths.db) { - return None; - } - } - - // Ready payloads are expected to be 2xx. Not-ready payloads are expected - // to be 503 (with optional legacy 2xx compatibility). - if ready && !(200..300).contains(&status) { - return None; - } - if !ready { - let expected_not_ready_status = status == 503 || (200..300).contains(&status); - if !expected_not_ready_status { - return None; - } - } - - let readiness_status = json.get("status").and_then(|value| value.as_str()); - let expected_status = if ready { "ready" } else { "starting" }; - if let Some(readiness_status) = readiness_status { - if readiness_status != expected_status { - return None; - } - } - - Some(ready) -} - -/// Poll readiness on the resolved bind host/port until success or timeout. -/// Prefers `/readiness` and falls back to `/health` for backward compatibility. -/// Requires runtime identity to match the expected local Cortex paths. -pub async fn wait_for_health(paths: &CortexPaths, timeout: Duration) -> bool { - let started = std::time::Instant::now(); - while started.elapsed() <= timeout { - if daemon_healthy(paths).await { - return true; - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - false -} - -#[cfg(test)] -mod tests { - use super::{ - build_owner_token, daemon_owner_signing_key_path, health_probe_base, - is_cortex_health_payload, issue_owner_token_for_spawn, load_or_create_owner_signing_key, - readiness_state_from_payload, validate_spawned_owner_claim, DAEMON_OWNER_TOKEN_TTL_SECS, - }; - use crate::auth::CortexPaths; - use serde_json::json; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_test_dir(name: &str) -> std::path::PathBuf { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - std::env::temp_dir().join(format!("cortex_lifecycle_{name}_{unique}")) - } - - #[test] - fn cortex_health_payload_accepts_expected_shapes() { - assert!(is_cortex_health_payload( - 200, - r#"{"status":"ok","runtime":{"version":"0.5.0","port":7437},"stats":{"memories":1}}"#, - Some(7437), - None, - )); - assert!(is_cortex_health_payload( - 200, - r#"{"status":"degraded","runtime":{"version":"0.5.0","port":7437},"stats":{"memories":1}}"#, - Some(7437), - None, - )); - } - - #[test] - fn cortex_health_payload_rejects_non_cortex_bodies() { - assert!(!is_cortex_health_payload( - 200, - r#"{"status":"ok"}"#, - Some(7437), - None - )); - assert!(!is_cortex_health_payload( - 200, - r#"{"status":"ok","runtime":{"version":"0.5.0"}}"#, - Some(7437), - None, - )); - assert!(!is_cortex_health_payload( - 200, - "ok", - Some(7437), - None - )); - assert!(!is_cortex_health_payload( - 500, - r#"{"status":"ok","runtime":{}}"#, - Some(7437), - None, - )); - assert!(!is_cortex_health_payload( - 200, - r#"{"status":"ok","runtime":{"version":"0.5.0","port":9000},"stats":{"memories":1}}"#, - Some(7437), - None, - )); - } - - #[test] - fn cortex_readiness_payload_reports_ready_and_starting_states() { - let ready = serde_json::json!({ - "status": "ready", - "ready": true, - "runtime": { "port": 7437 }, - "stats": { "home": "C:/cortex-test/example/.cortex" } - }) - .to_string(); - assert_eq!( - readiness_state_from_payload(200, &ready, Some(7437), None), - Some(true) - ); - - let starting = serde_json::json!({ - "status": "starting", - "ready": false, - "runtime": { "port": 7437 }, - "stats": { "home": "C:/cortex-test/example/.cortex" } - }) - .to_string(); - assert_eq!( - readiness_state_from_payload(503, &starting, Some(7437), None), - Some(false) - ); - } - - #[test] - fn cortex_readiness_payload_rejects_invalid_shapes() { - assert_eq!( - readiness_state_from_payload(200, r#"{"status":"ready"}"#, Some(7437), None), - None - ); - assert_eq!( - readiness_state_from_payload( - 200, - r#"{"status":"ready","ready":true,"runtime":{"port":9000},"stats":{"home":"C:/cortex-test/example/.cortex"}}"#, - Some(7437), - None - ), - None - ); - assert_eq!( - readiness_state_from_payload( - 500, - r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/example/.cortex"}}"#, - Some(7437), - None - ), - None - ); - } - - #[test] - fn cortex_health_payload_rejects_identity_mismatch_for_local_expectations() { - let home_dir = temp_test_dir("identity_mismatch"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - - let valid_body = json!({ - "status": "ok", - "stats": { - "memories": 1, - "home": paths.home.display().to_string() - }, - "runtime": { - "version": "0.5.0", - "port": paths.port, - "db_path": paths.db.display().to_string(), - "token_path": paths.token.display().to_string(), - "pid_path": paths.pid.display().to_string() - } - }) - .to_string(); - assert!(is_cortex_health_payload( - 200, - &valid_body, - Some(paths.port), - Some(&paths), - )); - - let bad_token_body = json!({ - "status": "ok", - "stats": { - "memories": 1, - "home": paths.home.display().to_string() - }, - "runtime": { - "version": "0.5.0", - "port": paths.port, - "db_path": paths.db.display().to_string(), - "token_path": "C:/wrong/token", - "pid_path": paths.pid.display().to_string() - } - }) - .to_string(); - assert!(!is_cortex_health_payload( - 200, - &bad_token_body, - Some(paths.port), - Some(&paths), - )); - - let _ = std::fs::remove_dir_all(&home_dir); - } - - #[test] - fn health_probe_base_formats_wildcard_and_ipv6_hosts() { - assert_eq!(health_probe_base("", 7437), "http://127.0.0.1:7437"); - assert_eq!(health_probe_base("0.0.0.0", 7437), "http://127.0.0.1:7437"); - assert_eq!( - health_probe_base("localhost", 7437), - "http://localhost:7437" - ); - assert_eq!(health_probe_base("::", 7437), "http://127.0.0.1:7437"); - assert_eq!(health_probe_base("[::]", 7437), "http://127.0.0.1:7437"); - assert_eq!(health_probe_base("::1", 7437), "http://[::1]:7437"); - assert_eq!(health_probe_base("[::1]", 7437), "http://[::1]:7437"); - } - - #[test] - fn owner_token_round_trip_validates_for_spawned_owner() { - let home_dir = temp_test_dir("owner_token_round_trip"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - let token = - issue_owner_token_for_spawn(&paths, "plugin-claude", 4242).expect("issue owner token"); - validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(4242), Some(&token)) - .expect("validate owner token"); - let _ = std::fs::remove_dir_all(&home_dir); - } - - #[cfg(unix)] - #[test] - fn owner_signing_key_file_is_owner_only() { - use std::os::unix::fs::PermissionsExt; - - let home_dir = temp_test_dir("owner_key_permissions"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - - let _ = load_or_create_owner_signing_key(&paths).expect("load owner signing key"); - - let mode = std::fs::metadata(daemon_owner_signing_key_path(&paths)) - .unwrap() - .permissions() - .mode() - & 0o777; - assert_eq!(mode, 0o600); - - let _ = std::fs::remove_dir_all(&home_dir); - } - - #[test] - fn owner_token_validation_rejects_parent_or_owner_mismatch() { - let home_dir = temp_test_dir("owner_token_mismatch"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - let token = - issue_owner_token_for_spawn(&paths, "plugin-claude", 1111).expect("issue owner token"); - - let wrong_parent = - validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(2222), Some(&token)) - .unwrap_err(); - assert!(wrong_parent.contains("parent mismatch")); - - let wrong_owner = - validate_spawned_owner_claim(&paths, Some("control-center"), Some(1111), Some(&token)) - .unwrap_err(); - assert!(wrong_owner.contains("signature mismatch")); - - let _ = std::fs::remove_dir_all(&home_dir); - } - - #[test] - fn owner_token_validation_rejects_missing_or_stale_token() { - let home_dir = temp_test_dir("owner_token_stale"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - - let missing = validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(9999), None) - .unwrap_err(); - assert!(missing.contains("missing ownership token")); - - let key = load_or_create_owner_signing_key(&paths).expect("load owner signing key"); - let stale_issued = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs()) - .unwrap_or_default() - .saturating_sub(DAEMON_OWNER_TOKEN_TTL_SECS + 10); - let stale_token = - build_owner_token(&key, "plugin-claude", 9999, stale_issued, "stale_nonce") - .expect("build stale token"); - let stale_error = validate_spawned_owner_claim( - &paths, - Some("plugin-claude"), - Some(9999), - Some(&stale_token), - ) - .unwrap_err(); - assert!(stale_error.contains("stale")); - - let _ = std::fs::remove_dir_all(&home_dir); - } - - #[test] - fn owner_token_validation_skips_unspawned_owner_claims() { - let home_dir = temp_test_dir("owner_token_unspawned"); - let home_str = home_dir.to_string_lossy().to_string(); - let paths = CortexPaths::resolve_with_overrides( - Some(&home_str), - None, - Some(7437), - Some("127.0.0.1"), - ); - validate_spawned_owner_claim(&paths, Some("control-center"), None, None) - .expect("unspawned owner claims should remain backwards compatible"); - let _ = std::fs::remove_dir_all(&home_dir); - } -} diff --git a/daemon-rs/src/daemon_lifecycle/mod.rs b/daemon-rs/src/daemon_lifecycle/mod.rs new file mode 100644 index 00000000..945c0b07 --- /dev/null +++ b/daemon-rs/src/daemon_lifecycle/mod.rs @@ -0,0 +1,299 @@ +use crate::auth::CortexPaths; +use fs2::FileExt; +use hmac::{Hmac, KeyInit, Mac}; +use sha2::Sha256; +use std::fs; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use uuid::Uuid; +const DAEMON_OWNER_SIGNING_KEY_FILE: &str = "daemon-owner-signing.key"; +const DAEMON_OWNER_TOKEN_VERSION: &str = "v1"; +const DAEMON_OWNER_TOKEN_TTL_SECS: u64 = 180; +pub const DAEMON_OWNER_TOKEN_ENV: &str = "CORTEX_DAEMON_OWNER_TOKEN"; +pub const SPAWN_PARENT_START_TIME_ENV: &str = "CORTEX_SPAWN_PARENT_START_TIME"; +#[derive(Debug, Clone, PartialEq, Eq)] +struct ParsedOwnerToken { + issued_at: u64, + parent_pid: u32, + nonce: String, + signature: Vec, +} +fn daemon_owner_runtime_dir(paths: &CortexPaths) -> PathBuf { + paths.home.join("runtime") +} +fn daemon_owner_signing_key_path(paths: &CortexPaths) -> PathBuf { + daemon_owner_runtime_dir(paths).join(DAEMON_OWNER_SIGNING_KEY_FILE) +} +fn generate_owner_signing_key() -> [u8; 32] { + let mut key = [0_u8; 32]; + key[..16].copy_from_slice(Uuid::new_v4().as_bytes()); + key[16..].copy_from_slice(Uuid::new_v4().as_bytes()); + key +} +fn load_or_create_owner_signing_key(paths: &CortexPaths) -> Result, String> { + let runtime_dir = daemon_owner_runtime_dir(paths); + fs::create_dir_all(&runtime_dir).map_err(|e| format!("create runtime dir: {e}"))?; + let key_path = daemon_owner_signing_key_path(paths); + let mut file = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&key_path) + .map_err(|e| format!("open daemon owner signing key: {e}"))?; + crate::auth::restrict_file_to_owner(&key_path).map_err(|e| format!("restrict daemon owner signing key permissions: {e}"))?; + file.lock_exclusive().map_err(|e| format!("lock daemon owner signing key: {e}"))?; + let mut key_bytes = Vec::new(); + file.read_to_end(&mut key_bytes).map_err(|e| format!("read daemon owner signing key: {e}"))?; + if key_bytes.len() != 32 { + key_bytes = generate_owner_signing_key().to_vec(); + file.set_len(0).map_err(|e| format!("truncate daemon owner signing key: {e}"))?; + file.seek(SeekFrom::Start(0)).map_err(|e| format!("seek daemon owner signing key: {e}"))?; + file.write_all(&key_bytes).map_err(|e| format!("write daemon owner signing key: {e}"))?; + file.flush().map_err(|e| format!("flush daemon owner signing key: {e}"))?; + } + let _ = file.unlock(); + Ok(key_bytes) +} +#[cfg(any(test, not(windows)))] +fn encode_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} +fn decode_hex(value: &str) -> Result, String> { + if !value.len().is_multiple_of(2) { + return Err("hex string length must be even".to_string()); + } + let mut out = Vec::with_capacity(value.len() / 2); + let bytes = value.as_bytes(); + let parse_nibble = |ch: u8| -> Result { + match ch { + b'0'..=b'9' => Ok(ch - b'0'), + b'a'..=b'f' => Ok(ch - b'a' + 10), + b'A'..=b'F' => Ok(ch - b'A' + 10), + _ => Err("invalid hex character".to_string()), + } + }; + for index in (0..bytes.len()).step_by(2) { + let hi = parse_nibble(bytes[index])?; + let lo = parse_nibble(bytes[index + 1])?; + out.push((hi << 4) | lo); + } + Ok(out) +} +fn sign_owner_token_claim(key: &[u8], owner_tag: &str, parent_pid: u32, issued_at: u64, nonce: &str) -> Result, String> { + type HmacSha256 = Hmac; + let mut mac = HmacSha256::new_from_slice(key).map_err(|e| format!("init owner token signer: {e}"))?; + let payload = format!("{}|{}|{}|{}|{}", DAEMON_OWNER_TOKEN_VERSION, owner_tag, parent_pid, issued_at, nonce); + mac.update(payload.as_bytes()); + Ok(mac.finalize().into_bytes().to_vec()) +} +#[cfg(any(test, not(windows)))] +fn build_owner_token(key: &[u8], owner_tag: &str, parent_pid: u32, issued_at: u64, nonce: &str) -> Result { + let signature = sign_owner_token_claim(key, owner_tag, parent_pid, issued_at, nonce)?; + Ok(format!("{}.{}.{}.{}.{}", DAEMON_OWNER_TOKEN_VERSION, issued_at, parent_pid, nonce, encode_hex(&signature))) +} +fn parse_owner_token(token: &str) -> Result { + let parts: Vec<&str> = token.trim().split('.').collect(); + if parts.len() != 5 { + return Err("owner token format is invalid".to_string()); + } + if parts[0] != DAEMON_OWNER_TOKEN_VERSION { + return Err("owner token version is unsupported".to_string()); + } + let issued_at = parts[1].parse::().map_err(|_| "owner token issued timestamp is invalid".to_string())?; + let parent_pid = parts[2].parse::().map_err(|_| "owner token parent pid is invalid".to_string())?; + let nonce = parts[3].trim().to_string(); + if nonce.is_empty() { + return Err("owner token nonce is missing".to_string()); + } + let signature = decode_hex(parts[4])?; + Ok(ParsedOwnerToken { issued_at, parent_pid, nonce, signature }) +} +#[cfg(any(test, not(windows)))] +pub fn issue_owner_token_for_spawn(paths: &CortexPaths, owner_tag: &str, parent_pid: u32) -> Result { + let key = load_or_create_owner_signing_key(paths)?; + let issued_at = SystemTime::now().duration_since(UNIX_EPOCH).map(|duration| duration.as_secs()).unwrap_or_default(); + let nonce = Uuid::new_v4().simple().to_string(); + build_owner_token(&key, owner_tag, parent_pid, issued_at, &nonce) +} +pub fn validate_spawned_owner_claim(paths: &CortexPaths, owner_tag: Option<&str>, parent_pid: Option, owner_token: Option<&str>) -> Result<(), String> { + let Some(owner_tag) = owner_tag.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(()); + }; + let Some(parent_pid) = parent_pid else { + return Ok(()); + }; + let token = owner_token + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| "spawned owner claim is missing ownership token".to_string())?; + let parsed = parse_owner_token(token)?; + if parsed.parent_pid != parent_pid { + return Err(format!("owner token parent mismatch (token={}, env={})", parsed.parent_pid, parent_pid)); + } + let now = SystemTime::now().duration_since(UNIX_EPOCH).map(|duration| duration.as_secs()).unwrap_or_default(); + if now.saturating_sub(parsed.issued_at) > DAEMON_OWNER_TOKEN_TTL_SECS { + return Err("owner token is stale".to_string()); + } + let key = load_or_create_owner_signing_key(paths)?; + let expected_signature = sign_owner_token_claim(&key, owner_tag, parsed.parent_pid, parsed.issued_at, &parsed.nonce)?; + if expected_signature != parsed.signature { + return Err("owner token signature mismatch".to_string()); + } + Ok(()) +} +fn health_probe_base(bind: &str, port: u16) -> String { + let host = crate::transport::http_host_for_bind(bind); + format!("http://{host}:{port}") +} +async fn daemon_healthy_at(bind: &str, port: u16, expected_paths: Option<&CortexPaths>) -> bool { + let client = match reqwest::Client::builder().timeout(Duration::from_secs(2)).build() { + Ok(client) => client, + Err(_) => return false, + }; + let resolved_paths = CortexPaths::resolve(); + let probe_paths = expected_paths.unwrap_or(&resolved_paths); + let base_url = health_probe_base(bind, port); + let probe_headers = [(String::from("X-Cortex-Request"), String::from("true"))]; + if let Ok((status, body)) = + crate::transport::request_with_local_ipc_fallback(&client, "GET", &base_url, "/readiness", probe_paths, &probe_headers, None, Duration::from_secs(2)) + .await + { + if let Some(ready) = readiness_state_from_payload(status.as_u16(), &body, Some(port), expected_paths) { + return ready; + } + } + let (status, body) = match crate::transport::request_with_local_ipc_fallback( + &client, + "GET", + &base_url, + "/health", + probe_paths, + &probe_headers, + None, + Duration::from_secs(2), + ) + .await + { + Ok(response) => response, + Err(_) => return false, + }; + is_cortex_health_payload(status.as_u16(), &body, Some(port), expected_paths) +} +pub async fn daemon_healthy(paths: &CortexPaths) -> bool { + daemon_healthy_at(&paths.bind, paths.port, Some(paths)).await +} +fn normalize_runtime_path(value: &str) -> String { + let mut normalized = value.trim().replace('\\', "/"); + while normalized.len() > 1 && normalized.ends_with('/') { + normalized.pop(); + } + #[cfg(windows)] + { + normalized = normalized.to_ascii_lowercase(); + } + normalized +} +fn path_field_matches(value: Option<&serde_json::Value>, expected: &Path) -> bool { + let expected = normalize_runtime_path(&expected.to_string_lossy()); + value.and_then(|field| field.as_str()).map(normalize_runtime_path).is_some_and(|actual| actual == expected) +} +pub(crate) fn is_cortex_health_payload(status: u16, body: &str, expected_port: Option, expected_paths: Option<&CortexPaths>) -> bool { + if !(200..300).contains(&status) { + return false; + } + let Ok(json) = serde_json::from_str::(body.trim()) else { + return false; + }; + let health_status = json.get("status").and_then(|value| value.as_str()); + let runtime = json.get("runtime").and_then(|value| value.as_object()); + let stats = json.get("stats").and_then(|value| value.as_object()); + let runtime_port = runtime + .and_then(|runtime| runtime.get("port")) + .and_then(|value| value.as_u64()) + .and_then(|value| u16::try_from(value).ok()); + if let Some(expected_port) = expected_port { + if runtime_port != Some(expected_port) { + return false; + } + } + if let Some(paths) = expected_paths { + if !path_field_matches(stats.and_then(|obj| obj.get("home")), &paths.home) { + return false; + } + if !path_field_matches(runtime.and_then(|obj| obj.get("token_path")), &paths.token) { + return false; + } + if !path_field_matches(runtime.and_then(|obj| obj.get("pid_path")), &paths.pid) { + return false; + } + if !path_field_matches(runtime.and_then(|obj| obj.get("db_path")), &paths.db) { + return false; + } + } + matches!(health_status, Some("ok" | "degraded")) && runtime.is_some() && stats.is_some() +} +pub(crate) fn readiness_state_from_payload(status: u16, body: &str, expected_port: Option, expected_paths: Option<&CortexPaths>) -> Option { + let Ok(json) = serde_json::from_str::(body.trim()) else { + return None; + }; + let ready = json.get("ready").and_then(|value| value.as_bool())?; + let runtime = json.get("runtime").and_then(|value| value.as_object())?; + let stats = json.get("stats").and_then(|value| value.as_object())?; + let runtime_port = runtime.get("port").and_then(|value| value.as_u64()).and_then(|value| u16::try_from(value).ok()); + if let Some(expected_port) = expected_port { + if runtime_port != Some(expected_port) { + return None; + } + } + if let Some(paths) = expected_paths { + if !path_field_matches(stats.get("home"), &paths.home) { + return None; + } + if !path_field_matches(runtime.get("token_path"), &paths.token) { + return None; + } + if !path_field_matches(runtime.get("pid_path"), &paths.pid) { + return None; + } + if !path_field_matches(runtime.get("db_path"), &paths.db) { + return None; + } + } + if ready && !(200..300).contains(&status) { + return None; + } + if !ready { + let expected_not_ready_status = status == 503 || (200..300).contains(&status); + if !expected_not_ready_status { + return None; + } + } + let readiness_status = json.get("status").and_then(|value| value.as_str()); + let expected_status = if ready { "ready" } else { "starting" }; + if let Some(readiness_status) = readiness_status { + if readiness_status != expected_status { + return None; + } + } + Some(ready) +} +pub async fn wait_for_health(paths: &CortexPaths, timeout: Duration) -> bool { + let started = std::time::Instant::now(); + while started.elapsed() <= timeout { + if daemon_healthy(paths).await { + return true; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + false +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/daemon_lifecycle/tests/mod.rs b/daemon-rs/src/daemon_lifecycle/tests/mod.rs new file mode 100644 index 00000000..5ec75031 --- /dev/null +++ b/daemon-rs/src/daemon_lifecycle/tests/mod.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::{ + build_owner_token, daemon_owner_signing_key_path, health_probe_base, is_cortex_health_payload, issue_owner_token_for_spawn, + load_or_create_owner_signing_key, readiness_state_from_payload, validate_spawned_owner_claim, DAEMON_OWNER_TOKEN_TTL_SECS, +}; +use crate::auth::CortexPaths; +use serde_json::json; +use std::time::{SystemTime, UNIX_EPOCH}; +fn temp_test_dir(name: &str) -> std::path::PathBuf { + let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos(); + std::env::temp_dir().join(format!("cortex_lifecycle_{name}_{unique}")) +} +#[test] +fn cortex_health_payload_accepts_expected_shapes() { + assert!(is_cortex_health_payload(200, r#"{"status":"ok","runtime":{"version":"0.5.0","port":7437},"stats":{"memories":1}}"#, Some(7437), None,)); + assert!(is_cortex_health_payload(200, r#"{"status":"degraded","runtime":{"version":"0.5.0","port":7437},"stats":{"memories":1}}"#, Some(7437), None,)); +} +#[test] +fn cortex_health_payload_rejects_non_cortex_bodies() { + assert!(!is_cortex_health_payload(200, r#"{"status":"ok"}"#, Some(7437), None)); + assert!(!is_cortex_health_payload(200, r#"{"status":"ok","runtime":{"version":"0.5.0"}}"#, Some(7437), None,)); + assert!(!is_cortex_health_payload(200, "ok", Some(7437), None)); + assert!(!is_cortex_health_payload(500, r#"{"status":"ok","runtime":{}}"#, Some(7437), None,)); + assert!(!is_cortex_health_payload(200, r#"{"status":"ok","runtime":{"version":"0.5.0","port":9000},"stats":{"memories":1}}"#, Some(7437), None,)); +} +#[test] +fn cortex_readiness_payload_reports_ready_and_starting_states() { + let ready = serde_json::json!({ + "status": "ready", + "ready": true, + "runtime": { "port": 7437 }, + "stats": { "home": "C:/cortex-test/example/.cortex" } + }) + .to_string(); + assert_eq!(readiness_state_from_payload(200, &ready, Some(7437), None), Some(true)); + let starting = serde_json::json!({ + "status": "starting", + "ready": false, + "runtime": { "port": 7437 }, + "stats": { "home": "C:/cortex-test/example/.cortex" } + }) + .to_string(); + assert_eq!(readiness_state_from_payload(503, &starting, Some(7437), None), Some(false)); +} +#[test] +fn cortex_readiness_payload_rejects_invalid_shapes() { + assert_eq!(readiness_state_from_payload(200, r#"{"status":"ready"}"#, Some(7437), None), None); + assert_eq!( + readiness_state_from_payload( + 200, + r#"{"status":"ready","ready":true,"runtime":{"port":9000},"stats":{"home":"C:/cortex-test/example/.cortex"}}"#, + Some(7437), + None + ), + None + ); + assert_eq!( + readiness_state_from_payload( + 500, + r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/example/.cortex"}}"#, + Some(7437), + None + ), + None + ); +} +#[test] +fn cortex_health_payload_rejects_identity_mismatch_for_local_expectations() { + let home_dir = temp_test_dir("identity_mismatch"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + let valid_body = json!({ + "status": "ok", + "stats": { + "memories": 1, + "home": paths.home.display().to_string() + }, + "runtime": { + "version": "0.5.0", + "port": paths.port, + "db_path": paths.db.display().to_string(), + "token_path": paths.token.display().to_string(), + "pid_path": paths.pid.display().to_string() + } + }) + .to_string(); + assert!(is_cortex_health_payload(200, &valid_body, Some(paths.port), Some(&paths),)); + let bad_token_body = json!({ + "status": "ok", + "stats": { + "memories": 1, + "home": paths.home.display().to_string() + }, + "runtime": { + "version": "0.5.0", + "port": paths.port, + "db_path": paths.db.display().to_string(), + "token_path": "C:/wrong/token", + "pid_path": paths.pid.display().to_string() + } + }) + .to_string(); + assert!(!is_cortex_health_payload(200, &bad_token_body, Some(paths.port), Some(&paths),)); + let _ = std::fs::remove_dir_all(&home_dir); +} +#[test] +fn health_probe_base_formats_wildcard_and_ipv6_hosts() { + assert_eq!(health_probe_base("", 7437), "http://127.0.0.1:7437"); + assert_eq!(health_probe_base("0.0.0.0", 7437), "http://127.0.0.1:7437"); + assert_eq!(health_probe_base("localhost", 7437), "http://localhost:7437"); + assert_eq!(health_probe_base("::", 7437), "http://127.0.0.1:7437"); + assert_eq!(health_probe_base("[::]", 7437), "http://127.0.0.1:7437"); + assert_eq!(health_probe_base("::1", 7437), "http://[::1]:7437"); + assert_eq!(health_probe_base("[::1]", 7437), "http://[::1]:7437"); +} +#[test] +fn owner_token_round_trip_validates_for_spawned_owner() { + let home_dir = temp_test_dir("owner_token_round_trip"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + let token = issue_owner_token_for_spawn(&paths, "plugin-claude", 4242).expect("issue owner token"); + validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(4242), Some(&token)).expect("validate owner token"); + let _ = std::fs::remove_dir_all(&home_dir); +} +#[cfg(unix)] +#[test] +fn owner_signing_key_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let home_dir = temp_test_dir("owner_key_permissions"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + let _ = load_or_create_owner_signing_key(&paths).expect("load owner signing key"); + let mode = std::fs::metadata(daemon_owner_signing_key_path(&paths)).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let _ = std::fs::remove_dir_all(&home_dir); +} +#[test] +fn owner_token_validation_rejects_parent_or_owner_mismatch() { + let home_dir = temp_test_dir("owner_token_mismatch"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + let token = issue_owner_token_for_spawn(&paths, "plugin-claude", 1111).expect("issue owner token"); + let wrong_parent = validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(2222), Some(&token)).unwrap_err(); + assert!(wrong_parent.contains("parent mismatch")); + let wrong_owner = validate_spawned_owner_claim(&paths, Some("control-center"), Some(1111), Some(&token)).unwrap_err(); + assert!(wrong_owner.contains("signature mismatch")); + let _ = std::fs::remove_dir_all(&home_dir); +} +#[test] +fn owner_token_validation_rejects_missing_or_stale_token() { + let home_dir = temp_test_dir("owner_token_stale"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + let missing = validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(9999), None).unwrap_err(); + assert!(missing.contains("missing ownership token")); + let key = load_or_create_owner_signing_key(&paths).expect("load owner signing key"); + let stale_issued = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default() + .saturating_sub(DAEMON_OWNER_TOKEN_TTL_SECS + 10); + let stale_token = build_owner_token(&key, "plugin-claude", 9999, stale_issued, "stale_nonce").expect("build stale token"); + let stale_error = validate_spawned_owner_claim(&paths, Some("plugin-claude"), Some(9999), Some(&stale_token)).unwrap_err(); + assert!(stale_error.contains("stale")); + let _ = std::fs::remove_dir_all(&home_dir); +} +#[test] +fn owner_token_validation_skips_unspawned_owner_claims() { + let home_dir = temp_test_dir("owner_token_unspawned"); + let home_str = home_dir.to_string_lossy().to_string(); + let paths = CortexPaths::resolve_with_overrides(Some(&home_str), None, Some(7437), Some("127.0.0.1")); + validate_spawned_owner_claim(&paths, Some("control-center"), None, None).expect("unspawned owner claims should remain backwards compatible"); + let _ = std::fs::remove_dir_all(&home_dir); +} diff --git a/daemon-rs/src/db/connection.rs b/daemon-rs/src/db/connection.rs index ced4d102..5ac0a58c 100644 --- a/daemon-rs/src/db/connection.rs +++ b/daemon-rs/src/db/connection.rs @@ -1,29 +1,20 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashSet; +use rusqlite::Connection; use std::path::Path; -use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::atomic::AtomicI64; use std::sync::OnceLock; -use std::time::{SystemTime, UNIX_EPOCH}; -use rusqlite::{params, Connection, OptionalExtension}; - - -use super::*; pub(crate) const BEST_EFFORT_CHECKPOINT_MIN_INTERVAL_MS: i64 = 5_000; pub(crate) const BEST_EFFORT_TRUNCATE_INTERVAL_MS: i64 = 5 * 60 * 1_000; pub const SQLITE_BUSY_TIMEOUT_MS: u64 = 5_000; pub const SQLITE_WAL_AUTOCHECKPOINT_PAGES: u64 = 1_000; pub(crate) static LAST_BEST_EFFORT_CHECKPOINT_MS: AtomicI64 = AtomicI64::new(0); pub(crate) static LAST_BEST_EFFORT_TRUNCATE_MS: AtomicI64 = AtomicI64::new(0); - #[derive(Debug, Clone, PartialEq, Eq)] pub struct SqliteVecStatus { pub available: bool, pub version: Option, pub error: Option, } - static SQLITE_VEC_REGISTRATION: OnceLock> = OnceLock::new(); - pub(crate) fn ensure_sqlite_vec_registered() -> Result<(), String> { SQLITE_VEC_REGISTRATION .get_or_init(|| { @@ -32,22 +23,14 @@ pub(crate) fn ensure_sqlite_vec_registered() -> Result<(), String> { *mut *mut std::os::raw::c_char, *const rusqlite::ffi::sqlite3_api_routines, ) -> std::os::raw::c_int; - unsafe extern "C" { #[link_name = "sqlite3_vec_init"] pub(crate) fn sqlite3_vec_init_auto_extension( - db: *mut rusqlite::ffi::sqlite3, - err_msg: *mut *mut std::os::raw::c_char, - api: *const rusqlite::ffi::sqlite3_api_routines, + db: *mut rusqlite::ffi::sqlite3, err_msg: *mut *mut std::os::raw::c_char, api: *const rusqlite::ffi::sqlite3_api_routines, ) -> std::os::raw::c_int; } - - // Keep the `sqlite-vec` crate referenced: its build script supplies - // the native `sqlite_vec0` library that defines this symbol. let _sqlite_vec_symbol: unsafe extern "C" fn() = sqlite_vec::sqlite3_vec_init; let init: SqliteVecEntryPoint = sqlite3_vec_init_auto_extension; - // SAFETY: `init` points to `sqlite3_vec_init` with SQLite's - // required auto-extension ABI and remains valid for the process. let rc = unsafe { rusqlite::ffi::sqlite3_auto_extension(Some(init)) }; if rc == 0 { Ok(()) @@ -57,31 +40,20 @@ pub(crate) fn ensure_sqlite_vec_registered() -> Result<(), String> { }) .clone() } - -/// Result of an auto-repair attempt. #[derive(Debug)] pub struct RepairResult { pub memories_recovered: usize, pub decisions_recovered: usize, pub corrupt_db_path: std::path::PathBuf, } - -/// Error type for auto-repair failures. pub enum RepairError { - /// Could not open the corrupted DB for reading. OpenCorrupt(rusqlite::Error), - /// Could not create a fresh DB for the repaired copy. OpenFresh(rusqlite::Error), - /// Data export from the corrupted DB failed. Export(rusqlite::Error), - /// Import into the fresh DB failed. Import(rusqlite::Error), - /// The repaired DB itself failed integrity_check. RepairIntegrityFailed, - /// File-system rename/copy operations failed. Io(std::io::Error), } - impl std::fmt::Debug for RepairError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -94,61 +66,25 @@ impl std::fmt::Debug for RepairError { } } } - -/// Open a SQLite connection at the given path. pub fn open(path: &Path) -> rusqlite::Result { let _ = ensure_sqlite_vec_registered(); Connection::open(path) } - pub fn sqlite_vec_status(conn: &Connection) -> SqliteVecStatus { if let Err(error) = ensure_sqlite_vec_registered() { - return SqliteVecStatus { - available: false, - version: None, - error: Some(error), - }; + return SqliteVecStatus { available: false, version: None, error: Some(error) }; } - match conn.query_row("SELECT vec_version()", [], |row| row.get::<_, String>(0)) { - Ok(version) => SqliteVecStatus { - available: true, - version: Some(version), - error: None, - }, - Err(error) => SqliteVecStatus { - available: false, - version: None, - error: Some(error.to_string()), - }, + Ok(version) => SqliteVecStatus { available: true, version: Some(version), error: None }, + Err(error) => SqliteVecStatus { available: false, version: None, error: Some(error.to_string()) }, } } - pub(crate) fn env_u64_clamped(name: &str, default: u64, min: u64, max: u64) -> u64 { - let parsed = std::env::var(name) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .unwrap_or(default); + let parsed = std::env::var(name).ok().and_then(|raw| raw.trim().parse::().ok()).unwrap_or(default); parsed.clamp(min, max) } - -/// Apply WAL mode, NORMAL synchronous writes, foreign-key enforcement, and -/// bounded SQLite lock waits. -/// -/// NOTE: PRAGMA synchronous=NORMAL is safe with WAL mode. From SQLite docs: -/// - FULL: Extra safety at the cost of significant performance (OS crash protection) -/// - NORMAL: All changes are synced before passing control to caller at critical moments -/// (process crash protection). With WAL checkpoint every 10s, data loss is limited to <10s. -/// This is the recommended setting for WAL mode workloads. pub fn configure(conn: &Connection) -> rusqlite::Result<()> { - // Defaults tuned for mixed desktop + daemon workloads; both are overridable - // to let operators adapt for RAM-constrained or high-throughput hosts. - let mmap_size = env_u64_clamped( - "CORTEX_DB_MMAP_SIZE_BYTES", - 268_435_456, - 64 * 1024 * 1024, - 4 * 1024 * 1024 * 1024, - ); + let mmap_size = env_u64_clamped("CORTEX_DB_MMAP_SIZE_BYTES", 268_435_456, 64 * 1024 * 1024, 4 * 1024 * 1024 * 1024); let cache_size_kib = env_u64_clamped("CORTEX_DB_CACHE_SIZE_KIB", 12_000, 2_000, 131_072); let cache_size = -(cache_size_kib as i64); let busy_timeout_ms = SQLITE_BUSY_TIMEOUT_MS; @@ -168,6 +104,4 @@ pub fn configure(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch(&pragmas)?; Ok(()) } - pub(crate) type MigrationDef = (&'static str, &'static str); - diff --git a/daemon-rs/src/db/maintenance.rs b/daemon-rs/src/db/maintenance.rs index e96b7fcd..d04250ec 100644 --- a/daemon-rs/src/db/maintenance.rs +++ b/daemon-rs/src/db/maintenance.rs @@ -1,13 +1,8 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashSet; +use super::*; +use rusqlite::{params, Connection, OptionalExtension}; use std::path::Path; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::OnceLock; +use std::sync::atomic::Ordering; use std::time::{SystemTime, UNIX_EPOCH}; -use rusqlite::{params, Connection, OptionalExtension}; - - -use super::*; pub fn migrate_focus_table(conn: &Connection) { let sql = r#" CREATE TABLE IF NOT EXISTS focus_sessions ( @@ -28,9 +23,6 @@ pub fn migrate_focus_table(conn: &Connection) { Err(e) => eprintln!("[db] Focus table migration: {e}"), } } - -/// Run schema migrations for progressive aging columns. -/// Safe to call repeatedly -- ALTER TABLE with IF NOT EXISTS-style error handling. pub(crate) fn migrate_aging_columns_with_logging(conn: &Connection, log_success: bool) { let migrations = [ "ALTER TABLE memories ADD COLUMN compressed_text TEXT", @@ -47,27 +39,19 @@ pub(crate) fn migrate_aging_columns_with_logging(conn: &Connection, log_success: } } } - pub(crate) fn unix_now_ms() -> i64 { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis(); i64::try_from(now).unwrap_or(i64::MAX) } - pub(crate) fn should_attempt_best_effort_checkpoint(now_ms: i64, last_checkpoint_ms: i64) -> bool { now_ms.saturating_sub(last_checkpoint_ms) >= BEST_EFFORT_CHECKPOINT_MIN_INTERVAL_MS } - pub(crate) fn should_attempt_truncate_checkpoint(now_ms: i64, last_truncate_ms: i64) -> bool { if last_truncate_ms <= 0 { return false; } now_ms.saturating_sub(last_truncate_ms) >= BEST_EFFORT_TRUNCATE_INTERVAL_MS } - -/// Attempt a WAL checkpoint; silently ignore any error. pub fn checkpoint_wal_best_effort(conn: &Connection) { let now_ms = unix_now_ms(); let last_checkpoint_ms = LAST_BEST_EFFORT_CHECKPOINT_MS.load(Ordering::Relaxed); @@ -75,59 +59,32 @@ pub fn checkpoint_wal_best_effort(conn: &Connection) { return; } if LAST_BEST_EFFORT_CHECKPOINT_MS - .compare_exchange( - last_checkpoint_ms, - now_ms, - Ordering::Relaxed, - Ordering::Relaxed, - ) + .compare_exchange(last_checkpoint_ms, now_ms, Ordering::Relaxed, Ordering::Relaxed) .is_err() { return; } - let mut last_truncate_ms = LAST_BEST_EFFORT_TRUNCATE_MS.load(Ordering::Relaxed); if last_truncate_ms <= 0 { LAST_BEST_EFFORT_TRUNCATE_MS.store(now_ms, Ordering::Relaxed); last_truncate_ms = now_ms; } - - if should_attempt_truncate_checkpoint(now_ms, last_truncate_ms) - && conn - .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") - .is_ok() - { + if should_attempt_truncate_checkpoint(now_ms, last_truncate_ms) && conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").is_ok() { LAST_BEST_EFFORT_TRUNCATE_MS.store(now_ms, Ordering::Relaxed); return; } - let _ = conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);"); } - #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub struct ExpiredCleanupCounts { pub memories_deleted: usize, pub decisions_deleted: usize, } - -/// Delete expired rows from tables that support TTL-based retention. pub fn delete_expired_entries(conn: &Connection) -> rusqlite::Result { - let memories_deleted = conn.execute( - "DELETE FROM memories WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", - [], - )?; - let decisions_deleted = conn.execute( - "DELETE FROM decisions WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", - [], - )?; - Ok(ExpiredCleanupCounts { - memories_deleted, - decisions_deleted, - }) + let memories_deleted = conn.execute("DELETE FROM memories WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", [])?; + let decisions_deleted = conn.execute("DELETE FROM decisions WHERE expires_at IS NOT NULL AND expires_at < datetime('now')", [])?; + Ok(ExpiredCleanupCounts { memories_deleted, decisions_deleted }) } - -/// Rebuild FTS5 indexes from base table data. Call once after schema migration -/// on databases that predate FTS5 support. pub fn rebuild_fts(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( "INSERT OR IGNORE INTO memories_fts(rowid, text, source, tags) @@ -137,11 +94,6 @@ pub fn rebuild_fts(conn: &Connection) -> rusqlite::Result<()> { )?; Ok(()) } - -/// Fully reindex FTS5 tables from canonical memory/decision rows. -/// -/// Unlike `rebuild_fts`, this removes stale rows first so deleted/orphaned -/// entries do not remain searchable. pub fn reindex_fts(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( "INSERT INTO memories_fts(memories_fts) VALUES('delete-all'); @@ -149,24 +101,13 @@ pub fn reindex_fts(conn: &Connection) -> rusqlite::Result<()> { )?; rebuild_fts(conn) } - -/// Seed FTS indexes at most once per database. -/// -/// Uses a marker row in `schema_migrations` so startup does not rescan the -/// entire corpus on every daemon restart. pub fn rebuild_fts_if_needed(conn: &Connection) -> rusqlite::Result { let already_seeded = conn - .query_row( - "SELECT 1 FROM schema_migrations WHERE version = 'fts_seeded_v1' LIMIT 1", - [], - |row| row.get::<_, i64>(0), - ) + .query_row("SELECT 1 FROM schema_migrations WHERE version = 'fts_seeded_v1' LIMIT 1", [], |row| row.get::<_, i64>(0)) .optional()?; - if already_seeded.is_some() { return Ok(false); } - rebuild_fts(conn)?; conn.execute( "INSERT OR IGNORE INTO schema_migrations (version, name, applied_at) @@ -175,46 +116,18 @@ pub fn rebuild_fts_if_needed(conn: &Connection) -> rusqlite::Result { )?; Ok(true) } - -/// Run `PRAGMA integrity_check` and return `true` when the database reports -/// `ok`. pub fn verify_integrity(conn: &Connection) -> rusqlite::Result { let result: String = conn.query_row("PRAGMA integrity_check", [], |row| row.get(0))?; Ok(result.trim().eq_ignore_ascii_case("ok")) } - -/// Run `PRAGMA quick_check` (B-tree structure only -- faster than integrity_check). -/// Returns `true` when the database passes. pub fn quick_check(conn: &Connection) -> bool { conn.query_row("PRAGMA quick_check", [], |row| row.get::<_, String>(0)) .map(|s| s.trim().eq_ignore_ascii_case("ok")) .unwrap_or(false) } - -/// Attempt to recover data from a corrupted DB using dump-and-rebuild. -/// -/// Steps: -/// 1. Open `db_path` read-only in a *separate* connection (does not touch the live connection). -/// 2. `SELECT *` from every data table -- data pages survive most B-tree corruption. -/// 3. Write all rows into a fresh temp DB at `{db_path}.repair_tmp`. -/// 4. Run `PRAGMA integrity_check` on the fresh DB. -/// 5. Rename `db_path` → `{db_path}.corrupt.{timestamp}` (never deleted). -/// 6. Rename `{db_path}.repair_tmp` → `db_path`. -/// -/// The original DB is preserved in all failure paths -- if any step before 5 -/// fails, `db_path` is unchanged. -/// -/// FTS virtual tables are re-created and populated from data; they are not -/// exported directly. pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result { - eprintln!( - "[cortex] auto_repair: beginning dump-and-rebuild of {}", - db_path.display() - ); - - // ── Step 1: open the corrupted DB read-only ──────────────────────────── + eprintln!("[cortex] auto_repair: beginning dump-and-rebuild of {}", db_path.display()); let corrupt_conn = Connection::open(db_path).map_err(RepairError::OpenCorrupt)?; - // Open read-only: ignore any write errors on WAL frames, read what we can. let busy_timeout_ms = SQLITE_BUSY_TIMEOUT_MS; let _ = corrupt_conn.execute_batch(&format!( r#" @@ -222,11 +135,6 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result Result)> = Vec::new(); let mut memories_recovered = 0usize; let mut decisions_recovered = 0usize; - for &table in DATA_TABLES { - // Check if the table even exists in the corrupted DB. let exists: bool = corrupt_conn - .query_row( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1", - params![table], - |_| Ok(()), - ) + .query_row("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1", params![table], |_| Ok(())) .is_ok(); if !exists { eprintln!("[cortex] auto_repair: table '{table}' not found in corrupt DB, skipping"); continue; } - - // Get column names via PRAGMA. - let mut col_stmt = corrupt_conn - .prepare(&format!("PRAGMA table_info({table})")) - .map_err(RepairError::Export)?; - let columns: Vec = col_stmt - .query_map([], |row| row.get::<_, String>(1)) - .map_err(RepairError::Export)? - .filter_map(|r| r.ok()) - .collect(); - + let mut col_stmt = corrupt_conn.prepare(&format!("PRAGMA table_info({table})")).map_err(RepairError::Export)?; + let columns: Vec = col_stmt.query_map([], |row| row.get::<_, String>(1)).map_err(RepairError::Export)?.filter_map(|r| r.ok()).collect(); if columns.is_empty() { eprintln!("[cortex] auto_repair: table '{table}' has no columns, skipping"); continue; } - - // SELECT * and build INSERT statements using rusqlite's dynamic row access. let col_list = columns.join(", "); let placeholders: Vec = (1..=columns.len()).map(|i| format!("?{i}")).collect(); let placeholder_list = placeholders.join(", "); - let mut data_stmt = match corrupt_conn.prepare(&format!("SELECT {col_list} FROM {table}")) { Ok(s) => s, Err(e) => { @@ -295,7 +181,6 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result r, @@ -304,13 +189,7 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result> = Vec::new(); loop { match rows.next() { @@ -324,12 +203,10 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result format!("{f}"), Ok(ValueRef::Text(t)) => { let s = String::from_utf8_lossy(t); - // Escape single quotes. format!("'{}'", s.replace('\'', "''")) } Ok(ValueRef::Blob(b)) => { - let hex: String = - b.iter().map(|byte| format!("{byte:02X}")).collect(); + let hex: String = b.iter().map(|byte| format!("{byte:02X}")).collect(); format!("X'{hex}'") } Err(_) => "NULL".to_string(), @@ -340,24 +217,17 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result break, Err(e) => { - // Row-level corruption: skip the bad row and continue. eprintln!("[cortex] auto_repair: row error in '{table}': {e} -- skipping row"); continue; } } } - - eprintln!( - "[cortex] auto_repair: exported {} rows from '{table}'", - row_values.len() - ); + eprintln!("[cortex] auto_repair: exported {} rows from '{table}'", row_values.len()); if table == "memories" { memories_recovered = row_values.len(); } else if table == "decisions" { decisions_recovered = row_values.len(); } - - // Convert to literal INSERT statements (values already SQL-safe). let inserts: Vec = row_values .into_iter() .map(|vals| { @@ -365,42 +235,23 @@ pub fn auto_repair(db_path: &Path, timestamp: &str) -> Result Result Result, -) -> rusqlite::Result { - if table != "memories" && table != "decisions" { - return Err(rusqlite::Error::InvalidParameterName(format!( - "archive_entries: unsupported table '{table}'" - ))); - } - if ids.is_empty() { - return Ok(0); - } - - let placeholders = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 1)) - .collect::>() - .join(", "); - - let sql = if owner_id.is_some() { - format!( - "UPDATE {table} SET status = 'archived' WHERE owner_id = ?{} AND id IN ({placeholders})", - ids.len() + 1 - ) - } else { - format!("UPDATE {table} SET status = 'archived' WHERE id IN ({placeholders})") - }; - - let mut stmt = conn.prepare(&sql)?; - let affected = if let Some(owner_id) = owner_id { - let mut values: Vec = ids - .iter() - .copied() - .map(rusqlite::types::Value::Integer) - .collect(); - values.push(rusqlite::types::Value::Integer(owner_id)); - stmt.execute(rusqlite::params_from_iter(values.iter()))? - } else { - stmt.execute(rusqlite::params_from_iter(ids.iter()))? - }; - Ok(affected) -} - -#[allow(dead_code)] -pub fn archive_entries(conn: &Connection, table: &str, ids: &[i64]) -> rusqlite::Result { - archive_entries_scoped(conn, table, ids, None) + Ok(RepairResult { memories_recovered, decisions_recovered, corrupt_db_path: corrupt_archive }) } - diff --git a/daemon-rs/src/db/migrations.rs b/daemon-rs/src/db/migrations.rs index b4cbab81..29556c5a 100644 --- a/daemon-rs/src/db/migrations.rs +++ b/daemon-rs/src/db/migrations.rs @@ -1,14 +1,7 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashSet; -use std::path::Path; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::OnceLock; -use std::time::{SystemTime, UNIX_EPOCH}; -use rusqlite::{params, Connection, OptionalExtension}; - - use super::*; -pub(crate) const SCHEMA_MIGRATIONS: [MigrationDef; 16] = [ +use rusqlite::{params, Connection}; +use std::collections::HashSet; +pub(crate) const SCHEMA_MIGRATIONS: [MigrationDef; 17] = [ ("001_initial_schema", "initial_schema"), ("002_aging_columns", "aging_columns"), ("003_focus_table", "focus_table"), @@ -25,54 +18,30 @@ pub(crate) const SCHEMA_MIGRATIONS: [MigrationDef; 16] = [ ("014", "temporal_semantics_fields"), ("015", "boot_audits"), ("016", "retention_classes"), + ("017", "recall_hot_path_indexes"), ]; - -/// Return ordered schema migration definitions. pub fn migration_definitions() -> &'static [MigrationDef] { &SCHEMA_MIGRATIONS } - pub(crate) fn migration_user_version(version: &str) -> i32 { - version - .chars() - .take_while(|ch| ch.is_ascii_digit()) - .collect::() - .parse::() - .unwrap_or(0) + version.chars().take_while(|ch| ch.is_ascii_digit()).collect::().parse::().unwrap_or(0) } - #[cfg_attr(not(test), allow(dead_code))] pub fn latest_schema_user_version() -> i32 { - migration_definitions() - .iter() - .map(|(version, _)| migration_user_version(version)) - .max() - .unwrap_or(0) + migration_definitions().iter().map(|(version, _)| migration_user_version(version)).max().unwrap_or(0) } - pub fn current_schema_user_version(conn: &Connection) -> rusqlite::Result { conn.query_row("PRAGMA user_version", [], |row| row.get(0)) } - pub fn set_schema_user_version(conn: &Connection, version: i32) -> rusqlite::Result<()> { conn.pragma_update(None, "user_version", version)?; Ok(()) } - -pub(crate) fn sync_schema_user_version( - conn: &Connection, - applied_versions: &HashSet, -) -> rusqlite::Result { - let version = applied_versions - .iter() - .map(|entry| migration_user_version(entry)) - .max() - .unwrap_or(0); +pub(crate) fn sync_schema_user_version(conn: &Connection, applied_versions: &HashSet) -> rusqlite::Result { + let version = applied_versions.iter().map(|entry| migration_user_version(entry)).max().unwrap_or(0); set_schema_user_version(conn, version)?; Ok(version) } - -/// Ensure schema migration tracking table exists. pub fn ensure_schema_migrations_table(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( r#" @@ -86,18 +55,11 @@ pub fn ensure_schema_migrations_table(conn: &Connection) -> rusqlite::Result<()> )?; Ok(()) } - pub(crate) fn migration_error(msg: impl Into) -> rusqlite::Error { rusqlite::Error::InvalidParameterName(msg.into()) } - -pub(crate) fn apply_migration_with_logging( - conn: &Connection, - version: &str, - log_success: bool, -) -> rusqlite::Result<()> { +pub(crate) fn apply_migration_with_logging(conn: &Connection, version: &str, log_success: bool) -> rusqlite::Result<()> { match version { - // Baseline marker for pre-versioned schemas. "001_initial_schema" => Ok(()), "002_aging_columns" => { migrate_aging_columns_with_logging(conn, log_success); @@ -108,9 +70,7 @@ pub(crate) fn apply_migration_with_logging( { Ok(()) } else { - Err(migration_error( - "aging migration did not create expected columns", - )) + Err(migration_error("aging migration did not create expected columns")) } } "003_focus_table" => { @@ -118,9 +78,7 @@ pub(crate) fn apply_migration_with_logging( if table_exists(conn, "focus_sessions") { Ok(()) } else { - Err(migration_error( - "focus table migration did not create focus_sessions", - )) + Err(migration_error("focus table migration did not create focus_sessions")) } } "004_crystal_tables" => { @@ -128,94 +86,34 @@ pub(crate) fn apply_migration_with_logging( if table_exists(conn, "memory_clusters") && table_exists(conn, "cluster_members") { Ok(()) } else { - Err(migration_error( - "crystal migration did not create memory_clusters/cluster_members", - )) + Err(migration_error("crystal migration did not create memory_clusters/cluster_members")) } } "005_quality_dedup_columns" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN merged_count INTEGER DEFAULT 0", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN quality INTEGER DEFAULT 50", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN merged_count INTEGER DEFAULT 0", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN quality INTEGER DEFAULT 50", - )?; - let _ = conn.execute( - "UPDATE memories SET merged_count = 0 WHERE merged_count IS NULL", - [], - ); + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN merged_count INTEGER DEFAULT 0")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN quality INTEGER DEFAULT 50")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN merged_count INTEGER DEFAULT 0")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN quality INTEGER DEFAULT 50")?; + let _ = conn.execute("UPDATE memories SET merged_count = 0 WHERE merged_count IS NULL", []); let _ = conn.execute("UPDATE memories SET quality = 50 WHERE quality IS NULL", []); - let _ = conn.execute( - "UPDATE decisions SET merged_count = 0 WHERE merged_count IS NULL", - [], - ); - let _ = conn.execute( - "UPDATE decisions SET quality = 50 WHERE quality IS NULL", - [], - ); + let _ = conn.execute("UPDATE decisions SET merged_count = 0 WHERE merged_count IS NULL", []); + let _ = conn.execute("UPDATE decisions SET quality = 50 WHERE quality IS NULL", []); Ok(()) } "006" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN expires_at TEXT", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN expires_at TEXT", - )?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN expires_at TEXT")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN expires_at TEXT")?; Ok(()) } "007" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN merged_count INTEGER DEFAULT 0", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN quality INTEGER DEFAULT 50", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN merged_count INTEGER DEFAULT 0", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN quality INTEGER DEFAULT 50", - )?; - let _ = conn.execute( - "UPDATE memories SET merged_count = 0 WHERE merged_count IS NULL", - [], - ); + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN merged_count INTEGER DEFAULT 0")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN quality INTEGER DEFAULT 50")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN merged_count INTEGER DEFAULT 0")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN quality INTEGER DEFAULT 50")?; + let _ = conn.execute("UPDATE memories SET merged_count = 0 WHERE merged_count IS NULL", []); let _ = conn.execute("UPDATE memories SET quality = 50 WHERE quality IS NULL", []); - let _ = conn.execute( - "UPDATE decisions SET merged_count = 0 WHERE merged_count IS NULL", - [], - ); - let _ = conn.execute( - "UPDATE decisions SET quality = 50 WHERE quality IS NULL", - [], - ); + let _ = conn.execute("UPDATE decisions SET merged_count = 0 WHERE merged_count IS NULL", []); + let _ = conn.execute("UPDATE decisions SET quality = 50 WHERE quality IS NULL", []); Ok(()) } "008" => { @@ -237,47 +135,14 @@ pub(crate) fn apply_migration_with_logging( Ok(()) } "009" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN source_client TEXT DEFAULT 'unknown'", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN source_model TEXT", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN reasoning_depth TEXT DEFAULT 'single-shot'", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN trust_score REAL DEFAULT 0.8", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN source_client TEXT DEFAULT 'unknown'", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN source_model TEXT", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN reasoning_depth TEXT DEFAULT 'single-shot'", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN trust_score REAL DEFAULT 0.8", - )?; - + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN source_client TEXT DEFAULT 'unknown'")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN source_model TEXT")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN reasoning_depth TEXT DEFAULT 'single-shot'")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN trust_score REAL DEFAULT 0.8")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN source_client TEXT DEFAULT 'unknown'")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN source_model TEXT")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN reasoning_depth TEXT DEFAULT 'single-shot'")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN trust_score REAL DEFAULT 0.8")?; let _ = conn.execute( "UPDATE memories SET source_client = COALESCE(NULLIF(lower(source_agent), ''), 'unknown') @@ -320,7 +185,6 @@ pub(crate) fn apply_migration_with_logging( resolved_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); - CREATE INDEX IF NOT EXISTS idx_decision_conflicts_source ON decision_conflicts(source_decision_id); CREATE INDEX IF NOT EXISTS idx_decision_conflicts_target @@ -350,7 +214,6 @@ pub(crate) fn apply_migration_with_logging( notes TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); - CREATE INDEX IF NOT EXISTS idx_agent_feedback_agent_created ON agent_feedback(owner_id, agent, created_at); CREATE INDEX IF NOT EXISTS idx_agent_feedback_task_created @@ -370,42 +233,34 @@ pub(crate) fn apply_migration_with_logging( DROP TRIGGER IF EXISTS decisions_fts_au; DROP TABLE IF EXISTS memories_fts; DROP TABLE IF EXISTS decisions_fts; - CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( text, source, tags, content=memories, content_rowid=id, tokenize='porter unicode61' ); - CREATE VIRTUAL TABLE IF NOT EXISTS decisions_fts USING fts5( decision, context, content=decisions, content_rowid=id, tokenize='porter unicode61' ); - CREATE TRIGGER IF NOT EXISTS memories_fts_ai AFTER INSERT ON memories BEGIN INSERT INTO memories_fts(rowid, text, source, tags) VALUES (new.id, new.text, new.source, new.tags); END; - CREATE TRIGGER IF NOT EXISTS memories_fts_ad AFTER DELETE ON memories BEGIN INSERT INTO memories_fts(memories_fts, rowid, text, source, tags) VALUES('delete', old.id, old.text, old.source, old.tags); END; - CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE ON memories BEGIN INSERT INTO memories_fts(memories_fts, rowid, text, source, tags) VALUES('delete', old.id, old.text, old.source, old.tags); INSERT INTO memories_fts(rowid, text, source, tags) VALUES (new.id, new.text, new.source, new.tags); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_ai AFTER INSERT ON decisions BEGIN INSERT INTO decisions_fts(rowid, decision, context) VALUES (new.id, new.decision, new.context); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_ad AFTER DELETE ON decisions BEGIN INSERT INTO decisions_fts(decisions_fts, rowid, decision, context) VALUES('delete', old.id, old.decision, old.context); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_au AFTER UPDATE ON decisions BEGIN INSERT INTO decisions_fts(decisions_fts, rowid, decision, context) VALUES('delete', old.id, old.decision, old.context); INSERT INTO decisions_fts(rowid, decision, context) VALUES (new.id, new.decision, new.context); @@ -427,9 +282,6 @@ pub(crate) fn apply_migration_with_logging( Ok(()) } "015" => { - // C5 — boot audit trail. One row per /boot call so operators - // can reconstruct "what did Cortex inject into the prompt at - // time T for agent X". 30-day auto-prune runs in handler. conn.execute_batch( r#" CREATE TABLE IF NOT EXISTS boot_audits ( @@ -444,7 +296,6 @@ pub(crate) fn apply_migration_with_logging( latency_ms INTEGER, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); - CREATE INDEX IF NOT EXISTS idx_boot_audits_created_at ON boot_audits(created_at); CREATE INDEX IF NOT EXISTS idx_boot_audits_agent_created @@ -454,49 +305,17 @@ pub(crate) fn apply_migration_with_logging( Ok(()) } "014" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN observed_at TEXT", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN valid_from TEXT", - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN valid_until TEXT", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN observed_at TEXT", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN valid_from TEXT", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN valid_until TEXT", - )?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN observed_at TEXT")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN valid_from TEXT")?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN valid_until TEXT")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN observed_at TEXT")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN valid_from TEXT")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN valid_until TEXT")?; Ok(()) } "016" => { - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN retention_class TEXT NOT NULL DEFAULT 'operational'", - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN retention_class TEXT NOT NULL DEFAULT 'operational'", - )?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN retention_class TEXT NOT NULL DEFAULT 'operational'")?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN retention_class TEXT NOT NULL DEFAULT 'operational'")?; let _ = conn.execute( "UPDATE memories SET retention_class = 'operational' WHERE retention_class IS NULL @@ -521,22 +340,32 @@ pub(crate) fn apply_migration_with_logging( )?; Ok(()) } - other => Err(migration_error(format!( - "unknown schema migration: {other}" - ))), + "017" => { + conn.execute_batch( + r#" + CREATE INDEX IF NOT EXISTS idx_memories_active_source_recent + ON memories(source, COALESCE(last_accessed, created_at) DESC) + WHERE status = 'active'; + CREATE INDEX IF NOT EXISTS idx_decisions_context_status + ON decisions(context, status); + CREATE INDEX IF NOT EXISTS idx_decisions_active_context_recent + ON decisions(context, COALESCE(last_accessed, created_at) DESC) + WHERE status = 'active'; + CREATE INDEX IF NOT EXISTS idx_embeddings_model_type_target_norm + ON embeddings(LOWER(COALESCE(model, '')), target_type, target_id); + "#, + )?; + Ok(()) + } + other => Err(migration_error(format!("unknown schema migration: {other}"))), } } - -/// Return already-applied migration versions. pub fn applied_migration_versions(conn: &Connection) -> rusqlite::Result> { ensure_schema_migrations_table(conn)?; - let mut stmt = - conn.prepare("SELECT version FROM schema_migrations ORDER BY id ASC, version ASC")?; + let mut stmt = conn.prepare("SELECT version FROM schema_migrations ORDER BY id ASC, version ASC")?; let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; Ok(rows.filter_map(|r| r.ok()).collect()) } - -/// Return pending migration versions in execution order. pub fn pending_migration_versions(conn: &Connection) -> rusqlite::Result> { let applied: HashSet = applied_migration_versions(conn)?.into_iter().collect(); let mut pending = Vec::new(); @@ -547,24 +376,17 @@ pub fn pending_migration_versions(conn: &Connection) -> rusqlite::Result usize { run_pending_migrations_with_logging(conn, true) } - -/// Execute pending schema migrations without printing successful ALTERs. pub fn run_pending_migrations_quiet(conn: &Connection) -> usize { run_pending_migrations_with_logging(conn, false) } - pub(crate) fn run_pending_migrations_with_logging(conn: &Connection, log_success: bool) -> usize { if let Err(e) = ensure_schema_migrations_table(conn) { eprintln!("[db] schema migration setup failed: {e}"); return 0; } - let mut applied_set: HashSet = match applied_migration_versions(conn) { Ok(v) => v.into_iter().collect(), Err(e) => { @@ -572,15 +394,11 @@ pub(crate) fn run_pending_migrations_with_logging(conn: &Connection, log_success return 0; } }; - let mut applied_count = 0usize; for (version, name) in migration_definitions() { if applied_set.contains(*version) { continue; } - - // Apply + record in one transaction so we never leave a migration half-recorded. - // `BEGIN IMMEDIATE` prevents concurrent writers from racing this step. let tx = match conn.unchecked_transaction() { Ok(tx) => tx, Err(e) => { @@ -593,10 +411,7 @@ pub(crate) fn run_pending_migrations_with_logging(conn: &Connection, log_success drop(tx); break; } - if let Err(e) = tx.execute( - "INSERT INTO schema_migrations (version, name) VALUES (?1, ?2)", - params![version, name], - ) { + if let Err(e) = tx.execute("INSERT INTO schema_migrations (version, name) VALUES (?1, ?2)", params![version, name]) { eprintln!("[db] failed to record migration {version} ({name}): {e}"); drop(tx); break; @@ -605,14 +420,11 @@ pub(crate) fn run_pending_migrations_with_logging(conn: &Connection, log_success eprintln!("[db] failed to commit migration {version} ({name}): {e}"); break; } - applied_set.insert((*version).to_string()); applied_count += 1; } - if let Err(e) = sync_schema_user_version(conn, &applied_set) { eprintln!("[db] failed to update PRAGMA user_version: {e}"); } - applied_count } diff --git a/daemon-rs/src/db/mod.rs b/daemon-rs/src/db/mod.rs index dc1f2fb1..34fe421c 100644 --- a/daemon-rs/src/db/mod.rs +++ b/daemon-rs/src/db/mod.rs @@ -1,32 +1,24 @@ -// SPDX-License-Identifier: MIT mod connection; +mod maintenance; mod migrations; mod schema; mod team; -mod maintenance; - #[cfg(test)] mod tests; - pub(crate) use connection::*; -pub(crate) use migrations::*; -pub(crate) use schema::*; -pub(crate) use team::*; +pub use connection::{configure, open, sqlite_vec_status, RepairError, RepairResult, SQLITE_BUSY_TIMEOUT_MS}; pub(crate) use maintenance::*; - -pub use connection::{open, configure, sqlite_vec_status, SQLITE_BUSY_TIMEOUT_MS, SQLITE_WAL_AUTOCHECKPOINT_PAGES, RepairResult, RepairError}; +pub use maintenance::{ + auto_repair, checkpoint_wal_best_effort, delete_expired_entries, migrate_focus_table, quick_check, rebuild_fts, rebuild_fts_if_needed, reindex_fts, + verify_integrity, +}; pub use migrations::{ - migration_definitions, latest_schema_user_version, current_schema_user_version, - set_schema_user_version, ensure_schema_migrations_table, applied_migration_versions, - pending_migration_versions, run_pending_migrations, run_pending_migrations_quiet, + applied_migration_versions, current_schema_user_version, migration_definitions, pending_migration_versions, run_pending_migrations, + run_pending_migrations_quiet, }; pub use schema::initialize_schema; +pub(crate) use team::*; pub use team::{ - current_mode, is_team_mode, migration_counts, create_team_mode_tables, upsert_owner_user, - migrate_to_team_mode, ensure_default_team_membership, table_exists, -}; -pub use maintenance::{ - checkpoint_wal_best_effort, delete_expired_entries, ExpiredCleanupCounts, rebuild_fts, - reindex_fts, rebuild_fts_if_needed, verify_integrity, quick_check, auto_repair, - archive_entries_scoped, archive_entries, migrate_focus_table, + create_team_mode_tables, current_mode, ensure_default_team_membership, is_team_mode, migrate_to_team_mode, migration_counts, table_exists, + upsert_owner_user, }; diff --git a/daemon-rs/src/db/schema.rs b/daemon-rs/src/db/schema.rs index 5c0c8b31..e1ed6d43 100644 --- a/daemon-rs/src/db/schema.rs +++ b/daemon-rs/src/db/schema.rs @@ -1,22 +1,9 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashSet; -use std::path::Path; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::OnceLock; -use std::time::{SystemTime, UNIX_EPOCH}; -use rusqlite::{params, Connection, OptionalExtension}; - - -use super::*; - -/// Create all 12 application tables and supporting indexes if they do not -/// already exist. +use rusqlite::Connection; pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( r#" PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; - CREATE TABLE IF NOT EXISTS memories ( id INTEGER PRIMARY KEY AUTOINCREMENT, text TEXT NOT NULL, @@ -48,7 +35,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS decisions ( id INTEGER PRIMARY KEY AUTOINCREMENT, decision TEXT NOT NULL, @@ -81,7 +67,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS decision_conflicts ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_decision_id INTEGER REFERENCES decisions(id) ON DELETE SET NULL, @@ -97,7 +82,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { resolved_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS embeddings ( id INTEGER PRIMARY KEY AUTOINCREMENT, target_type TEXT NOT NULL, @@ -106,7 +90,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { model TEXT DEFAULT 'nomic-embed-text', created_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, type TEXT NOT NULL, @@ -114,7 +97,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { source_agent TEXT, created_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS co_occurrence ( source_a TEXT NOT NULL, source_b TEXT NOT NULL, @@ -122,7 +104,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { last_seen TEXT DEFAULT (datetime('now')), PRIMARY KEY (source_a, source_b) ); - CREATE TABLE IF NOT EXISTS locks ( id TEXT PRIMARY KEY, path TEXT NOT NULL UNIQUE, @@ -130,7 +111,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { locked_at TEXT NOT NULL, expires_at TEXT ); - CREATE TABLE IF NOT EXISTS activities ( id TEXT PRIMARY KEY, agent TEXT NOT NULL, @@ -138,7 +118,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { files_json TEXT NOT NULL DEFAULT '[]', timestamp TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS messages ( id TEXT PRIMARY KEY, sender TEXT NOT NULL, @@ -146,7 +125,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { message TEXT NOT NULL, timestamp TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS sessions ( agent TEXT PRIMARY KEY, session_id TEXT NOT NULL, @@ -157,7 +135,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { last_heartbeat TEXT NOT NULL, expires_at TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS tasks ( task_id TEXT PRIMARY KEY, title TEXT NOT NULL, @@ -173,7 +150,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { completed_at TEXT, summary TEXT ); - CREATE TABLE IF NOT EXISTS feed ( id TEXT PRIMARY KEY, agent TEXT NOT NULL, @@ -187,13 +163,11 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { timestamp TEXT NOT NULL, tokens INTEGER NOT NULL DEFAULT 0 ); - CREATE TABLE IF NOT EXISTS feed_acks ( agent TEXT PRIMARY KEY, last_seen_id TEXT NOT NULL, updated_at TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS client_permissions ( owner_id INTEGER NOT NULL DEFAULT 0, client_id TEXT NOT NULL, @@ -203,17 +177,21 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { granted_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (owner_id, client_id, permission, scope) ); - CREATE INDEX IF NOT EXISTS idx_client_permissions_client ON client_permissions(owner_id, client_id); - CREATE INDEX IF NOT EXISTS idx_cooccur_a ON co_occurrence(source_a); CREATE INDEX IF NOT EXISTS idx_cooccur_b ON co_occurrence(source_b); - -- Performance indexes (added 2026-03-31) CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status); CREATE INDEX IF NOT EXISTS idx_memories_source_status ON memories(source, status); + CREATE INDEX IF NOT EXISTS idx_memories_active_source_recent + ON memories(source, COALESCE(last_accessed, created_at) DESC) + WHERE status = 'active'; CREATE INDEX IF NOT EXISTS idx_decisions_status ON decisions(status); + CREATE INDEX IF NOT EXISTS idx_decisions_context_status ON decisions(context, status); + CREATE INDEX IF NOT EXISTS idx_decisions_active_context_recent + ON decisions(context, COALESCE(last_accessed, created_at) DESC) + WHERE status = 'active'; CREATE INDEX IF NOT EXISTS idx_decision_conflicts_source ON decision_conflicts(source_decision_id); CREATE INDEX IF NOT EXISTS idx_decision_conflicts_target ON decision_conflicts(target_decision_id); CREATE INDEX IF NOT EXISTS idx_decision_conflicts_status_created ON decision_conflicts(status, created_at); @@ -222,6 +200,8 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { ON embeddings(LOWER(COALESCE(model, ''))); CREATE INDEX IF NOT EXISTS idx_embeddings_target_model_norm ON embeddings(target_type, target_id, LOWER(COALESCE(model, ''))); + CREATE INDEX IF NOT EXISTS idx_embeddings_model_type_target_norm + ON embeddings(LOWER(COALESCE(model, '')), target_type, target_id); CREATE INDEX IF NOT EXISTS idx_events_type_created ON events(type, created_at); CREATE INDEX IF NOT EXISTS idx_events_created ON events(created_at); CREATE INDEX IF NOT EXISTS idx_events_type_id ON events(type, id); @@ -236,7 +216,6 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status); CREATE INDEX IF NOT EXISTS idx_tasks_status_created ON tasks(status, created_at); CREATE INDEX IF NOT EXISTS idx_locks_expires ON locks(expires_at); - CREATE TABLE IF NOT EXISTS context_cache ( cache_key TEXT PRIMARY KEY, content_hash TEXT NOT NULL, @@ -245,14 +224,12 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { created_at TEXT DEFAULT (datetime('now')), hits INTEGER DEFAULT 0 ); - CREATE TABLE IF NOT EXISTS schema_migrations ( id INTEGER PRIMARY KEY AUTOINCREMENT, version TEXT NOT NULL UNIQUE, name TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')) ); - -- FTS5 full-text search indexes (porter+unicode61 for stemming + unicode tokenization) CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( text, source, tags, @@ -260,14 +237,12 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { content_rowid=id, tokenize='porter unicode61' ); - CREATE VIRTUAL TABLE IF NOT EXISTS decisions_fts USING fts5( decision, context, content=decisions, content_rowid=id, tokenize='porter unicode61' ); - -- Relevance feedback: tracks which recalled results were actually useful CREATE TABLE IF NOT EXISTS recall_feedback ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -280,10 +255,8 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { agent TEXT NOT NULL DEFAULT 'unknown', created_at TEXT DEFAULT (datetime('now')) ); - CREATE INDEX IF NOT EXISTS idx_feedback_result ON recall_feedback(result_source); CREATE INDEX IF NOT EXISTS idx_feedback_created ON recall_feedback(created_at); - CREATE TABLE IF NOT EXISTS event_savings_rollups ( day TEXT NOT NULL, hour INTEGER NOT NULL, @@ -298,12 +271,10 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { updated_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (day, hour, operation) ); - CREATE INDEX IF NOT EXISTS idx_event_savings_rollups_day ON event_savings_rollups(day); CREATE INDEX IF NOT EXISTS idx_event_savings_rollups_operation_day ON event_savings_rollups(operation, day); - CREATE TABLE IF NOT EXISTS agent_feedback ( id INTEGER PRIMARY KEY AUTOINCREMENT, owner_id INTEGER NOT NULL DEFAULT 0, @@ -320,34 +291,27 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { notes TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); - CREATE INDEX IF NOT EXISTS idx_agent_feedback_agent_created ON agent_feedback(owner_id, agent, created_at); CREATE INDEX IF NOT EXISTS idx_agent_feedback_task_created ON agent_feedback(owner_id, task_class, created_at); - -- Triggers to keep FTS in sync with base tables CREATE TRIGGER IF NOT EXISTS memories_fts_ai AFTER INSERT ON memories BEGIN INSERT INTO memories_fts(rowid, text, source, tags) VALUES (new.id, new.text, new.source, new.tags); END; - CREATE TRIGGER IF NOT EXISTS memories_fts_ad AFTER DELETE ON memories BEGIN INSERT INTO memories_fts(memories_fts, rowid, text, source, tags) VALUES('delete', old.id, old.text, old.source, old.tags); END; - CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE ON memories BEGIN INSERT INTO memories_fts(memories_fts, rowid, text, source, tags) VALUES('delete', old.id, old.text, old.source, old.tags); INSERT INTO memories_fts(rowid, text, source, tags) VALUES (new.id, new.text, new.source, new.tags); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_ai AFTER INSERT ON decisions BEGIN INSERT INTO decisions_fts(rowid, decision, context) VALUES (new.id, new.decision, new.context); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_ad AFTER DELETE ON decisions BEGIN INSERT INTO decisions_fts(decisions_fts, rowid, decision, context) VALUES('delete', old.id, old.decision, old.context); END; - CREATE TRIGGER IF NOT EXISTS decisions_fts_au AFTER UPDATE ON decisions BEGIN INSERT INTO decisions_fts(decisions_fts, rowid, decision, context) VALUES('delete', old.id, old.decision, old.context); INSERT INTO decisions_fts(rowid, decision, context) VALUES (new.id, new.decision, new.context); @@ -356,4 +320,3 @@ pub fn initialize_schema(conn: &Connection) -> rusqlite::Result<()> { )?; Ok(()) } - diff --git a/daemon-rs/src/db/team.rs b/daemon-rs/src/db/team.rs index bf57bec1..6df7d906 100644 --- a/daemon-rs/src/db/team.rs +++ b/daemon-rs/src/db/team.rs @@ -1,35 +1,14 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashSet; -use std::path::Path; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::OnceLock; -use std::time::{SystemTime, UNIX_EPOCH}; -use rusqlite::{params, Connection, OptionalExtension}; - - -use super::*; -/// Return the current runtime mode (`solo` or `team`). +use rusqlite::{params, Connection}; pub fn current_mode(conn: &Connection) -> String { if !table_exists(conn, "config") { return "solo".to_string(); } - conn.query_row( - "SELECT value FROM config WHERE key = 'mode' LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) - .unwrap_or_else(|_| "solo".to_string()) + conn.query_row("SELECT value FROM config WHERE key = 'mode' LIMIT 1", [], |row| row.get::<_, String>(0)) + .unwrap_or_else(|_| "solo".to_string()) } - -/// Check whether the database is in team mode. pub fn is_team_mode(conn: &Connection) -> bool { current_mode(conn) == "team" } - -/// Per-table row counts for owner-aware tables after migration. -/// -/// Returns `(table_name, count)` pairs. If a table lacks `owner_id` (solo mode), -/// its count is 0 rather than erroring. pub fn migration_counts(conn: &Connection) -> Vec<(String, i64)> { pub(crate) const TABLES: &[&str] = &[ "memories", @@ -45,17 +24,12 @@ pub fn migration_counts(conn: &Connection) -> Vec<(String, i64)> { "activities", "focus_sessions", ]; - TABLES .iter() .map(|&table| { let count = if table_has_column(conn, table, "owner_id") { - conn.query_row( - &format!("SELECT COUNT(*) FROM {table} WHERE owner_id IS NOT NULL"), - [], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0) + conn.query_row(&format!("SELECT COUNT(*) FROM {table} WHERE owner_id IS NOT NULL"), [], |row| row.get::<_, i64>(0)) + .unwrap_or(0) } else { 0 }; @@ -63,8 +37,6 @@ pub fn migration_counts(conn: &Connection) -> Vec<(String, i64)> { }) .collect() } - -/// Create the base team-mode tables (`config`, `users`, `teams`, `team_members`). pub fn create_team_mode_tables(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( r#" @@ -72,7 +44,6 @@ pub fn create_team_mode_tables(conn: &Connection) -> rusqlite::Result<()> { key TEXT PRIMARY KEY, value TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, @@ -83,14 +54,12 @@ pub fn create_team_mode_tables(conn: &Connection) -> rusqlite::Result<()> { created_at TEXT DEFAULT (datetime('now')), last_active_at TEXT ); - CREATE TABLE IF NOT EXISTS teams ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE NOT NULL, parent_team_id INTEGER REFERENCES teams(id), created_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS team_members ( team_id INTEGER NOT NULL REFERENCES teams(id) ON DELETE CASCADE, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, @@ -101,20 +70,10 @@ pub fn create_team_mode_tables(conn: &Connection) -> rusqlite::Result<()> { ); "#, )?; - conn.execute( - "INSERT OR IGNORE INTO config (key, value) VALUES ('mode', 'solo')", - [], - )?; + conn.execute("INSERT OR IGNORE INTO config (key, value) VALUES ('mode', 'solo')", [])?; Ok(()) } - -/// Create or rotate the owner user entry and return its `users.id`. -pub fn upsert_owner_user( - conn: &Connection, - username: &str, - display_name: Option<&str>, - api_key_hash: &str, -) -> rusqlite::Result { +pub fn upsert_owner_user(conn: &Connection, username: &str, display_name: Option<&str>, api_key_hash: &str) -> rusqlite::Result { conn.execute( "INSERT INTO users (username, display_name, api_key_hash, role) VALUES (?1, ?2, ?3, 'owner') @@ -124,122 +83,29 @@ pub fn upsert_owner_user( role = 'owner'", params![username, display_name, api_key_hash], )?; - - conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![username], - |row| row.get::<_, i64>(0), - ) + conn.query_row("SELECT id FROM users WHERE username = ?1", params![username], |row| row.get::<_, i64>(0)) } - -/// Apply team-mode schema migration on top of an existing solo database. -/// -/// This is idempotent and safe to call repeatedly. pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Result<()> { create_team_mode_tables(conn)?; - - // Core memory tables. - ensure_column( - conn, - "memories", - &format!( - "ALTER TABLE memories ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - ensure_column( - conn, - "memories", - "ALTER TABLE memories ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))", - )?; - - ensure_column( - conn, - "decisions", - &format!( - "ALTER TABLE decisions ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - ensure_column( - conn, - "decisions", - "ALTER TABLE decisions ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))", - )?; - - // Crystal tables are named memory_clusters / cluster_members in this codebase. - ensure_column( - conn, - "memory_clusters", - &format!( - "ALTER TABLE memory_clusters ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; + ensure_column(conn, "memories", &format!("ALTER TABLE memories ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "memories", "ALTER TABLE memories ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))")?; + ensure_column(conn, "decisions", &format!("ALTER TABLE decisions ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "decisions", "ALTER TABLE decisions ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))")?; + ensure_column(conn, "memory_clusters", &format!("ALTER TABLE memory_clusters ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; ensure_column( conn, "memory_clusters", "ALTER TABLE memory_clusters ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))", )?; - - ensure_column( - conn, - "recall_feedback", - &format!( - "ALTER TABLE recall_feedback ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - - // Conductor tables. - ensure_column( - conn, - "tasks", - &format!( - "ALTER TABLE tasks ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - ensure_column( - conn, - "tasks", - "ALTER TABLE tasks ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))", - )?; - - ensure_column( - conn, - "messages", - &format!( - "ALTER TABLE messages ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - - ensure_column( - conn, - "feed", - &format!( - "ALTER TABLE feed ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - ensure_column( - conn, - "feed", - "ALTER TABLE feed ADD COLUMN visibility TEXT DEFAULT 'team' CHECK (visibility IN ('private', 'team', 'shared'))", - )?; - - ensure_column( - conn, - "focus_sessions", - &format!( - "ALTER TABLE focus_sessions ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - ensure_column( - conn, - "activities", - &format!( - "ALTER TABLE activities ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)" - ), - )?; - - // Recreate sessions table for owner-scoped uniqueness. - if !table_has_column(conn, "sessions", "id") || !table_has_column(conn, "sessions", "owner_id") - { + ensure_column(conn, "recall_feedback", &format!("ALTER TABLE recall_feedback ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "tasks", &format!("ALTER TABLE tasks ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "tasks", "ALTER TABLE tasks ADD COLUMN visibility TEXT DEFAULT 'private' CHECK (visibility IN ('private', 'team', 'shared'))")?; + ensure_column(conn, "messages", &format!("ALTER TABLE messages ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "feed", &format!("ALTER TABLE feed ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "feed", "ALTER TABLE feed ADD COLUMN visibility TEXT DEFAULT 'team' CHECK (visibility IN ('private', 'team', 'shared'))")?; + ensure_column(conn, "focus_sessions", &format!("ALTER TABLE focus_sessions ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + ensure_column(conn, "activities", &format!("ALTER TABLE activities ADD COLUMN owner_id INTEGER DEFAULT {owner_id} REFERENCES users(id)"))?; + if !table_has_column(conn, "sessions", "id") || !table_has_column(conn, "sessions", "owner_id") { conn.execute_batch("DROP TABLE IF EXISTS sessions_new;")?; conn.execute_batch(&format!( "CREATE TABLE sessions_new ( @@ -266,8 +132,6 @@ pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Resul } conn.execute_batch("ALTER TABLE sessions_new RENAME TO sessions;")?; } - - // Recreate locks table for owner-scoped uniqueness. if !table_has_column(conn, "locks", "owner_id") { conn.execute_batch("DROP TABLE IF EXISTS locks_new;")?; conn.execute_batch(&format!( @@ -291,8 +155,6 @@ pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Resul } conn.execute_batch("ALTER TABLE locks_new RENAME TO locks;")?; } - - // Recreate feed_acks table for owner-scoped composite primary key. if !table_has_column(conn, "feed_acks", "owner_id") { conn.execute_batch("DROP TABLE IF EXISTS feed_acks_new;")?; conn.execute_batch(&format!( @@ -314,8 +176,6 @@ pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Resul } conn.execute_batch("ALTER TABLE feed_acks_new RENAME TO feed_acks;")?; } - - // Backfill ownership and sensible defaults. for table in [ "memories", "decisions", @@ -335,28 +195,11 @@ pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Resul let _ = conn.execute(&sql, params![owner_id])?; } } - let _ = conn.execute( - "UPDATE memories SET visibility = 'private' WHERE visibility IS NULL", - [], - )?; - let _ = conn.execute( - "UPDATE decisions SET visibility = 'private' WHERE visibility IS NULL", - [], - )?; - let _ = conn.execute( - "UPDATE memory_clusters SET visibility = 'private' WHERE visibility IS NULL", - [], - )?; - let _ = conn.execute( - "UPDATE tasks SET visibility = 'private' WHERE visibility IS NULL", - [], - )?; - let _ = conn.execute( - "UPDATE feed SET visibility = 'team' WHERE visibility IS NULL", - [], - )?; - - // Team indexes. + let _ = conn.execute("UPDATE memories SET visibility = 'private' WHERE visibility IS NULL", [])?; + let _ = conn.execute("UPDATE decisions SET visibility = 'private' WHERE visibility IS NULL", [])?; + let _ = conn.execute("UPDATE memory_clusters SET visibility = 'private' WHERE visibility IS NULL", [])?; + let _ = conn.execute("UPDATE tasks SET visibility = 'private' WHERE visibility IS NULL", [])?; + let _ = conn.execute("UPDATE feed SET visibility = 'team' WHERE visibility IS NULL", [])?; conn.execute_batch( r#" CREATE INDEX IF NOT EXISTS idx_memories_owner ON memories(owner_id) WHERE owner_id IS NOT NULL; @@ -384,44 +227,23 @@ pub fn migrate_to_team_mode(conn: &Connection, owner_id: i64) -> rusqlite::Resul "#, )?; conn.execute("INSERT OR IGNORE INTO teams (name) VALUES ('default')", [])?; - let default_team_id: i64 = conn.query_row( - "SELECT id FROM teams WHERE name = 'default' LIMIT 1", - [], - |row| row.get(0), - )?; - conn.execute( - "INSERT OR IGNORE INTO team_members (team_id, user_id, role) VALUES (?1, ?2, 'admin')", - params![default_team_id, owner_id], - )?; - - conn.execute( - "INSERT OR IGNORE INTO config (key, value) VALUES ('mode', 'solo')", - [], - )?; + let default_team_id: i64 = conn.query_row("SELECT id FROM teams WHERE name = 'default' LIMIT 1", [], |row| row.get(0))?; + conn.execute("INSERT OR IGNORE INTO team_members (team_id, user_id, role) VALUES (?1, ?2, 'admin')", params![default_team_id, owner_id])?; + conn.execute("INSERT OR IGNORE INTO config (key, value) VALUES ('mode', 'solo')", [])?; conn.execute("UPDATE config SET value = 'team' WHERE key = 'mode'", [])?; conn.execute( "INSERT INTO config (key, value) VALUES ('owner_user_id', ?1) ON CONFLICT(key) DO UPDATE SET value = excluded.value", params![owner_id.to_string()], )?; - Ok(()) } - -/// Ensure a default team exists and owner is a member/admin. pub fn ensure_default_team_membership(conn: &Connection, owner_id: i64) -> rusqlite::Result { conn.execute("INSERT OR IGNORE INTO teams (name) VALUES ('default')", [])?; - let team_id: i64 = - conn.query_row("SELECT id FROM teams WHERE name = 'default'", [], |row| { - row.get(0) - })?; - conn.execute( - "INSERT OR IGNORE INTO team_members (team_id, user_id, role) VALUES (?1, ?2, 'admin')", - params![team_id, owner_id], - )?; + let team_id: i64 = conn.query_row("SELECT id FROM teams WHERE name = 'default'", [], |row| row.get(0))?; + conn.execute("INSERT OR IGNORE INTO team_members (team_id, user_id, role) VALUES (?1, ?2, 'admin')", params![team_id, owner_id])?; Ok(team_id) } - pub(crate) fn ensure_column(conn: &Connection, table: &str, alter_sql: &str) -> rusqlite::Result<()> { if !table_exists(conn, table) { return Ok(()); @@ -432,16 +254,10 @@ pub(crate) fn ensure_column(conn: &Connection, table: &str, alter_sql: &str) -> Err(e) => Err(e), } } - pub fn table_exists(conn: &Connection, table: &str) -> bool { - conn.query_row( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ?1 LIMIT 1", - params![table], - |_| Ok(()), - ) - .is_ok() + conn.query_row("SELECT 1 FROM sqlite_master WHERE type='table' AND name = ?1 LIMIT 1", params![table], |_| Ok(())) + .is_ok() } - pub(crate) fn table_has_column(conn: &Connection, table: &str, column: &str) -> bool { if !table_exists(conn, table) { return false; @@ -461,4 +277,3 @@ pub(crate) fn table_has_column(conn: &Connection, table: &str, column: &str) -> } false } - diff --git a/daemon-rs/src/db/tests.rs b/daemon-rs/src/db/tests/mod.rs similarity index 65% rename from daemon-rs/src/db/tests.rs rename to daemon-rs/src/db/tests/mod.rs index 2b4f36b8..b4ef9f10 100644 --- a/daemon-rs/src/db/tests.rs +++ b/daemon-rs/src/db/tests/mod.rs @@ -1,14 +1,8 @@ // SPDX-License-Identifier: MIT -//! Schema and migration integrity only. See Info/testing-philosophy.md. - #[cfg(test)] mod tests { - use crate::db::{ - configure, delete_expired_entries, initialize_schema, rebuild_fts_if_needed, - run_pending_migrations, - }; + use crate::db::{configure, delete_expired_entries, initialize_schema, rebuild_fts_if_needed, run_pending_migrations}; use rusqlite::Connection; - #[test] fn open_configure_schema_roundtrip() { let conn = Connection::open_in_memory().expect("open db"); @@ -16,40 +10,25 @@ mod tests { initialize_schema(&conn).expect("schema"); run_pending_migrations(&conn); let tables: i64 = conn - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('decisions', 'memories', 'events')", - [], - |row| row.get(0), - ) + .query_row("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('decisions', 'memories', 'events')", [], |row| row.get(0)) .expect("count tables"); assert_eq!(tables, 3); } - #[test] fn run_pending_migrations_is_idempotent() { - let path = std::env::temp_dir().join(format!( - "cortex-db-migrate-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock") - .as_nanos() - )); + let path = std::env::temp_dir() + .join(format!("cortex-db-migrate-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).expect("clock").as_nanos())); let conn = Connection::open(&path).expect("open file db"); configure(&conn).expect("configure"); initialize_schema(&conn).expect("schema"); run_pending_migrations(&conn); - let first: i64 = conn - .query_row("SELECT COUNT(*) FROM schema_migrations", [], |row| row.get(0)) - .unwrap_or(0); + let first: i64 = conn.query_row("SELECT COUNT(*) FROM schema_migrations", [], |row| row.get(0)).unwrap_or(0); run_pending_migrations(&conn); - let second: i64 = conn - .query_row("SELECT COUNT(*) FROM schema_migrations", [], |row| row.get(0)) - .unwrap_or(0); + let second: i64 = conn.query_row("SELECT COUNT(*) FROM schema_migrations", [], |row| row.get(0)).unwrap_or(0); assert_eq!(first, second); drop(conn); let _ = std::fs::remove_file(path); } - #[test] fn delete_expired_entries_removes_only_expired_rows() { let conn = Connection::open_in_memory().expect("open db"); @@ -70,12 +49,9 @@ mod tests { .unwrap(); let removed = delete_expired_entries(&conn).expect("delete expired"); assert_eq!(removed.decisions_deleted, 1); - let remaining: i64 = conn - .query_row("SELECT COUNT(*) FROM decisions", [], |row| row.get(0)) - .unwrap(); + let remaining: i64 = conn.query_row("SELECT COUNT(*) FROM decisions", [], |row| row.get(0)).unwrap(); assert_eq!(remaining, 1); } - #[test] fn rebuild_fts_if_needed_rebuilds_empty_index() { let conn = Connection::open_in_memory().expect("open db"); @@ -90,11 +66,7 @@ mod tests { .unwrap(); rebuild_fts_if_needed(&conn).expect("fts seed"); let hits: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions_fts WHERE decisions_fts MATCH 'smoke'", - [], - |row| row.get(0), - ) + .query_row("SELECT COUNT(*) FROM decisions_fts WHERE decisions_fts MATCH 'smoke'", [], |row| row.get(0)) .unwrap_or(0); assert!(hits >= 1); } diff --git a/daemon-rs/src/embeddings/download.rs b/daemon-rs/src/embeddings/download.rs index 89d07183..6ef42ce3 100644 --- a/daemon-rs/src/embeddings/download.rs +++ b/daemon-rs/src/embeddings/download.rs @@ -1,31 +1,17 @@ -// SPDX-License-Identifier: MIT +use super::profiles::resolve_profile; use std::io::Write; use std::path::{Path, PathBuf}; - -use super::profiles::resolve_profile; - -/// Return the models directory, downloading missing files from HuggingFace if -/// necessary. Returns `None` on download failure (keyword-only search will be -/// used as a fallback). pub async fn ensure_model_downloaded() -> Option { let models_dir = dirs::home_dir()?.join(".cortex").join("models"); ensure_model_downloaded_in(&models_dir).await } - -/// Ensure embedding assets exist in a specific models directory. pub async fn ensure_model_downloaded_in(models_dir: &Path) -> Option { let profile = resolve_profile(); std::fs::create_dir_all(models_dir).ok()?; - if profile.assets_exist(models_dir) { return Some(models_dir.to_path_buf()); } - - eprintln!( - "[embeddings] Downloading embedding model '{}' (first run)...", - profile.display_name - ); - + eprintln!("[embeddings] Downloading embedding model '{}' (first run)...", profile.display_name); for asset in profile.missing_assets(models_dir) { let asset_path = models_dir.join(asset.file); match download_file(asset.url, &asset_path).await { @@ -36,40 +22,24 @@ pub async fn ensure_model_downloaded_in(models_dir: &Path) -> Option { } } } - Some(models_dir.to_path_buf()) } - async fn download_file(url: &str, dest: &Path) -> Result<(), String> { if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; } - - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(600)) - .build() - .map_err(|e| e.to_string())?; - + let client = reqwest::Client::builder().timeout(std::time::Duration::from_secs(600)).build().map_err(|e| e.to_string())?; let mut resp = client.get(url).send().await.map_err(|e| e.to_string())?; - if !resp.status().is_success() { return Err(format!("HTTP {}", resp.status())); } - - let tmp_dest = dest.with_file_name(format!( - "{}.tmp", - dest.file_name() - .and_then(|name| name.to_str()) - .unwrap_or("download") - )); + let tmp_dest = dest.with_file_name(format!("{}.tmp", dest.file_name().and_then(|name| name.to_str()).unwrap_or("download"))); let mut file = std::fs::File::create(&tmp_dest).map_err(|e| e.to_string())?; while let Some(chunk) = resp.chunk().await.map_err(|e| e.to_string())? { file.write_all(&chunk).map_err(|e| e.to_string())?; } file.sync_all().map_err(|e| e.to_string())?; drop(file); - std::fs::rename(&tmp_dest, dest).map_err(|e| e.to_string())?; - Ok(()) } diff --git a/daemon-rs/src/embeddings/engine.rs b/daemon-rs/src/embeddings/engine.rs index eca5d35f..acc3ca05 100644 --- a/daemon-rs/src/embeddings/engine.rs +++ b/daemon-rs/src/embeddings/engine.rs @@ -1,15 +1,9 @@ -// SPDX-License-Identifier: MIT +use super::profiles::{resolve_profile, resolved_pool_size, EmbeddingInputKind, PoolingStrategy, TEXT_TRUNCATE_BYTES}; use ort::session::Session; use ort::value::Tensor; use std::borrow::Cow; -use std::path::{Path, PathBuf}; +use std::path::Path; use tokenizers::Tokenizer; - -use super::profiles::{ - resolve_profile, resolved_pool_size, EmbeddingInputKind, EmbeddingModelProfile, PoolingStrategy, - TEXT_TRUNCATE_BYTES, -}; - pub struct EmbeddingEngine { sessions: Vec>, next: std::sync::atomic::AtomicUsize, @@ -23,10 +17,7 @@ pub struct EmbeddingEngine { normalize: bool, include_token_type_ids: bool, } - impl EmbeddingEngine { - /// Try to load from cached model files. Returns `None` when files are - /// missing or corrupt. Opens `POOL_SIZE` independent ONNX sessions. pub fn load(models_dir: &Path) -> Option { match Self::try_load(models_dir) { Ok(engine) => Some(engine), @@ -36,40 +27,23 @@ impl EmbeddingEngine { } } } - fn try_load(models_dir: &Path) -> Result { let profile = resolve_profile(); let pool_size = resolved_pool_size(); let model_path = models_dir.join(profile.model_file); let tok_path = models_dir.join(profile.tokenizer_file); - let missing_assets = profile.missing_assets(models_dir); if !missing_assets.is_empty() { - let missing = missing_assets - .iter() - .map(|asset| asset.file) - .collect::>() - .join(", "); - return Err(format!( - "model assets missing ({missing}) at {}", - models_dir.display() - )); + let missing = missing_assets.iter().map(|asset| asset.file).collect::>().join(", "); + return Err(format!("model assets missing ({missing}) at {}", models_dir.display())); } - - let tokenizer = Tokenizer::from_file(&tok_path) - .map_err(|error| format!("failed to load tokenizer {}: {error}", tok_path.display()))?; - + let tokenizer = Tokenizer::from_file(&tok_path).map_err(|error| format!("failed to load tokenizer {}: {error}", tok_path.display()))?; let mut sessions = Vec::with_capacity(pool_size); for index in 0..pool_size { - let session = Self::build_session(&model_path) - .map_err(|error| format!("session {} failed: {error}", index + 1))?; + let session = Self::build_session(&model_path).map_err(|error| format!("session {} failed: {error}", index + 1))?; sessions.push(std::sync::Mutex::new(session)); } - - eprintln!( - "[embeddings] Session pool: {pool_size} sessions loaded for {}", - profile.display_name, - ); + eprintln!("[embeddings] Session pool: {pool_size} sessions loaded for {}", profile.display_name,); Ok(Self { sessions, next: std::sync::atomic::AtomicUsize::new(0), @@ -84,44 +58,27 @@ impl EmbeddingEngine { include_token_type_ids: profile.include_token_type_ids, }) } - fn build_session(model_path: &Path) -> Result { let tuned = Session::builder() .map_err(|error| format!("session builder init failed: {error}")) - .and_then(|builder| { - builder - .with_intra_threads(2) - .map_err(|error| format!("with_intra_threads(2) failed: {error}")) - }) + .and_then(|builder| builder.with_intra_threads(2).map_err(|error| format!("with_intra_threads(2) failed: {error}"))) .and_then(|mut builder| { - builder.commit_from_file(model_path).map_err(|error| { - format!( - "commit_from_file (tuned threads) failed for {}: {error}", - model_path.display() - ) - }) + builder + .commit_from_file(model_path) + .map_err(|error| format!("commit_from_file (tuned threads) failed for {}: {error}", model_path.display())) }); - match tuned { Ok(session) => Ok(session), Err(tuned_error) => { let fallback = Session::builder() .map_err(|error| format!("session builder fallback init failed: {error}"))? .commit_from_file(model_path) - .map_err(|error| { - format!( - "commit_from_file (fallback threads) failed for {}: {error}", - model_path.display() - ) - })?; - eprintln!( - "[embeddings] Falling back to default ORT session threading after tuned setup failed: {tuned_error}" - ); + .map_err(|error| format!("commit_from_file (fallback threads) failed for {}: {error}", model_path.display()))?; + eprintln!("[embeddings] Falling back to default ORT session threading after tuned setup failed: {tuned_error}"); Ok(fallback) } } } - fn truncate_to_char_boundary(text: &str, max_bytes: usize) -> &str { if text.len() <= max_bytes { return text; @@ -132,7 +89,6 @@ impl EmbeddingEngine { } &text[..end] } - fn input_text<'a>(&self, text: &'a str, kind: EmbeddingInputKind) -> Cow<'a, str> { let prefix = match kind { EmbeddingInputKind::Query => self.query_prefix, @@ -144,16 +100,13 @@ impl EmbeddingEngine { Cow::Owned(format!("{prefix}{text}")) } } - fn embed_with_kind(&self, text: &str, kind: EmbeddingInputKind) -> Option> { let input = self.input_text(text, kind); let truncated = Self::truncate_to_char_boundary(input.as_ref(), TEXT_TRUNCATE_BYTES); let encoding = self.tokenizer.encode(truncated, true).ok()?; - let ids = encoding.get_ids(); let attention = encoding.get_attention_mask(); let type_ids = encoding.get_type_ids(); - let len = ids.len().min(self.max_input_tokens); if len == 0 { return None; @@ -161,83 +114,48 @@ impl EmbeddingEngine { let ids = &ids[..len]; let attention = &attention[..len]; let type_ids = &type_ids[..len]; - let shape = vec![1i64, len as i64]; let ids_vec: Vec = ids.iter().map(|&x| x as i64).collect(); let mask_vec: Vec = attention.iter().map(|&x| x as i64).collect(); let type_vec: Vec = type_ids.iter().map(|&x| x as i64).collect(); - let ids_tensor = Tensor::from_array((shape.clone(), ids_vec)).ok()?; let mask_tensor = Tensor::from_array((shape.clone(), mask_vec)).ok()?; let type_tensor = Tensor::from_array((shape, type_vec)).ok()?; - - // Round-robin session selection across the configured session pool. - let idx = - self.next.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % self.sessions.len(); + let idx = self.next.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % self.sessions.len(); let mut session = self.sessions[idx].lock().ok()?; let outputs = if self.include_token_type_ids { - session.run(ort::inputs![ - "input_ids" => ids_tensor, - "attention_mask" => mask_tensor, - "token_type_ids" => type_tensor, - ]) + session.run(ort::inputs!["input_ids"=>ids_tensor,"attention_mask"=>mask_tensor, +"token_type_ids"=>type_tensor,]) } else { - session.run(ort::inputs![ - "input_ids" => ids_tensor, - "attention_mask" => mask_tensor, - ]) + session.run(ort::inputs!["input_ids"=>ids_tensor,"attention_mask"=>mask_tensor,]) } .ok()?; - let (shape, data) = outputs[0].try_extract_tensor::().ok()?; let dims: Vec = shape.iter().copied().collect(); - if dims.len() != 3 || dims[2] as usize != self.dimension { eprintln!("[embeddings] Unexpected output shape: {dims:?}"); return None; } - let seq_len_out = dims[1] as usize; - Self::pool_output( - data, - self.dimension, - seq_len_out, - attention, - self.pooling, - self.normalize, - ) + Self::pool_output(data, self.dimension, seq_len_out, attention, self.pooling, self.normalize) } - - fn pool_output( - data: &[f32], - dimension: usize, - seq_len_out: usize, - attention: &[u32], - pooling: PoolingStrategy, - normalize: bool, - ) -> Option> { + fn pool_output(data: &[f32], dimension: usize, seq_len_out: usize, attention: &[u32], pooling: PoolingStrategy, normalize: bool) -> Option> { if dimension == 0 || seq_len_out == 0 || data.len() < seq_len_out * dimension { return None; } - let mut pooled = vec![0.0f32; dimension]; match pooling { PoolingStrategy::Mean => { let mut mask_sum = 0.0f32; let attention_fallback_index = attention.len().saturating_sub(1); for seq_idx in 0..seq_len_out { - let mask_val = attention - .get(seq_idx) - .or_else(|| attention.get(attention_fallback_index)) - .copied() - .unwrap_or(1) as f32; + let mask_val = attention.get(seq_idx).or_else(|| attention.get(attention_fallback_index)).copied().unwrap_or(1) as f32; mask_sum += mask_val; let offset = seq_idx * dimension; for dim in 0..dimension { pooled[dim] += data[offset + dim] * mask_val; } } - if mask_sum > 0.0 { for v in &mut pooled { *v /= mask_sum; @@ -249,16 +167,11 @@ impl EmbeddingEngine { } PoolingStrategy::LastToken => { let attention_limit = seq_len_out.min(attention.len()); - let last_idx = attention - .iter() - .take(attention_limit) - .rposition(|mask| *mask != 0) - .unwrap_or(seq_len_out - 1); + let last_idx = attention.iter().take(attention_limit).rposition(|mask| *mask != 0).unwrap_or(seq_len_out - 1); let offset = last_idx * dimension; pooled.copy_from_slice(data.get(offset..offset + dimension)?); } } - if normalize { let norm: f32 = pooled.iter().map(|x| x * x).sum::().sqrt(); if norm > 0.0 { @@ -267,43 +180,24 @@ impl EmbeddingEngine { } } } - Some(pooled) } - - /// Generate a passage embedding for `text` using the selected profile. pub fn embed(&self, text: &str) -> Option> { self.embed_with_kind(text, EmbeddingInputKind::Passage) } - - /// Generate a query embedding for retrieval. Profiles such as BGE apply a - /// query instruction prefix here while stored passages remain unprefixed. pub fn embed_query(&self, text: &str) -> Option> { self.embed_with_kind(text, EmbeddingInputKind::Query) } - pub async fn embed_async(self: std::sync::Arc, text: String) -> Option> { - tokio::task::spawn_blocking(move || self.embed(&text)) - .await - .ok() - .flatten() + tokio::task::spawn_blocking(move || self.embed(&text)).await.ok().flatten() } - pub async fn embed_query_async(self: std::sync::Arc, text: String) -> Option> { - tokio::task::spawn_blocking(move || self.embed_query(&text)) - .await - .ok() - .flatten() + tokio::task::spawn_blocking(move || self.embed_query(&text)).await.ok().flatten() } - pub fn dimension(&self) -> usize { self.dimension } - pub fn model_key(&self) -> &'static str { self.model_key } } - -// --------------------------------------------------------------------------- -// Vector utilities diff --git a/daemon-rs/src/embeddings/mod.rs b/daemon-rs/src/embeddings/mod.rs index 59ff9e8e..23cf32ca 100644 --- a/daemon-rs/src/embeddings/mod.rs +++ b/daemon-rs/src/embeddings/mod.rs @@ -1,23 +1,12 @@ -// SPDX-License-Identifier: MIT -//! In-process ONNX embedding engine. - -mod profiles; -mod engine; -mod vectors; mod download; - +mod engine; +mod profiles; #[cfg(test)] -mod tests { - // Embedding model internals are not release-gated; see Info/testing-philosophy.md. -} - -pub use profiles::{ - selected_model_assets_exist, selected_model_key, selected_model_selection, EmbeddingModelSelection, -}; +mod tests; +mod vectors; +pub use download::{ensure_model_downloaded, ensure_model_downloaded_in}; pub use engine::EmbeddingEngine; +pub use profiles::{selected_model_assets_exist, selected_model_key, selected_model_selection}; pub use vectors::{ - blob_to_vector, cosine_similarity, is_pq8_blob, legacy_f32_blob_to_vector, pq8_blob_to_vector, - vector_to_blob, vector_to_legacy_f32_blob, vector_to_pq8_blob, PQ8_FORMAT_VERSION, - PQ8_HEADER_BYTES, PQ8_MAGIC_BYTE, + blob_to_vector, cosine_similarity, legacy_f32_blob_to_vector, vector_to_blob, vector_to_pq8_blob, PQ8_FORMAT_VERSION, PQ8_HEADER_BYTES, PQ8_MAGIC_BYTE, }; -pub use download::{ensure_model_downloaded, ensure_model_downloaded_in}; diff --git a/daemon-rs/src/embeddings/profiles.rs b/daemon-rs/src/embeddings/profiles.rs index 5f35ed72..dc5025c9 100644 --- a/daemon-rs/src/embeddings/profiles.rs +++ b/daemon-rs/src/embeddings/profiles.rs @@ -1,17 +1,13 @@ -// SPDX-License-Identifier: MIT use std::path::Path; - const MODEL_ENV_KEY: &str = "CORTEX_EMBEDDING_MODEL"; const POOL_ENV_KEY: &str = "CORTEX_EMBED_SESSION_POOL_SIZE"; pub(crate) const TEXT_TRUNCATE_BYTES: usize = 2000; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PoolingStrategy { Mean, Cls, LastToken, } - impl PoolingStrategy { fn as_str(self) -> &'static str { match self { @@ -21,19 +17,16 @@ impl PoolingStrategy { } } } - #[derive(Clone, Copy, Debug)] pub(crate) enum EmbeddingInputKind { Query, Passage, } - #[derive(Clone, Copy, Debug)] pub(crate) struct EmbeddingModelAsset { pub(crate) file: &'static str, pub(crate) url: &'static str, } - pub(crate) struct EmbeddingModelProfile { pub(crate) key: &'static str, pub(crate) display_name: &'static str, @@ -50,21 +43,13 @@ pub(crate) struct EmbeddingModelProfile { pub(crate) normalize: bool, pub(crate) include_token_type_ids: bool, } - impl EmbeddingModelProfile { fn primary_assets(&self) -> [EmbeddingModelAsset; 2] { [ - EmbeddingModelAsset { - file: self.model_file, - url: self.model_url, - }, - EmbeddingModelAsset { - file: self.tokenizer_file, - url: self.tokenizer_url, - }, + EmbeddingModelAsset { file: self.model_file, url: self.model_url }, + EmbeddingModelAsset { file: self.tokenizer_file, url: self.tokenizer_url }, ] } - pub(crate) fn missing_assets(&self, models_dir: &Path) -> Vec { let primary = self.primary_assets(); primary @@ -74,12 +59,10 @@ impl EmbeddingModelProfile { .filter(|asset| !models_dir.join(asset.file).exists()) .collect() } - pub(crate) fn assets_exist(&self, models_dir: &Path) -> bool { self.missing_assets(models_dir).is_empty() } } - const ALL_MINILM_L6_V2: EmbeddingModelProfile = EmbeddingModelProfile { key: "all-minilm-l6-v2", display_name: "all-MiniLM-L6-v2", @@ -87,10 +70,8 @@ const ALL_MINILM_L6_V2: EmbeddingModelProfile = EmbeddingModelProfile { max_input_tokens: 256, model_file: "all-MiniLM-L6-v2.onnx", tokenizer_file: "tokenizer.json", - model_url: - "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx", - tokenizer_url: - "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json", + model_url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx", + tokenizer_url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json", auxiliary_files: &[], query_prefix: "", passage_prefix: "", @@ -98,7 +79,6 @@ const ALL_MINILM_L6_V2: EmbeddingModelProfile = EmbeddingModelProfile { normalize: true, include_token_type_ids: true, }; - const ALL_MINILM_L12_V2: EmbeddingModelProfile = EmbeddingModelProfile { key: "all-minilm-l12-v2", display_name: "all-MiniLM-L12-v2", @@ -115,7 +95,6 @@ const ALL_MINILM_L12_V2: EmbeddingModelProfile = EmbeddingModelProfile { normalize: true, include_token_type_ids: true, }; - const BGE_BASE_EN_V1_5: EmbeddingModelProfile = EmbeddingModelProfile { key: "bge-base-en-v1.5", display_name: "bge-base-en-v1.5", @@ -132,7 +111,6 @@ const BGE_BASE_EN_V1_5: EmbeddingModelProfile = EmbeddingModelProfile { normalize: true, include_token_type_ids: true, }; - const QWEN3_EMBEDDING_0_6B: EmbeddingModelProfile = EmbeddingModelProfile { key: "qwen3-embedding-0.6b", display_name: "Qwen3-Embedding-0.6B", @@ -149,9 +127,7 @@ const QWEN3_EMBEDDING_0_6B: EmbeddingModelProfile = EmbeddingModelProfile { normalize: true, include_token_type_ids: false, }; - const DEFAULT_PROFILE: &EmbeddingModelProfile = &BGE_BASE_EN_V1_5; - #[derive(Clone, Copy, Debug)] pub struct EmbeddingModelSelection { pub key: &'static str, @@ -162,35 +138,24 @@ pub struct EmbeddingModelSelection { pub tokenizer_file: &'static str, pub pooling: &'static str, } - fn normalize_model_key(raw: &str) -> String { raw.trim().to_ascii_lowercase().replace('_', "-") } - pub(crate) fn resolve_profile() -> &'static EmbeddingModelProfile { match std::env::var(MODEL_ENV_KEY) { Ok(raw) => match normalize_model_key(&raw).as_str() { - "all-minilm-l6-v2" | "all-minilm-l6v2" | "minilm-l6" | "minilm-legacy" => { - &ALL_MINILM_L6_V2 - } - "all-minilm-l12-v2" | "all-minilm-l12v2" | "minilm-l12" | "minilm-modern" - | "minilm" => &ALL_MINILM_L12_V2, + "all-minilm-l6-v2" | "all-minilm-l6v2" | "minilm-l6" | "minilm-legacy" => &ALL_MINILM_L6_V2, + "all-minilm-l12-v2" | "all-minilm-l12v2" | "minilm-l12" | "minilm-modern" | "minilm" => &ALL_MINILM_L12_V2, "bge-base-en-v1.5" | "bge-base-en-v15" | "bge-base" | "bge" => &BGE_BASE_EN_V1_5, - "qwen3-embedding-0.6b" | "qwen3-embedding-06b" | "qwen3-embedding" | "qwen3" => { - &QWEN3_EMBEDDING_0_6B - } + "qwen3-embedding-0.6b" | "qwen3-embedding-06b" | "qwen3-embedding" | "qwen3" => &QWEN3_EMBEDDING_0_6B, unknown => { - eprintln!( - "[embeddings] Unknown {MODEL_ENV_KEY}='{unknown}', falling back to {}", - DEFAULT_PROFILE.key - ); + eprintln!("[embeddings] Unknown {MODEL_ENV_KEY}='{unknown}', falling back to {}", DEFAULT_PROFILE.key); DEFAULT_PROFILE } }, Err(_) => DEFAULT_PROFILE, } } - pub fn selected_model_selection() -> EmbeddingModelSelection { let profile = resolve_profile(); EmbeddingModelSelection { @@ -203,31 +168,20 @@ pub fn selected_model_selection() -> EmbeddingModelSelection { pooling: profile.pooling.as_str(), } } - pub fn selected_model_key() -> &'static str { selected_model_selection().key } - pub fn selected_model_assets_exist(models_dir: &Path) -> bool { resolve_profile().assets_exist(models_dir) } - -// --------------------------------------------------------------------------- -// Engine -// --------------------------------------------------------------------------- - const DEFAULT_POOL_SIZE: usize = 1; const MAX_POOL_SIZE: usize = 8; - pub(crate) fn resolved_pool_size() -> usize { match std::env::var(POOL_ENV_KEY) { Ok(raw) => match raw.trim().parse::() { Ok(parsed) => parsed.clamp(1, MAX_POOL_SIZE), Err(_) => { - eprintln!( - "[embeddings] Invalid {POOL_ENV_KEY}='{}'; using default {}", - raw, DEFAULT_POOL_SIZE - ); + eprintln!("[embeddings] Invalid {POOL_ENV_KEY}='{}'; using default {}", raw, DEFAULT_POOL_SIZE); DEFAULT_POOL_SIZE } }, diff --git a/daemon-rs/src/embeddings/tests/mod.rs b/daemon-rs/src/embeddings/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/embeddings/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/embeddings/vectors.rs b/daemon-rs/src/embeddings/vectors.rs index c2f4d4c8..f1665559 100644 --- a/daemon-rs/src/embeddings/vectors.rs +++ b/daemon-rs/src/embeddings/vectors.rs @@ -1,6 +1,3 @@ -// SPDX-License-Identifier: MIT -/// Cosine similarity between two f32 slices (assumed L2-normalised, but this -/// implementation handles the general case too). pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { if a.len() != b.len() || a.is_empty() { return 0.0; @@ -8,22 +5,18 @@ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { if a.iter().chain(b.iter()).any(|value| !value.is_finite()) { return 0.0; } - let mut dot = 0.0f32; let mut norm_a = 0.0f32; let mut norm_b = 0.0f32; - for i in 0..a.len() { dot += a[i] * b[i]; norm_a += a[i] * a[i]; norm_b += b[i] * b[i]; } - let denom = norm_a.sqrt() * norm_b.sqrt(); if denom == 0.0 || !denom.is_finite() { return 0.0; } - let similarity = dot / denom; if similarity.is_finite() { similarity.clamp(0.0, 1.0) @@ -31,104 +24,34 @@ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { 0.0 } } - -/// Encode a `Vec` as a SQLite BLOB. As of v0.6.0 this writes the -/// compact PQ8 format (~4x smaller than LE f32). Reads transparently -/// handle both formats via `blob_to_vector` — see `pq8_blob_to_vector` -/// and `legacy_f32_blob_to_vector` for format-specific entry points. pub fn vector_to_blob(vec: &[f32]) -> Vec { vector_to_pq8_blob(vec) } - -/// Strict legacy encoder: writes LE f32 packed bytes. Used by tests that -/// need to assert behaviour on legacy blobs, and by any one-off migration -/// tool that needs to produce the old wire format. #[allow(dead_code)] pub fn vector_to_legacy_f32_blob(vec: &[f32]) -> Vec { vec.iter().flat_map(|f| f.to_le_bytes()).collect() } - -/// Decode a SQLite BLOB back to `Vec`. Auto-detects PQ8 quantized -/// blobs vs legacy LE-f32 blobs so the read path transparently handles -/// the mixed corpus during the backfill window. Callers that specifically -/// need the legacy decoder can call `legacy_f32_blob_to_vector` directly. pub fn blob_to_vector(blob: &[u8]) -> Vec { if let Some(v) = pq8_blob_to_vector(blob) { return v; } legacy_f32_blob_to_vector(blob) } - -/// Strict legacy decoder: treat the blob as a packed LE-f32 array. Used by -/// tests and any caller that knows it is reading pre-PQ8 data. pub fn legacy_f32_blob_to_vector(blob: &[u8]) -> Vec { - blob.chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect() + blob.chunks_exact(4).map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect() } - -// --------------------------------------------------------------------------- -// PQ8 (per-vector symmetric int8) quantization -// -// Embedding vectors are the second-largest table in a mature Cortex DB. The -// f32 representation costs 4 * D bytes per row (3072 bytes for BGE-base). -// Symmetric int8 quantization with a single per-vector f32 scale collapses -// that to D + 5 bytes (773 bytes for BGE-base) — a ~4x reduction — while -// preserving cosine similarity to within a few hundredths in practice. -// -// BGE produces L2-normalised vectors so values land in [-1, 1] tightly, -// which means the quantization scale stays small and round-trip error is -// uniform. For non-normalised models the scale tracks the per-vector -// max(|v|) so the dynamic range of any single vector is fully used. -// -// Blob layout (PQ8_FORMAT_VERSION = 0x02): -// -// byte 0: magic = PQ8_MAGIC_BYTE (0xC8 — distinct from any -// byte that can appear at the head of an LE f32 storing -// a normalised value) -// byte 1: format version (0x02) -// bytes 2..6: scale (LE f32). Zero implies an all-zero vector. -// bytes 6..6+D: D signed int8 values, one per dimension -// -// Total: D + 6 bytes. For D=768 that is 774 bytes vs 3072 bytes of f32 — -// a 3.97x compression ratio. The 6-byte header amortises trivially. -// --------------------------------------------------------------------------- - -/// Magic byte that uniquely identifies a PQ8 blob. Chosen so it cannot -/// appear as the leading byte of an LE-encoded f32 holding a typical -/// normalised value: 0xC8 corresponds to LE float values around -1e22. pub const PQ8_MAGIC_BYTE: u8 = 0xC8; -/// Current PQ8 wire format version. Future formats bump this. pub const PQ8_FORMAT_VERSION: u8 = 0x02; -/// Header size in bytes: magic(1) + version(1) + scale(4). pub const PQ8_HEADER_BYTES: usize = 6; - -/// Quantize a Vec to a compact int8 blob. Returns the raw bytes ready -/// for SQLite storage. Lossless when the input is all-zero (scale = 0). pub fn vector_to_pq8_blob(vec: &[f32]) -> Vec { let mut out = Vec::with_capacity(PQ8_HEADER_BYTES + vec.len()); out.push(PQ8_MAGIC_BYTE); out.push(PQ8_FORMAT_VERSION); - - // Scale is the per-vector max absolute value mapped onto int8::MAX so - // every vector uses its full int8 dynamic range. NaN/inf inputs are - // treated as zero; we never want a poisoned scale to corrupt storage. - let max_abs = vec - .iter() - .copied() - .filter(|v| v.is_finite()) - .fold(0.0f32, |acc, v| acc.max(v.abs())); - - let scale = if max_abs > 0.0 { - max_abs / i8::MAX as f32 - } else { - 0.0 - }; + let max_abs = vec.iter().copied().filter(|v| v.is_finite()).fold(0.0f32, |acc, v| acc.max(v.abs())); + let scale = if max_abs > 0.0 { max_abs / i8::MAX as f32 } else { 0.0 }; out.extend_from_slice(&scale.to_le_bytes()); - for &v in vec { let q = if scale > 0.0 && v.is_finite() { - // round-half-to-even via f32::round, then clamp into int8. let scaled = (v / scale).round().clamp(i8::MIN as f32, i8::MAX as f32); scaled as i8 } else { @@ -138,15 +61,9 @@ pub fn vector_to_pq8_blob(vec: &[f32]) -> Vec { } out } - -/// True iff the blob is a PQ8-encoded vector (magic + version match). pub fn is_pq8_blob(blob: &[u8]) -> bool { blob.len() >= PQ8_HEADER_BYTES && blob[0] == PQ8_MAGIC_BYTE && blob[1] == PQ8_FORMAT_VERSION } - -/// Decode a PQ8 blob back to Vec. Returns None if the blob is not a -/// valid PQ8 payload — callers should fall back to `blob_to_vector` on -/// legacy LE-f32 storage in that case. pub fn pq8_blob_to_vector(blob: &[u8]) -> Option> { if !is_pq8_blob(blob) { return None; @@ -158,7 +75,6 @@ pub fn pq8_blob_to_vector(blob: &[u8]) -> Option> { let body = &blob[PQ8_HEADER_BYTES..]; let mut out = Vec::with_capacity(body.len()); if scale == 0.0 { - // All-zero vector. Preserve the original length. out.resize(body.len(), 0.0); return Some(out); } @@ -168,16 +84,7 @@ pub fn pq8_blob_to_vector(blob: &[u8]) -> Option> { } Some(out) } - -/// Convenience: max absolute element-wise error between two equal-length -/// f32 slices. Used by tests to bound quantization round-trip error. #[cfg(test)] pub(crate) fn max_abs_error(a: &[f32], b: &[f32]) -> f32 { - a.iter() - .zip(b.iter()) - .map(|(x, y)| (x - y).abs()) - .fold(0.0f32, f32::max) + a.iter().zip(b.iter()).map(|(x, y)| (x - y).abs()).fold(0.0f32, f32::max) } - -// --------------------------------------------------------------------------- -// Model management diff --git a/daemon-rs/src/eval.rs b/daemon-rs/src/eval.rs deleted file mode 100644 index 59229933..00000000 --- a/daemon-rs/src/eval.rs +++ /dev/null @@ -1,687 +0,0 @@ -// SPDX-License-Identifier: MIT -use chrono::Utc; -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; - -const RATE_GATED_METRICS: [(&str, bool); 6] = [ - ("taskSuccessRate", true), - ("firstPassSuccess", true), - ("contradictionRate", false), - ("staleMemoryHitRate", false), - ("lowTrustHitRate", false), - ("consensusPromotionPrecision", true), -]; - -#[derive(Default, Clone)] -struct TaskEvalAggregate { - total: i64, - success: i64, - first_pass_success: i64, - retries_total: i64, - latencies_valid_ms: Vec, -} - -impl TaskEvalAggregate { - fn observe(&mut self, outcome: &str, retries: Option, latency_ms: Option) { - self.total += 1; - let retries_value = retries.unwrap_or(0).max(0); - self.retries_total += retries_value; - - if outcome == "success" { - self.success += 1; - if retries_value == 0 { - self.first_pass_success += 1; - } - } - if matches!(outcome, "success" | "partial") { - if let Some(latency) = latency_ms { - self.latencies_valid_ms.push(latency.max(0)); - } - } - } - - fn task_success_rate(&self) -> f64 { - ratio(self.success, self.total) - } - - fn first_pass_success(&self) -> f64 { - ratio(self.first_pass_success, self.total) - } - - fn retry_count(&self) -> f64 { - ratio(self.retries_total, self.total) - } - - fn median_time_to_valid_result_ms(&self) -> f64 { - median_i64(&self.latencies_valid_ms).unwrap_or(0.0) - } - - fn as_json(&self) -> Value { - json!({ - "sampleCount": self.total, - "taskSuccessRate": self.task_success_rate(), - "firstPassSuccess": self.first_pass_success(), - "medianTimeToValidResultMs": self.median_time_to_valid_result_ms(), - "retryCount": self.retry_count() - }) - } -} - -fn is_baseline_task_class(task_class: &str) -> bool { - task_class - .trim() - .to_ascii_lowercase() - .starts_with("baseline") -} - -fn collect_task_metrics( - conn: &Connection, - since_modifier: &str, -) -> (TaskEvalAggregate, TaskEvalAggregate) { - let mut baseline = TaskEvalAggregate::default(); - let mut assisted = TaskEvalAggregate::default(); - let mut stmt = match conn.prepare( - "SELECT task_class, outcome, retries, latency_ms - FROM agent_feedback - WHERE created_at >= datetime('now', ?1)", - ) { - Ok(stmt) => stmt, - Err(_) => return (baseline, assisted), - }; - let rows = match stmt.query_map(params![since_modifier], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - )) - }) { - Ok(rows) => rows, - Err(_) => return (baseline, assisted), - }; - - for row in rows.flatten() { - let (task_class, outcome, retries, latency_ms) = row; - if is_baseline_task_class(&task_class) { - baseline.observe(&outcome, retries, latency_ms); - } else { - assisted.observe(&outcome, retries, latency_ms); - } - } - - (baseline, assisted) -} - -/// Build a local reliability/memory-quality snapshot over the requested horizon. -pub fn build_eval_snapshot(conn: &Connection, horizon_days: i64) -> Value { - let horizon_days = horizon_days.clamp(1, 180); - let since_modifier = format!("-{horizon_days} days"); - - let open_conflicts: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'disputed' AND disputes_id IS NOT NULL", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let active_memories: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active'", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let active_decisions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'active'", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let decayed_memories: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active' AND score < 0.5 AND pinned = 0", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let decayed_decisions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'active' AND score < 0.5 AND pinned = 0", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let recent_conflicts: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'decision_conflict' AND created_at >= datetime('now', ?1)", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let recent_resolutions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'decision_resolve' AND created_at >= datetime('now', ?1)", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let recent_recalls: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'recall_query' AND created_at >= datetime('now', ?1)", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - - let recent_memory_hits: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1)", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let stale_memory_hits: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1) - AND (score < 0.5 OR (expires_at IS NOT NULL AND expires_at <= datetime('now')))", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let recent_total_hits: i64 = conn - .query_row( - "SELECT - (SELECT COUNT(*) FROM memories - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1)) - + (SELECT COUNT(*) FROM decisions - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1))", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let recent_low_trust_hits: i64 = conn - .query_row( - "SELECT - (SELECT COUNT(*) FROM memories - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1) - AND trust_score < 0.5) - + (SELECT COUNT(*) FROM decisions - WHERE status = 'active' - AND retrievals > 0 - AND last_accessed IS NOT NULL - AND last_accessed >= datetime('now', ?1) - AND trust_score < 0.5)", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - - let promoted_consensus: i64 = conn - .query_row( - "SELECT COALESCE(SUM(CAST(json_extract(data, '$.promoted') AS INTEGER)), 0) - FROM events - WHERE type = 'consensus' - AND created_at >= datetime('now', ?1) - AND json_extract(data, '$.action') = 'promoted'", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - let failed_consensus: i64 = conn - .query_row( - "SELECT COALESCE(SUM(CAST(json_extract(data, '$.failed') AS INTEGER)), 0) - FROM events - WHERE type = 'consensus' - AND created_at >= datetime('now', ?1) - AND json_extract(data, '$.action') = 'promoted'", - params![since_modifier.as_str()], - |row| row.get(0), - ) - .unwrap_or(0); - - let (baseline_tasks, assisted_tasks) = collect_task_metrics(conn, since_modifier.as_str()); - let baseline_json = baseline_tasks.as_json(); - let assisted_json = assisted_tasks.as_json(); - - let total_active = active_memories + active_decisions; - let conflict_burden = ratio(open_conflicts, active_decisions); - let decay_burden = ratio(decayed_memories + decayed_decisions, total_active); - let resolution_velocity = recent_resolutions as f64 / horizon_days as f64; - let contradiction_rate = ratio(recent_conflicts, recent_recalls); - let stale_memory_hit_rate = ratio(stale_memory_hits, recent_memory_hits); - let low_trust_hit_rate = ratio(recent_low_trust_hits, recent_total_hits); - let consensus_promotion_precision = - ratio(promoted_consensus, promoted_consensus + failed_consensus); - - let success_rate_delta = diff_signal( - assisted_json.get("taskSuccessRate").and_then(Value::as_f64), - baseline_json.get("taskSuccessRate").and_then(Value::as_f64), - ); - let first_pass_delta = diff_signal( - assisted_json - .get("firstPassSuccess") - .and_then(Value::as_f64), - baseline_json - .get("firstPassSuccess") - .and_then(Value::as_f64), - ); - let median_latency_delta_ms = diff_signal( - assisted_json - .get("medianTimeToValidResultMs") - .and_then(Value::as_f64), - baseline_json - .get("medianTimeToValidResultMs") - .and_then(Value::as_f64), - ); - let retry_delta = diff_signal( - assisted_json.get("retryCount").and_then(Value::as_f64), - baseline_json.get("retryCount").and_then(Value::as_f64), - ); - - json!({ - "ok": true, - "windowDays": horizon_days, - "snapshotAt": Utc::now().to_rfc3339(), - "totals": { - "activeMemories": active_memories, - "activeDecisions": active_decisions, - "openConflicts": open_conflicts - }, - "window": { - "recentConflicts": recent_conflicts, - "recentResolutions": recent_resolutions, - "recentRecallQueries": recent_recalls, - "recentMemoryHits": recent_memory_hits, - "recentTotalHits": recent_total_hits, - "recentLowTrustHits": recent_low_trust_hits, - "recentConsensusPromotions": promoted_consensus, - "recentConsensusPromotionFailures": failed_consensus - }, - "taskMetrics": { - "baseline": baseline_json, - "assisted": assisted_json, - "delta": { - "taskSuccessRate": success_rate_delta, - "firstPassSuccess": first_pass_delta, - "medianTimeToValidResultMs": median_latency_delta_ms, - "retryCount": retry_delta - } - }, - "signals": { - "conflictBurden": conflict_burden, - "decayBurden": decay_burden, - "resolutionVelocity": resolution_velocity, - "contradictionRate": contradiction_rate, - "taskSuccessRate": assisted_tasks.task_success_rate(), - "firstPassSuccess": assisted_tasks.first_pass_success(), - "medianTimeToValidResultMs": assisted_tasks.median_time_to_valid_result_ms(), - "retryCount": assisted_tasks.retry_count(), - "staleMemoryHitRate": stale_memory_hit_rate, - "lowTrustHitRate": low_trust_hit_rate, - "consensusPromotionPrecision": consensus_promotion_precision - } - }) -} - -/// Compare two eval snapshots and report whether current metrics stay within the -/// allowed regression envelope. -pub fn build_eval_regression_gate(current: &Value, baseline: &Value, max_regression: f64) -> Value { - let max_regression = max_regression.clamp(0.0, 1.0); - let mut checks = Vec::new(); - let mut failed = Vec::new(); - - for (metric, higher_is_better) in RATE_GATED_METRICS { - let current_value = current - .get("signals") - .and_then(|signals| signals.get(metric)) - .and_then(Value::as_f64); - let baseline_value = baseline - .get("signals") - .and_then(|signals| signals.get(metric)) - .and_then(Value::as_f64); - let status = evaluate_regression( - metric, - higher_is_better, - current_value, - baseline_value, - max_regression, - ); - if status.get("regressed").and_then(Value::as_bool) == Some(true) { - failed.push(status.clone()); - } - checks.push(status); - } - - json!({ - "ok": failed.is_empty(), - "maxRegression": max_regression, - "checkedMetrics": checks, - "failedMetrics": failed - }) -} - -fn evaluate_regression( - metric: &str, - higher_is_better: bool, - current_value: Option, - baseline_value: Option, - max_regression: f64, -) -> Value { - let (Some(current), Some(baseline)) = (current_value, baseline_value) else { - return json!({ - "metric": metric, - "direction": if higher_is_better { "higher_is_better" } else { "lower_is_better" }, - "status": "skipped_missing_value", - "current": current_value, - "baseline": baseline_value, - "regressed": false - }); - }; - - let raw_delta = current - baseline; - let relative_delta = if baseline.abs() > f64::EPSILON { - raw_delta / baseline.abs() - } else { - raw_delta - }; - let regressed = if higher_is_better { - -relative_delta > max_regression - } else { - relative_delta > max_regression - }; - - json!({ - "metric": metric, - "direction": if higher_is_better { "higher_is_better" } else { "lower_is_better" }, - "status": if regressed { "regressed" } else { "ok" }, - "current": current, - "baseline": baseline, - "delta": raw_delta, - "relativeDelta": relative_delta, - "regressed": regressed - }) -} - -fn diff_signal(current: Option, baseline: Option) -> Value { - match (current, baseline) { - (Some(current), Some(baseline)) => json!(current - baseline), - _ => Value::Null, - } -} - -fn median_i64(values: &[i64]) -> Option { - if values.is_empty() { - return None; - } - let mut sorted = values.to_vec(); - sorted.sort_unstable(); - let mid = sorted.len() / 2; - if sorted.len().is_multiple_of(2) { - Some((sorted[mid - 1] as f64 + sorted[mid] as f64) / 2.0) - } else { - Some(sorted[mid] as f64) - } -} - -fn ratio(numerator: i64, denominator: i64) -> f64 { - if denominator <= 0 { - 0.0 - } else { - numerator as f64 / denominator as f64 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn eval_snapshot_computes_expected_signals() { - let conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - - conn.execute( - "INSERT INTO memories - (text, source, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) - VALUES ('m1', 'tests::eval', 'active', 0.2, 0.8, 3, datetime('now'), 0, datetime('now'), datetime('now'))", - [], - ) - .expect("insert memory m1"); - conn.execute( - "INSERT INTO memories - (text, source, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) - VALUES ('m2', 'tests::eval', 'active', 0.9, 0.4, 1, datetime('now'), 0, datetime('now'), datetime('now'))", - [], - ) - .expect("insert memory m2"); - conn.execute( - "INSERT INTO decisions - (decision, context, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) - VALUES ('d1', 'ctx', 'active', 0.3, 0.3, 1, datetime('now'), 0, datetime('now'), datetime('now'))", - [], - ) - .expect("insert decision d1"); - conn.execute( - "INSERT INTO decisions - (decision, context, status, score, pinned, disputes_id, created_at, updated_at) - VALUES ('d2', 'ctx', 'disputed', 0.9, 0, 1, datetime('now'), datetime('now'))", - [], - ) - .expect("insert disputed decision"); - - conn.execute( - "INSERT INTO agent_feedback - (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) - VALUES (0, 'codex', 'baseline:debug', 'success', 0.8, 0.8, 500, 1, 1200, datetime('now'))", - [], - ) - .expect("insert baseline success"); - conn.execute( - "INSERT INTO agent_feedback - (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) - VALUES (0, 'codex', 'baseline:debug', 'failure', 0.2, 0.2, 700, 2, 1300, datetime('now'))", - [], - ) - .expect("insert baseline failure"); - conn.execute( - "INSERT INTO agent_feedback - (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) - VALUES (0, 'codex', 'debug', 'success', 0.9, 0.9, 300, 0, 1000, datetime('now'))", - [], - ) - .expect("insert assisted success"); - conn.execute( - "INSERT INTO agent_feedback - (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) - VALUES (0, 'codex', 'debug', 'partial', 0.7, 0.7, 400, 1, 1100, datetime('now'))", - [], - ) - .expect("insert assisted partial"); - - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('decision_conflict', '{}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert conflict event"); - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('decision_resolve', '{}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert resolve event"); - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('recall_query', '{}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert recall event"); - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('recall_query', '{}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert second recall event"); - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('consensus', '{\"action\":\"promoted\",\"promoted\":2,\"failed\":1}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert consensus event"); - conn.execute( - "INSERT INTO events (type, data, source_agent, created_at) - VALUES ('consensus', '{\"action\":\"promoted\",\"promoted\":1,\"failed\":0}', 'tests::eval', datetime('now'))", - [], - ) - .expect("insert second consensus event"); - - let snapshot = build_eval_snapshot(&conn, 30); - let totals = snapshot.get("totals").expect("totals"); - let window = snapshot.get("window").expect("window"); - let signals = snapshot.get("signals").expect("signals"); - let tasks = snapshot.get("taskMetrics").expect("task metrics"); - - assert_eq!( - totals.get("activeMemories").and_then(Value::as_i64), - Some(2) - ); - assert_eq!( - totals.get("activeDecisions").and_then(Value::as_i64), - Some(1) - ); - assert_eq!(totals.get("openConflicts").and_then(Value::as_i64), Some(1)); - assert_eq!( - window.get("recentConflicts").and_then(Value::as_i64), - Some(1) - ); - assert_eq!( - window.get("recentResolutions").and_then(Value::as_i64), - Some(1) - ); - assert_eq!( - window.get("recentRecallQueries").and_then(Value::as_i64), - Some(2) - ); - assert_eq!( - signals.get("conflictBurden").and_then(Value::as_f64), - Some(1.0) - ); - let decay_burden = signals - .get("decayBurden") - .and_then(Value::as_f64) - .expect("decay burden"); - assert!( - (decay_burden - (2.0 / 3.0)).abs() < 0.0001, - "expected 2/3 decay burden, got {decay_burden}" - ); - assert_eq!( - signals.get("contradictionRate").and_then(Value::as_f64), - Some(0.5) - ); - assert_eq!( - signals.get("taskSuccessRate").and_then(Value::as_f64), - Some(0.5) - ); - assert_eq!( - signals.get("firstPassSuccess").and_then(Value::as_f64), - Some(0.5) - ); - assert_eq!( - signals - .get("medianTimeToValidResultMs") - .and_then(Value::as_f64), - Some(350.0) - ); - assert_eq!(signals.get("retryCount").and_then(Value::as_f64), Some(0.5)); - let stale_memory_hit_rate = signals - .get("staleMemoryHitRate") - .and_then(Value::as_f64) - .expect("stale memory hit rate"); - assert!( - (stale_memory_hit_rate - 0.5).abs() < 0.0001, - "expected stale memory hit rate 0.5, got {stale_memory_hit_rate}" - ); - let low_trust_hit_rate = signals - .get("lowTrustHitRate") - .and_then(Value::as_f64) - .expect("low trust hit rate"); - assert!( - (low_trust_hit_rate - (2.0 / 3.0)).abs() < 0.0001, - "expected low trust hit rate 2/3, got {low_trust_hit_rate}" - ); - let consensus_precision = signals - .get("consensusPromotionPrecision") - .and_then(Value::as_f64) - .expect("consensus precision"); - assert!( - (consensus_precision - 0.75).abs() < 0.0001, - "expected consensus precision 0.75, got {consensus_precision}" - ); - assert_eq!( - tasks["assisted"]["sampleCount"].as_i64(), - Some(2), - "assisted task sample count" - ); - assert_eq!( - tasks["baseline"]["sampleCount"].as_i64(), - Some(2), - "baseline task sample count" - ); - } - - #[test] - fn eval_regression_gate_flags_rate_regressions() { - let baseline = json!({ - "signals": { - "taskSuccessRate": 0.8, - "firstPassSuccess": 0.7, - "contradictionRate": 0.10, - "staleMemoryHitRate": 0.10, - "lowTrustHitRate": 0.20, - "consensusPromotionPrecision": 0.9 - } - }); - let current = json!({ - "signals": { - "taskSuccessRate": 0.5, - "firstPassSuccess": 0.65, - "contradictionRate": 0.14, - "staleMemoryHitRate": 0.08, - "lowTrustHitRate": 0.18, - "consensusPromotionPrecision": 0.88 - } - }); - - let gate = build_eval_regression_gate(¤t, &baseline, 0.20); - assert_eq!(gate["ok"].as_bool(), Some(false)); - let failed = gate["failedMetrics"] - .as_array() - .expect("failed metrics list should be present"); - assert!( - failed - .iter() - .any(|entry| entry.get("metric").and_then(Value::as_str) == Some("taskSuccessRate")), - "taskSuccessRate regression should be reported" - ); - } -} diff --git a/daemon-rs/src/eval/mod.rs b/daemon-rs/src/eval/mod.rs new file mode 100644 index 00000000..cfd737b9 --- /dev/null +++ b/daemon-rs/src/eval/mod.rs @@ -0,0 +1,286 @@ +use chrono::Utc; +use rusqlite::{params, Connection}; +use serde_json::{json, Value}; +const RATE_GATED_METRICS: [(&str, bool); 6] = [ + ("taskSuccessRate", true), + ("firstPassSuccess", true), + ("contradictionRate", false), + ("staleMemoryHitRate", false), + ("lowTrustHitRate", false), + ("consensusPromotionPrecision", true), +]; +#[derive(Default, Clone)] +struct TaskEvalAggregate { + total: i64, + success: i64, + first_pass_success: i64, + retries_total: i64, + latencies_valid_ms: Vec, +} +impl TaskEvalAggregate { + fn observe(&mut self, outcome: &str, retries: Option, latency_ms: Option) { + self.total += 1; + let retries_value = retries.unwrap_or(0).max(0); + self.retries_total += retries_value; + if outcome == "success" { + self.success += 1; + if retries_value == 0 { + self.first_pass_success += 1; + } + } + if matches!(outcome, "success" | "partial") { + if let Some(latency) = latency_ms { + self.latencies_valid_ms.push(latency.max(0)); + } + } + } + fn task_success_rate(&self) -> f64 { + ratio(self.success, self.total) + } + fn first_pass_success(&self) -> f64 { + ratio(self.first_pass_success, self.total) + } + fn retry_count(&self) -> f64 { + ratio(self.retries_total, self.total) + } + fn median_time_to_valid_result_ms(&self) -> f64 { + median_i64(&self.latencies_valid_ms).unwrap_or(0.0) + } + fn as_json(&self) -> Value { + json!({"sampleCount":self.total,"taskSuccessRate":self.task_success_rate(),"firstPassSuccess":self. +first_pass_success(),"medianTimeToValidResultMs":self.median_time_to_valid_result_ms(),"retryCount":self.retry_count()}) + } +} +fn is_baseline_task_class(task_class: &str) -> bool { + task_class.trim().to_ascii_lowercase().starts_with("baseline") +} +fn collect_task_metrics(conn: &Connection, since_modifier: &str) -> (TaskEvalAggregate, TaskEvalAggregate) { + let mut baseline = TaskEvalAggregate::default(); + let mut assisted = TaskEvalAggregate::default(); + let mut stmt = match conn.prepare( + "SELECT task_class, outcome, retries, latency_ms + FROM agent_feedback + WHERE created_at >= datetime('now', ?1)", + ) { + Ok(stmt) => stmt, + Err(_) => return (baseline, assisted), + }; + let rows = match stmt.query_map(params![since_modifier], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?, row.get::<_, Option>(3)?)) + }) { + Ok(rows) => rows, + Err(_) => return (baseline, assisted), + }; + for row in rows.flatten() { + let (task_class, outcome, retries, latency_ms) = row; + if is_baseline_task_class(&task_class) { + baseline.observe(&outcome, retries, latency_ms); + } else { + assisted.observe(&outcome, retries, latency_ms); + } + } + (baseline, assisted) +} +pub fn build_eval_snapshot(conn: &Connection, horizon_days: i64) -> Value { + let horizon_days = horizon_days.clamp(1, 180); + let since_modifier = format!("-{horizon_days} days"); + let open_conflicts: i64 = conn + .query_row("SELECT COUNT(*) FROM decisions WHERE status = 'disputed' AND disputes_id IS NOT NULL", [], |row| row.get(0)) + .unwrap_or(0); + let active_memories: i64 = conn.query_row("SELECT COUNT(*) FROM memories WHERE status = 'active'", [], |row| row.get(0)).unwrap_or(0); + let active_decisions: i64 = conn.query_row("SELECT COUNT(*) FROM decisions WHERE status = 'active'", [], |row| row.get(0)).unwrap_or(0); + let decayed_memories: i64 = conn + .query_row("SELECT COUNT(*) FROM memories WHERE status = 'active' AND score < 0.5 AND pinned = 0", [], |row| row.get(0)) + .unwrap_or(0); + let decayed_decisions: i64 = conn + .query_row("SELECT COUNT(*) FROM decisions WHERE status = 'active' AND score < 0.5 AND pinned = 0", [], |row| row.get(0)) + .unwrap_or(0); + let recent_conflicts: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events WHERE type = 'decision_conflict' AND created_at >= datetime('now', ?1)", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let recent_resolutions: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events WHERE type = 'decision_resolve' AND created_at >= datetime('now', ?1)", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let recent_recalls: i64 = conn + .query_row("SELECT COUNT(*) FROM events WHERE type = 'recall_query' AND created_at >= datetime('now', ?1)", params![since_modifier.as_str()], |row| { + row.get(0) + }) + .unwrap_or(0); + let recent_memory_hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM memories + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1)", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let stale_memory_hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM memories + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1) + AND (score < 0.5 OR (expires_at IS NOT NULL AND expires_at <= datetime('now')))", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let recent_total_hits: i64 = conn + .query_row( + "SELECT + (SELECT COUNT(*) FROM memories + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1)) + + (SELECT COUNT(*) FROM decisions + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1))", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let recent_low_trust_hits: i64 = conn + .query_row( + "SELECT + (SELECT COUNT(*) FROM memories + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1) + AND trust_score < 0.5) + + (SELECT COUNT(*) FROM decisions + WHERE status = 'active' + AND retrievals > 0 + AND last_accessed IS NOT NULL + AND last_accessed >= datetime('now', ?1) + AND trust_score < 0.5)", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let promoted_consensus: i64 = conn + .query_row( + "SELECT COALESCE(SUM(CAST(json_extract(data, '$.promoted') AS INTEGER)), 0) + FROM events + WHERE type = 'consensus' + AND created_at >= datetime('now', ?1) + AND json_extract(data, '$.action') = 'promoted'", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let failed_consensus: i64 = conn + .query_row( + "SELECT COALESCE(SUM(CAST(json_extract(data, '$.failed') AS INTEGER)), 0) + FROM events + WHERE type = 'consensus' + AND created_at >= datetime('now', ?1) + AND json_extract(data, '$.action') = 'promoted'", + params![since_modifier.as_str()], + |row| row.get(0), + ) + .unwrap_or(0); + let (baseline_tasks, assisted_tasks) = collect_task_metrics(conn, since_modifier.as_str()); + let baseline_json = baseline_tasks.as_json(); + let assisted_json = assisted_tasks.as_json(); + let total_active = active_memories + active_decisions; + let conflict_burden = ratio(open_conflicts, active_decisions); + let decay_burden = ratio(decayed_memories + decayed_decisions, total_active); + let resolution_velocity = recent_resolutions as f64 / horizon_days as f64; + let contradiction_rate = ratio(recent_conflicts, recent_recalls); + let stale_memory_hit_rate = ratio(stale_memory_hits, recent_memory_hits); + let low_trust_hit_rate = ratio(recent_low_trust_hits, recent_total_hits); + let consensus_promotion_precision = ratio(promoted_consensus, promoted_consensus + failed_consensus); + let success_rate_delta = + diff_signal(assisted_json.get("taskSuccessRate").and_then(Value::as_f64), baseline_json.get("taskSuccessRate").and_then(Value::as_f64)); + let first_pass_delta = + diff_signal(assisted_json.get("firstPassSuccess").and_then(Value::as_f64), baseline_json.get("firstPassSuccess").and_then(Value::as_f64)); + let median_latency_delta_ms = diff_signal( + assisted_json.get("medianTimeToValidResultMs").and_then(Value::as_f64), + baseline_json.get("medianTimeToValidResultMs").and_then(Value::as_f64), + ); + let retry_delta = diff_signal(assisted_json.get("retryCount").and_then(Value::as_f64), baseline_json.get("retryCount").and_then(Value::as_f64)); + json!({"ok":true,"windowDays":horizon_days,"snapshotAt":Utc:: +now().to_rfc3339(),"totals":{"activeMemories":active_memories,"activeDecisions":active_decisions,"openConflicts":open_conflicts}, +"window":{"recentConflicts":recent_conflicts,"recentResolutions":recent_resolutions,"recentRecallQueries":recent_recalls, +"recentMemoryHits":recent_memory_hits,"recentTotalHits":recent_total_hits,"recentLowTrustHits":recent_low_trust_hits, +"recentConsensusPromotions":promoted_consensus,"recentConsensusPromotionFailures":failed_consensus},"taskMetrics":{"baseline": +baseline_json,"assisted":assisted_json,"delta":{"taskSuccessRate":success_rate_delta,"firstPassSuccess":first_pass_delta, +"medianTimeToValidResultMs":median_latency_delta_ms,"retryCount":retry_delta}},"signals":{"conflictBurden":conflict_burden, +"decayBurden":decay_burden,"resolutionVelocity":resolution_velocity,"contradictionRate":contradiction_rate,"taskSuccessRate": +assisted_tasks.task_success_rate(),"firstPassSuccess":assisted_tasks.first_pass_success(),"medianTimeToValidResultMs": +assisted_tasks.median_time_to_valid_result_ms(),"retryCount":assisted_tasks.retry_count(),"staleMemoryHitRate": +stale_memory_hit_rate,"lowTrustHitRate":low_trust_hit_rate,"consensusPromotionPrecision":consensus_promotion_precision}}) +} +pub fn build_eval_regression_gate(current: &Value, baseline: &Value, max_regression: f64) -> Value { + let max_regression = max_regression.clamp(0.0, 1.0); + let mut checks = Vec::new(); + let mut failed = Vec::new(); + for (metric, higher_is_better) in RATE_GATED_METRICS { + let current_value = current.get("signals").and_then(|signals| signals.get(metric)).and_then(Value::as_f64); + let baseline_value = baseline.get("signals").and_then(|signals| signals.get(metric)).and_then(Value::as_f64); + let status = evaluate_regression(metric, higher_is_better, current_value, baseline_value, max_regression); + if status.get("regressed").and_then(Value::as_bool) == Some(true) { + failed.push(status.clone()); + } + checks.push(status); + } + json!({"ok":failed.is_empty(),"maxRegression":max_regression,"checkedMetrics":checks, +"failedMetrics":failed}) +} +fn evaluate_regression(metric: &str, higher_is_better: bool, current_value: Option, baseline_value: Option, max_regression: f64) -> Value { + let (Some(current), Some(baseline)) = (current_value, baseline_value) else { + return json!({"metric":metric +,"direction":if higher_is_better{"higher_is_better"}else{"lower_is_better"},"status":"skipped_missing_value","current": +current_value,"baseline":baseline_value,"regressed":false}); + }; + let raw_delta = current - baseline; + let relative_delta = if baseline.abs() > f64::EPSILON { raw_delta / baseline.abs() } else { raw_delta }; + let regressed = if higher_is_better { -relative_delta > max_regression } else { relative_delta > max_regression }; + json!({"metric":metric,"direction":if higher_is_better{"higher_is_better"}else{"lower_is_better"}, +"status":if regressed{"regressed"}else{"ok"},"current":current,"baseline":baseline,"delta":raw_delta,"relativeDelta": +relative_delta,"regressed":regressed}) +} +fn diff_signal(current: Option, baseline: Option) -> Value { + match (current, baseline) { + (Some(current), Some(baseline)) => json!(current - baseline), + _ => Value::Null, + } +} +fn median_i64(values: &[i64]) -> Option { + if values.is_empty() { + return None; + } + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + let mid = sorted.len() / 2; + if sorted.len().is_multiple_of(2) { + Some((sorted[mid - 1] as f64 + sorted[mid] as f64) / 2.0) + } else { + Some(sorted[mid] as f64) + } +} +fn ratio(numerator: i64, denominator: i64) -> f64 { + if denominator <= 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 + } +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/eval/tests/mod.rs b/daemon-rs/src/eval/tests/mod.rs new file mode 100644 index 00000000..64bfe647 --- /dev/null +++ b/daemon-rs/src/eval/tests/mod.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +#[test] +fn eval_snapshot_computes_expected_signals() { + let conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + conn.execute( + "INSERT INTO memories + (text, source, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) + VALUES ('m1', 'tests::eval', 'active', 0.2, 0.8, 3, datetime('now'), 0, datetime('now'), datetime('now'))", + [], + ) + .expect("insert memory m1"); + conn.execute( + "INSERT INTO memories + (text, source, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) + VALUES ('m2', 'tests::eval', 'active', 0.9, 0.4, 1, datetime('now'), 0, datetime('now'), datetime('now'))", + [], + ) + .expect("insert memory m2"); + conn.execute( + "INSERT INTO decisions + (decision, context, status, score, trust_score, retrievals, last_accessed, pinned, created_at, updated_at) + VALUES ('d1', 'ctx', 'active', 0.3, 0.3, 1, datetime('now'), 0, datetime('now'), datetime('now'))", + [], + ) + .expect("insert decision d1"); + conn.execute( + "INSERT INTO decisions + (decision, context, status, score, pinned, disputes_id, created_at, updated_at) + VALUES ('d2', 'ctx', 'disputed', 0.9, 0, 1, datetime('now'), datetime('now'))", + [], + ) + .expect("insert disputed decision"); + conn.execute( + "INSERT INTO agent_feedback + (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) + VALUES (0, 'codex', 'baseline:debug', 'success', 0.8, 0.8, 500, 1, 1200, datetime('now'))", + [], + ) + .expect("insert baseline success"); + conn.execute( + "INSERT INTO agent_feedback + (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) + VALUES (0, 'codex', 'baseline:debug', 'failure', 0.2, 0.2, 700, 2, 1300, datetime('now'))", + [], + ) + .expect("insert baseline failure"); + conn.execute( + "INSERT INTO agent_feedback + (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) + VALUES (0, 'codex', 'debug', 'success', 0.9, 0.9, 300, 0, 1000, datetime('now'))", + [], + ) + .expect("insert assisted success"); + conn.execute( + "INSERT INTO agent_feedback + (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, created_at) + VALUES (0, 'codex', 'debug', 'partial', 0.7, 0.7, 400, 1, 1100, datetime('now'))", + [], + ) + .expect("insert assisted partial"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('decision_conflict', '{}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert conflict event"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('decision_resolve', '{}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert resolve event"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('recall_query', '{}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert recall event"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('recall_query', '{}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert second recall event"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('consensus', '{\"action\":\"promoted\",\"promoted\":2,\"failed\":1}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert consensus event"); + conn.execute( + "INSERT INTO events (type, data, source_agent, created_at) + VALUES ('consensus', '{\"action\":\"promoted\",\"promoted\":1,\"failed\":0}', 'tests::eval', datetime('now'))", + [], + ) + .expect("insert second consensus event"); + let snapshot = build_eval_snapshot(&conn, 30); + let totals = snapshot.get("totals").expect("totals"); + let window = snapshot.get("window").expect("window"); + let signals = snapshot.get("signals").expect("signals"); + let tasks = snapshot.get("taskMetrics").expect("task metrics"); + assert_eq!(totals.get("activeMemories").and_then(Value::as_i64), Some(2)); + assert_eq!(totals.get("activeDecisions").and_then(Value::as_i64), Some(1)); + assert_eq!(totals.get("openConflicts").and_then(Value::as_i64), Some(1)); + assert_eq!(window.get("recentConflicts").and_then(Value::as_i64), Some(1)); + assert_eq!(window.get("recentResolutions").and_then(Value::as_i64), Some(1)); + assert_eq!(window.get("recentRecallQueries").and_then(Value::as_i64), Some(2)); + assert_eq!(signals.get("conflictBurden").and_then(Value::as_f64), Some(1.0)); + let decay_burden = signals.get("decayBurden").and_then(Value::as_f64).expect("decay burden"); + assert!((decay_burden - (2.0 / 3.0)).abs() < 0.0001, "expected 2/3 decay burden, got {decay_burden}"); + assert_eq!(signals.get("contradictionRate").and_then(Value::as_f64), Some(0.5)); + assert_eq!(signals.get("taskSuccessRate").and_then(Value::as_f64), Some(0.5)); + assert_eq!(signals.get("firstPassSuccess").and_then(Value::as_f64), Some(0.5)); + assert_eq!(signals.get("medianTimeToValidResultMs").and_then(Value::as_f64), Some(350.0)); + assert_eq!(signals.get("retryCount").and_then(Value::as_f64), Some(0.5)); + let stale_memory_hit_rate = signals.get("staleMemoryHitRate").and_then(Value::as_f64).expect("stale memory hit rate"); + assert!((stale_memory_hit_rate - 0.5).abs() < 0.0001, "expected stale memory hit rate 0.5, got {stale_memory_hit_rate}"); + let low_trust_hit_rate = signals.get("lowTrustHitRate").and_then(Value::as_f64).expect("low trust hit rate"); + assert!((low_trust_hit_rate - (2.0 / 3.0)).abs() < 0.0001, "expected low trust hit rate 2/3, got {low_trust_hit_rate}"); + let consensus_precision = signals.get("consensusPromotionPrecision").and_then(Value::as_f64).expect("consensus precision"); + assert!((consensus_precision - 0.75).abs() < 0.0001, "expected consensus precision 0.75, got {consensus_precision}"); + assert_eq!(tasks["assisted"]["sampleCount"].as_i64(), Some(2), "assisted task sample count"); + assert_eq!(tasks["baseline"]["sampleCount"].as_i64(), Some(2), "baseline task sample count"); +} +#[test] +fn eval_regression_gate_flags_rate_regressions() { + let baseline = json!({ + "signals": { + "taskSuccessRate": 0.8, + "firstPassSuccess": 0.7, + "contradictionRate": 0.10, + "staleMemoryHitRate": 0.10, + "lowTrustHitRate": 0.20, + "consensusPromotionPrecision": 0.9 + } + }); + let current = json!({ + "signals": { + "taskSuccessRate": 0.5, + "firstPassSuccess": 0.65, + "contradictionRate": 0.14, + "staleMemoryHitRate": 0.08, + "lowTrustHitRate": 0.18, + "consensusPromotionPrecision": 0.88 + } + }); + let gate = build_eval_regression_gate(¤t, &baseline, 0.20); + assert_eq!(gate["ok"].as_bool(), Some(false)); + let failed = gate["failedMetrics"].as_array().expect("failed metrics list should be present"); + assert!(failed.iter().any(|entry| entry.get("metric").and_then(Value::as_str) == Some("taskSuccessRate")), "taskSuccessRate regression should be reported"); +} diff --git a/daemon-rs/src/export_data.rs b/daemon-rs/src/export_data.rs deleted file mode 100644 index ae2b5cf6..00000000 --- a/daemon-rs/src/export_data.rs +++ /dev/null @@ -1,943 +0,0 @@ -// SPDX-License-Identifier: MIT -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; - -pub use crate::api_types::{ExportFormat, ImportCounts, ImportOptions, ImportPayload}; - -pub const DEFAULT_EXPORT_PAGE_LIMIT: usize = 1000; -pub const MAX_EXPORT_PAGE_LIMIT: usize = 5000; - -fn normalize_memory_entry_type(raw: Option<&str>) -> String { - let normalized = raw - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_ascii_lowercase) - .unwrap_or_else(|| "fact".to_string()); - match normalized.as_str() { - "memory" | "note" | "finding" | "observation" | "fact" => "fact".to_string(), - "episode" | "event" => "episode".to_string(), - "procedure" | "playbook" | "runbook" | "howto" | "how-to" => "procedure".to_string(), - "evidence" | "citation" | "reference" => "evidence".to_string(), - "decision" | "policy" | "rule" => "decision".to_string(), - other => other.to_string(), - } -} - -fn normalize_decision_entry_type(raw: Option<&str>) -> String { - let normalized = raw - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_ascii_lowercase) - .unwrap_or_else(|| "decision".to_string()); - match normalized.as_str() { - "decision" | "policy" | "rule" => "decision".to_string(), - "procedure" | "playbook" | "runbook" => "procedure".to_string(), - "evidence" | "citation" | "reference" => "evidence".to_string(), - "fact" | "memory" | "note" => "fact".to_string(), - other => other.to_string(), - } -} - -pub fn export_json_value(conn: &Connection) -> Value { - let memories = query_table_json( - conn, - "SELECT id, text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM memories WHERE status = 'active'", - ); - let decisions = query_table_json( - conn, - "SELECT id, decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM decisions WHERE status = 'active'", - ); - - json!({ - "version": 1, - "exported_at": now_iso(), - "memories": memories, - "decisions": decisions, - "memories_count": memories.len(), - "decisions_count": decisions.len(), - }) -} - -pub fn export_json_page_value( - conn: &Connection, - limit: usize, - memories_offset: usize, - decisions_offset: usize, -) -> Value { - let limit = limit.clamp(1, MAX_EXPORT_PAGE_LIMIT); - let (memories, memories_has_more) = query_table_json_page( - conn, - "SELECT id, text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM memories WHERE status = 'active' ORDER BY id LIMIT ?1 OFFSET ?2", - limit, - memories_offset, - ); - let (decisions, decisions_has_more) = query_table_json_page( - conn, - "SELECT id, decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM decisions WHERE status = 'active' ORDER BY id LIMIT ?1 OFFSET ?2", - limit, - decisions_offset, - ); - - json!({ - "version": 1, - "mode": "page", - "exported_at": now_iso(), - "limit": limit, - "memories_offset": memories_offset, - "decisions_offset": decisions_offset, - "next_memories_offset": if memories_has_more { - Some(memories_offset.saturating_add(memories.len())) - } else { - None:: - }, - "next_decisions_offset": if decisions_has_more { - Some(decisions_offset.saturating_add(decisions.len())) - } else { - None:: - }, - "truncated": memories_has_more || decisions_has_more, - "memories": memories, - "decisions": decisions, - "memories_count": memories.len(), - "decisions_count": decisions.len(), - }) -} - -pub fn export_json_changeset_value(conn: &Connection, since: Option<&str>) -> Value { - let cursor = now_iso(); - let memories = query_table_json_since( - conn, - "SELECT id, text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM memories WHERE status = 'active' \ - AND (?1 IS NULL OR COALESCE(updated_at, created_at) > ?1) \ - AND COALESCE(updated_at, created_at) <= ?2", - since, - &cursor, - ); - let decisions = query_table_json_since( - conn, - "SELECT id, decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ - retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM decisions WHERE status = 'active' \ - AND (?1 IS NULL OR COALESCE(updated_at, created_at) > ?1) \ - AND COALESCE(updated_at, created_at) <= ?2", - since, - &cursor, - ); - - json!({ - "version": 1, - "mode": "changeset", - "exported_at": cursor, - "since": since, - "cursor": cursor, - "memories": memories, - "decisions": decisions, - "memories_count": memories.len(), - "decisions_count": decisions.len(), - }) -} - -pub fn export_sql_text(conn: &Connection) -> String { - let mut lines: Vec = vec![ - "-- Cortex export".to_string(), - format!("-- Exported at {}", now_iso()), - "BEGIN TRANSACTION;".to_string(), - ]; - - if let Ok(mut stmt) = conn.prepare( - "SELECT text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, observed_at, valid_from, valid_until FROM memories WHERE status = 'active'", - ) { - let rows = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - row.get::<_, Option>(9)?, - row.get::<_, Option>(10)?, - row.get::<_, Option>(11)?, - row.get::<_, Option>(12)?, - row.get::<_, Option>(13)?, - row.get::<_, Option>(14)?, - )) - }); - if let Ok(rows) = rows { - for row in rows.flatten() { - let ( - text, - source, - typ, - tags, - agent, - source_client, - source_model, - confidence, - reasoning_depth, - trust_score, - score, - retention_class, - observed_at, - valid_from, - valid_until, - ) = row; - lines.push(format!( - "INSERT INTO memories (text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, observed_at, valid_from, valid_until, status) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, 'active');", - sql_quote(&text), - sql_quote_opt(&source), - sql_quote_opt(&typ), - sql_quote_opt(&tags), - sql_quote_opt(&agent), - sql_quote_opt(&source_client), - sql_quote_opt(&source_model), - confidence.unwrap_or(0.8), - sql_quote_opt(&reasoning_depth), - trust_score.unwrap_or(confidence.unwrap_or(0.8)), - score.unwrap_or(1.0), - sql_quote(&retention_class.unwrap_or_else(|| "operational".to_string())), - sql_quote_opt(&observed_at), - sql_quote_opt(&valid_from), - sql_quote_opt(&valid_until), - )); - } - } - } - - if let Ok(mut stmt) = conn.prepare( - "SELECT decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, observed_at, valid_from, valid_until FROM decisions WHERE status = 'active'", - ) { - let rows = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - row.get::<_, Option>(9)?, - row.get::<_, Option>(10)?, - row.get::<_, Option>(11)?, - row.get::<_, Option>(12)?, - row.get::<_, Option>(13)?, - )) - }); - if let Ok(rows) = rows { - for row in rows.flatten() { - let ( - decision, - context, - typ, - agent, - source_client, - source_model, - confidence, - reasoning_depth, - trust_score, - score, - retention_class, - observed_at, - valid_from, - valid_until, - ) = row; - lines.push(format!( - "INSERT INTO decisions (decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, observed_at, valid_from, valid_until, status) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, 'active');", - sql_quote(&decision), - sql_quote_opt(&context), - sql_quote_opt(&typ), - sql_quote_opt(&agent), - sql_quote_opt(&source_client), - sql_quote_opt(&source_model), - confidence.unwrap_or(0.8), - sql_quote_opt(&reasoning_depth), - trust_score.unwrap_or(confidence.unwrap_or(0.8)), - score.unwrap_or(1.0), - sql_quote(&retention_class.unwrap_or_else(|| "operational".to_string())), - sql_quote_opt(&observed_at), - sql_quote_opt(&valid_from), - sql_quote_opt(&valid_until), - )); - } - } - } - - lines.push("COMMIT;".to_string()); - lines.join("\n") -} - -pub fn import_payload( - conn: &mut Connection, - payload: &ImportPayload, - options: &ImportOptions, -) -> Result { - let mut counts = ImportCounts::default(); - let visibility = options.visibility.as_deref().unwrap_or("private"); - let fallback = options.source_agent_fallback.as_str(); - - let memories_has_owner = column_exists(conn, "memories", "owner_id"); - let memories_has_visibility = column_exists(conn, "memories", "visibility"); - let decisions_has_owner = column_exists(conn, "decisions", "owner_id"); - let decisions_has_visibility = column_exists(conn, "decisions", "visibility"); - let tx = conn - .transaction() - .map_err(|e| format!("failed to start import transaction: {e}"))?; - - if let Some(memories) = &payload.memories { - for (idx, m) in memories.iter().enumerate() { - let entry_type = normalize_memory_entry_type(m.entry_type.as_deref()); - let inserted = if memories_has_owner && memories_has_visibility { - tx.execute( - "INSERT INTO memories (text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until, owner_id, visibility) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'active', ?13, ?14, ?15, ?16, ?17)", - params![ - m.text, - m.source, - entry_type, - m.tags, - m.source_agent.as_deref().unwrap_or(fallback), - m.source_client - .as_deref() - .unwrap_or(m.source_agent.as_deref().unwrap_or(fallback)), - m.source_model.as_deref(), - m.confidence.unwrap_or(0.8), - m.reasoning_depth.as_deref().unwrap_or("single-shot"), - m.trust_score.unwrap_or(m.confidence.unwrap_or(0.8)), - m.score.unwrap_or(1.0), - m.retention_class.unwrap_or_default().as_str(), - m.observed_at.as_deref(), - m.valid_from.as_deref(), - m.valid_until.as_deref(), - options.owner_id, - visibility, - ], - ) - } else { - tx.execute( - "INSERT INTO memories (text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'active', ?13, ?14, ?15)", - params![ - m.text, - m.source, - entry_type, - m.tags, - m.source_agent.as_deref().unwrap_or(fallback), - m.source_client - .as_deref() - .unwrap_or(m.source_agent.as_deref().unwrap_or(fallback)), - m.source_model.as_deref(), - m.confidence.unwrap_or(0.8), - m.reasoning_depth.as_deref().unwrap_or("single-shot"), - m.trust_score.unwrap_or(m.confidence.unwrap_or(0.8)), - m.score.unwrap_or(1.0), - m.retention_class.unwrap_or_default().as_str(), - m.observed_at.as_deref(), - m.valid_from.as_deref(), - m.valid_until.as_deref(), - ], - ) - }; - - match inserted { - Ok(_) => counts.memories += 1, - Err(e) => return Err(format!("failed to import memories[{idx}]: {e}")), - } - } - } - - if let Some(decisions) = &payload.decisions { - for (idx, d) in decisions.iter().enumerate() { - let entry_type = normalize_decision_entry_type(d.entry_type.as_deref()); - let inserted = if decisions_has_owner && decisions_has_visibility { - tx.execute( - "INSERT INTO decisions (decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until, owner_id, visibility) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 'active', ?12, ?13, ?14, ?15, ?16)", - params![ - d.decision, - d.context, - entry_type, - d.source_agent.as_deref().unwrap_or(fallback), - d.source_client - .as_deref() - .unwrap_or(d.source_agent.as_deref().unwrap_or(fallback)), - d.source_model.as_deref(), - d.confidence.unwrap_or(0.8), - d.reasoning_depth.as_deref().unwrap_or("single-shot"), - d.trust_score.unwrap_or(d.confidence.unwrap_or(0.8)), - d.score.unwrap_or(1.0), - d.retention_class.unwrap_or_default().as_str(), - d.observed_at.as_deref(), - d.valid_from.as_deref(), - d.valid_until.as_deref(), - options.owner_id, - visibility, - ], - ) - } else { - tx.execute( - "INSERT INTO decisions (decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 'active', ?12, ?13, ?14)", - params![ - d.decision, - d.context, - entry_type, - d.source_agent.as_deref().unwrap_or(fallback), - d.source_client - .as_deref() - .unwrap_or(d.source_agent.as_deref().unwrap_or(fallback)), - d.source_model.as_deref(), - d.confidence.unwrap_or(0.8), - d.reasoning_depth.as_deref().unwrap_or("single-shot"), - d.trust_score.unwrap_or(d.confidence.unwrap_or(0.8)), - d.score.unwrap_or(1.0), - d.retention_class.unwrap_or_default().as_str(), - d.observed_at.as_deref(), - d.valid_from.as_deref(), - d.valid_until.as_deref(), - ], - ) - }; - - match inserted { - Ok(_) => counts.decisions += 1, - Err(e) => return Err(format!("failed to import decisions[{idx}]: {e}")), - } - } - } - - tx.commit() - .map_err(|e| format!("failed to commit import transaction: {e}"))?; - Ok(counts) -} - -fn query_table_json(conn: &Connection, sql: &str) -> Vec { - let mut stmt = match conn.prepare(sql) { - Ok(s) => s, - Err(_) => return vec![], - }; - - let column_count = stmt.column_count(); - let column_names: Vec = (0..column_count) - .map(|i| stmt.column_name(i).unwrap_or("?").to_string()) - .collect(); - - stmt.query_map([], |row| { - let mut obj = serde_json::Map::new(); - for (i, name) in column_names.iter().enumerate() { - let val: Value = match row.get_ref(i) { - Ok(rusqlite::types::ValueRef::Null) => Value::Null, - Ok(rusqlite::types::ValueRef::Integer(n)) => json!(n), - Ok(rusqlite::types::ValueRef::Real(f)) => json!(f), - Ok(rusqlite::types::ValueRef::Text(s)) => { - json!(std::str::from_utf8(s).unwrap_or("")) - } - Ok(rusqlite::types::ValueRef::Blob(_)) => Value::Null, - Err(_) => Value::Null, - }; - obj.insert(name.clone(), val); - } - Ok(Value::Object(obj)) - }) - .ok() - .into_iter() - .flatten() - .filter_map(|r| r.ok()) - .collect() -} - -fn query_table_json_page( - conn: &Connection, - sql: &str, - limit: usize, - offset: usize, -) -> (Vec, bool) { - let fetch_limit = limit.saturating_add(1); - let mut rows = query_table_json_page_inner(conn, sql, fetch_limit, offset); - let has_more = rows.len() > limit; - if has_more { - rows.truncate(limit); - } - (rows, has_more) -} - -fn query_table_json_page_inner( - conn: &Connection, - sql: &str, - limit: usize, - offset: usize, -) -> Vec { - let mut stmt = match conn.prepare(sql) { - Ok(s) => s, - Err(_) => return vec![], - }; - - let column_count = stmt.column_count(); - let column_names: Vec = (0..column_count) - .map(|i| stmt.column_name(i).unwrap_or("?").to_string()) - .collect(); - - stmt.query_map(params![limit as i64, offset as i64], |row| { - let mut obj = serde_json::Map::new(); - for (i, name) in column_names.iter().enumerate() { - let val: Value = match row.get_ref(i) { - Ok(rusqlite::types::ValueRef::Null) => Value::Null, - Ok(rusqlite::types::ValueRef::Integer(n)) => json!(n), - Ok(rusqlite::types::ValueRef::Real(f)) => json!(f), - Ok(rusqlite::types::ValueRef::Text(s)) => { - json!(std::str::from_utf8(s).unwrap_or("")) - } - Ok(rusqlite::types::ValueRef::Blob(_)) => Value::Null, - Err(_) => Value::Null, - }; - obj.insert(name.clone(), val); - } - Ok(Value::Object(obj)) - }) - .ok() - .into_iter() - .flatten() - .filter_map(|r| r.ok()) - .collect() -} - -fn query_table_json_since( - conn: &Connection, - sql: &str, - since: Option<&str>, - cursor: &str, -) -> Vec { - let mut stmt = match conn.prepare(sql) { - Ok(s) => s, - Err(_) => return vec![], - }; - - let column_count = stmt.column_count(); - let column_names: Vec = (0..column_count) - .map(|i| stmt.column_name(i).unwrap_or("?").to_string()) - .collect(); - - stmt.query_map(params![since, cursor], |row| { - let mut obj = serde_json::Map::new(); - for (i, name) in column_names.iter().enumerate() { - let val: Value = match row.get_ref(i) { - Ok(rusqlite::types::ValueRef::Null) => Value::Null, - Ok(rusqlite::types::ValueRef::Integer(n)) => json!(n), - Ok(rusqlite::types::ValueRef::Real(f)) => json!(f), - Ok(rusqlite::types::ValueRef::Text(s)) => { - json!(std::str::from_utf8(s).unwrap_or("")) - } - Ok(rusqlite::types::ValueRef::Blob(_)) => Value::Null, - Err(_) => Value::Null, - }; - obj.insert(name.clone(), val); - } - Ok(Value::Object(obj)) - }) - .ok() - .into_iter() - .flatten() - .filter_map(|r| r.ok()) - .collect() -} - -fn now_iso() -> String { - chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) -} - -fn sql_quote(s: &str) -> String { - format!("'{}'", s.replace('\'', "''")) -} - -fn sql_quote_opt(s: &Option) -> String { - match s { - Some(v) => sql_quote(v), - None => "NULL".to_string(), - } -} - -fn column_exists(conn: &Connection, table: &str, column: &str) -> bool { - let mut stmt = match conn.prepare(&format!("PRAGMA table_info({table})")) { - Ok(v) => v, - Err(_) => return false, - }; - let rows = match stmt.query_map([], |row| row.get::<_, String>(1)) { - Ok(v) => v, - Err(_) => return false, - }; - for name in rows.flatten() { - if name == column { - return true; - } - } - false -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn export_changeset_filters_rows_by_since_cutoff() { - let conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - - conn.execute( - "INSERT INTO memories (text, source, status, created_at, updated_at) - VALUES (?1, ?2, 'active', ?3, ?4)", - params![ - "old memory", - "sync::old-memory", - "2026-01-01T00:00:00Z", - "2026-01-01T00:00:00Z" - ], - ) - .expect("insert old memory"); - conn.execute( - "INSERT INTO memories (text, source, status, created_at, updated_at) - VALUES (?1, ?2, 'active', ?3, ?4)", - params![ - "new memory", - "sync::new-memory", - "2026-03-01T00:00:00Z", - "2026-03-01T00:00:00Z" - ], - ) - .expect("insert new memory"); - conn.execute( - "INSERT INTO decisions (decision, context, status, created_at, updated_at) - VALUES (?1, ?2, 'active', ?3, ?4)", - params![ - "old decision", - "sync::old-decision", - "2026-01-01T00:00:00Z", - "2026-01-01T00:00:00Z" - ], - ) - .expect("insert old decision"); - conn.execute( - "INSERT INTO decisions (decision, context, status, created_at, updated_at) - VALUES (?1, ?2, 'active', ?3, ?4)", - params![ - "new decision", - "sync::new-decision", - "2026-03-01T00:00:00Z", - "2026-03-01T00:00:00Z" - ], - ) - .expect("insert new decision"); - - let changeset = export_json_changeset_value(&conn, Some("2026-02-01T00:00:00Z")); - let memories = changeset - .get("memories") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let decisions = changeset - .get("decisions") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - - assert_eq!(memories.len(), 1, "only new memory should be exported"); - assert_eq!(decisions.len(), 1, "only new decision should be exported"); - assert_eq!( - memories[0].get("source").and_then(Value::as_str), - Some("sync::new-memory") - ); - assert_eq!( - decisions[0].get("context").and_then(Value::as_str), - Some("sync::new-decision") - ); - } - - #[test] - fn export_changeset_respects_cursor_upper_bound() { - let conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - - conn.execute( - "INSERT INTO memories (text, source, status, created_at, updated_at) - VALUES (?1, ?2, 'active', ?3, ?4)", - params![ - "future memory", - "sync::future-memory", - "9999-01-01T00:00:00Z", - "9999-01-01T00:00:00Z" - ], - ) - .expect("insert future memory"); - - let changeset = export_json_changeset_value(&conn, None); - let memories = changeset - .get("memories") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - assert!( - memories.is_empty(), - "rows newer than cursor should be excluded" - ); - assert!( - changeset - .get("cursor") - .and_then(Value::as_str) - .is_some_and(|cursor| !cursor.trim().is_empty()), - "changeset cursor should always be emitted" - ); - } - - #[test] - fn export_json_page_limits_rows_and_emits_next_offsets() { - let conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - - for idx in 0..3 { - conn.execute( - "INSERT INTO memories (text, source, status) VALUES (?1, ?2, 'active')", - params![format!("memory {idx}"), format!("page::memory::{idx}")], - ) - .expect("insert memory"); - } - for idx in 0..2 { - conn.execute( - "INSERT INTO decisions (decision, context, status) VALUES (?1, ?2, 'active')", - params![format!("decision {idx}"), format!("page::decision::{idx}")], - ) - .expect("insert decision"); - } - - let first_page = export_json_page_value(&conn, 2, 0, 0); - assert_eq!( - first_page - .get("memories") - .and_then(Value::as_array) - .map(Vec::len), - Some(2) - ); - assert_eq!( - first_page - .get("decisions") - .and_then(Value::as_array) - .map(Vec::len), - Some(2) - ); - assert_eq!( - first_page - .get("next_memories_offset") - .and_then(Value::as_u64), - Some(2) - ); - assert_eq!( - first_page - .get("next_decisions_offset") - .and_then(Value::as_u64), - None - ); - assert_eq!( - first_page.get("truncated").and_then(Value::as_bool), - Some(true) - ); - - let second_page = export_json_page_value(&conn, 2, 2, 0); - let memories = second_page - .get("memories") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - assert_eq!(memories.len(), 1); - assert_eq!( - memories[0].get("source").and_then(Value::as_str), - Some("page::memory::2") - ); - assert_eq!( - second_page - .get("next_memories_offset") - .and_then(Value::as_u64), - None - ); - } - - #[test] - fn import_payload_normalizes_types_and_preserves_temporal_fields() { - let mut conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - - let payload = ImportPayload { - memories: Some(vec![crate::api_types::ImportMemory { - text: "deployment runbook".to_string(), - source: Some("ops".to_string()), - entry_type: Some("note".to_string()), - tags: Some("deploy".to_string()), - source_agent: Some("importer".to_string()), - source_client: Some("tests".to_string()), - source_model: Some("model-a".to_string()), - confidence: Some(0.91), - reasoning_depth: Some("analysis".to_string()), - trust_score: Some(0.88), - score: Some(1.2), - observed_at: Some("2026-04-18T10:00:00Z".to_string()), - valid_from: Some("2026-04-18T00:00:00Z".to_string()), - valid_until: Some("2026-05-18T00:00:00Z".to_string()), - retention_class: Some(crate::api_types::RetentionClass::Operational), - }]), - decisions: Some(vec![crate::api_types::ImportDecision { - decision: "route traffic via canary".to_string(), - context: Some("release gate".to_string()), - entry_type: Some("rule".to_string()), - source_agent: Some("importer".to_string()), - source_client: Some("tests".to_string()), - source_model: Some("model-b".to_string()), - confidence: Some(0.86), - reasoning_depth: Some("analysis".to_string()), - trust_score: Some(0.83), - score: Some(1.1), - observed_at: Some("2026-04-18T11:00:00Z".to_string()), - valid_from: Some("2026-04-18T00:00:00Z".to_string()), - valid_until: Some("2026-05-01T00:00:00Z".to_string()), - retention_class: Some(crate::api_types::RetentionClass::Audit), - }]), - }; - - let counts = import_payload(&mut conn, &payload, &ImportOptions::default()) - .expect("import should succeed"); - assert_eq!(counts.memories, 1); - assert_eq!(counts.decisions, 1); - - let memory_row: ( - String, - String, - Option, - Option, - Option, - ) = conn - .query_row( - "SELECT type, retention_class, observed_at, valid_from, valid_until FROM memories LIMIT 1", - [], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - )) - }, - ) - .expect("memory row"); - assert_eq!(memory_row.0, "fact"); - assert_eq!(memory_row.1, "operational"); - assert_eq!(memory_row.2.as_deref(), Some("2026-04-18T10:00:00Z")); - assert_eq!(memory_row.3.as_deref(), Some("2026-04-18T00:00:00Z")); - assert_eq!(memory_row.4.as_deref(), Some("2026-05-18T00:00:00Z")); - - let decision_row: ( - String, - String, - Option, - Option, - Option, - ) = conn - .query_row( - "SELECT type, retention_class, observed_at, valid_from, valid_until FROM decisions LIMIT 1", - [], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - )) - }, - ) - .expect("decision row"); - assert_eq!(decision_row.0, "decision"); - assert_eq!(decision_row.1, "audit"); - assert_eq!(decision_row.2.as_deref(), Some("2026-04-18T11:00:00Z")); - assert_eq!(decision_row.3.as_deref(), Some("2026-04-18T00:00:00Z")); - assert_eq!(decision_row.4.as_deref(), Some("2026-05-01T00:00:00Z")); - } - - #[test] - fn import_payload_rolls_back_and_reports_failed_rows() { - let mut conn = Connection::open_in_memory().expect("open sqlite"); - crate::db::configure(&conn).expect("configure sqlite"); - crate::db::initialize_schema(&conn).expect("initialize schema"); - crate::db::run_pending_migrations(&conn); - conn.execute( - "CREATE TRIGGER fail_import_memory BEFORE INSERT ON memories - WHEN NEW.source = 'fail' - BEGIN - SELECT RAISE(ABORT, 'forced import failure'); - END", - [], - ) - .expect("create failure trigger"); - - let payload = ImportPayload { - memories: Some(vec![ - crate::api_types::ImportMemory { - text: "first memory".to_string(), - source: Some("ok".to_string()), - entry_type: None, - tags: None, - source_agent: None, - source_client: None, - source_model: None, - confidence: None, - reasoning_depth: None, - trust_score: None, - score: None, - observed_at: None, - valid_from: None, - valid_until: None, - retention_class: None, - }, - crate::api_types::ImportMemory { - text: "second memory".to_string(), - source: Some("fail".to_string()), - entry_type: None, - tags: None, - source_agent: None, - source_client: None, - source_model: None, - confidence: None, - reasoning_depth: None, - trust_score: None, - score: None, - observed_at: None, - valid_from: None, - valid_until: None, - retention_class: None, - }, - ]), - decisions: None, - }; - - let err = import_payload(&mut conn, &payload, &ImportOptions::default()) - .expect_err("second memory should fail"); - assert!(err.contains("memories[1]")); - - let row_count: i64 = conn - .query_row("SELECT COUNT(*) FROM memories", [], |row| row.get(0)) - .expect("count memories"); - assert_eq!(row_count, 0, "import should roll back earlier rows"); - } -} diff --git a/daemon-rs/src/export_data/mod.rs b/daemon-rs/src/export_data/mod.rs new file mode 100644 index 00000000..cfa80ee5 --- /dev/null +++ b/daemon-rs/src/export_data/mod.rs @@ -0,0 +1,192 @@ +pub use crate::api_types::{ImportCounts, ImportOptions, ImportPayload}; +use rusqlite::{params, Connection}; +use serde_json::{json, Value}; +pub const DEFAULT_EXPORT_PAGE_LIMIT: usize = 1000; +pub const MAX_EXPORT_PAGE_LIMIT: usize = 5000; +fn normalize_entry_type(raw: Option<&str>, default: &str, aliases: &[(&[&str], &str)]) -> String { + let normalized = raw.map(str::trim).filter(|value| !value.is_empty()).map(str::to_ascii_lowercase).unwrap_or_else(|| default.to_string()); + for (keys, mapped) in aliases { + if keys.contains(&normalized.as_str()) { + return (*mapped).to_string(); + } + } + normalized +} +fn normalize_memory_entry_type(raw: Option<&str>) -> String { + normalize_entry_type( + raw, + "fact", + &[ + (&["memory", "note", "finding", "observation", "fact"], "fact"), + (&["episode", "event"], "episode"), + (&["procedure", "playbook", "runbook", "howto", "how-to"], "procedure"), + (&["evidence", "citation", "reference"], "evidence"), + (&["decision", "policy", "rule"], "decision"), + ], + ) +} +fn normalize_decision_entry_type(raw: Option<&str>) -> String { + normalize_entry_type( + raw, + "decision", + &[ + (&["decision", "policy", "rule"], "decision"), + (&["procedure", "playbook", "runbook"], "procedure"), + (&["evidence", "citation", "reference"], "evidence"), + (&["fact", "memory", "note"], "fact"), + ], + ) +} +pub fn export_json_page_value(conn: &Connection, limit: usize, memories_offset: usize, decisions_offset: usize) -> Value { + let limit = limit.clamp(1, MAX_EXPORT_PAGE_LIMIT); + let(memories,memories_has_more)= +query_table_json_page(conn, +"SELECT id, text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ + retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM memories WHERE status = 'active' ORDER BY id LIMIT ?1 OFFSET ?2" +,limit,memories_offset,); + let(decisions,decisions_has_more)=query_table_json_page(conn, +"SELECT id, decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, retention_class, status, score, \ + retrievals, pinned, observed_at, valid_from, valid_until, created_at, updated_at FROM decisions WHERE status = 'active' ORDER BY id LIMIT ?1 OFFSET ?2" +,limit,decisions_offset,); + json!({"version":1,"mode":"page","exported_at":now_iso(),"limit":limit,"memories_offset":memories_offset +,"decisions_offset":decisions_offset,"next_memories_offset":if memories_has_more{Some(memories_offset.saturating_add(memories.len( +)))}else{None::},"next_decisions_offset":if decisions_has_more{Some(decisions_offset.saturating_add(decisions.len()))}else{ +None::},"truncated":memories_has_more||decisions_has_more,"memories":memories,"decisions":decisions,"memories_count": +memories.len(),"decisions_count":decisions.len(),}) +} +pub fn export_json_changeset_value(conn: &Connection, since: Option<&str>) -> Value { + let cursor = now_iso(); + let lower = since.unwrap_or("0000-00-00T00:00:00Z"); + let memories = query_rows_json( + conn, + "SELECT id, text, source, type, status, created_at, updated_at FROM memories WHERE status = 'active' AND updated_at > ?1 AND updated_at <= ?2 ORDER BY id", + &[&lower, &cursor], + ); + let decisions = query_rows_json( + conn, + "SELECT id, decision, context, type, status, created_at, updated_at FROM decisions WHERE status = 'active' AND updated_at > ?1 AND updated_at <= ?2 ORDER BY id", + &[&lower, &cursor], + ); + json!({"version":1,"mode":"changeset","cursor":cursor,"since":since,"memories":memories,"decisions":decisions}) +} +pub fn import_payload(conn: &mut Connection, payload: &ImportPayload, options: &ImportOptions) -> Result { + let mut counts = ImportCounts::default(); + let visibility = options.visibility.as_deref().unwrap_or("private"); + let fallback = options.source_agent_fallback.as_str(); + let memories_has_owner = column_exists(conn, "memories", "owner_id"); + let memories_has_visibility = column_exists(conn, "memories", "visibility"); + let decisions_has_owner = column_exists(conn, "decisions", "owner_id"); + let decisions_has_visibility = column_exists(conn, "decisions", "visibility"); + let tx = conn.transaction().map_err(|e| format!("failed to start import transaction: {e}"))?; + if let Some(memories) = &payload.memories { + for (idx, m) in memories.iter().enumerate() { + let entry_type = normalize_memory_entry_type(m.entry_type.as_deref()); + let inserted = if memories_has_owner && memories_has_visibility { + tx. +execute( +"INSERT INTO memories (text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until, owner_id, visibility) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'active', ?13, ?14, ?15, ?16, ?17)" +,params![m.text,m.source,entry_type,m.tags,m.source_agent.as_deref().unwrap_or(fallback),m.source_client.as_deref().unwrap_or(m. +source_agent.as_deref().unwrap_or(fallback)),m.source_model.as_deref(),m.confidence.unwrap_or(0.8),m.reasoning_depth.as_deref(). +unwrap_or("single-shot"),m.trust_score.unwrap_or(m.confidence.unwrap_or(0.8)),m.score.unwrap_or(1.0),m.retention_class. +unwrap_or_default().as_str(),m.observed_at.as_deref(),m.valid_from.as_deref(),m.valid_until.as_deref(),options.owner_id,visibility +,],) + } else { + tx.execute( +"INSERT INTO memories (text, source, type, tags, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'active', ?13, ?14, ?15)" +,params![m.text,m.source,entry_type,m.tags,m.source_agent.as_deref().unwrap_or(fallback),m.source_client.as_deref().unwrap_or(m. +source_agent.as_deref().unwrap_or(fallback)),m.source_model.as_deref(),m.confidence.unwrap_or(0.8),m.reasoning_depth.as_deref(). +unwrap_or("single-shot"),m.trust_score.unwrap_or(m.confidence.unwrap_or(0.8)),m.score.unwrap_or(1.0),m.retention_class. +unwrap_or_default().as_str(),m.observed_at.as_deref(),m.valid_from.as_deref(),m.valid_until.as_deref(),],) + }; + match inserted { + Ok(_) => counts.memories += 1, + Err(e) => return Err(format!("failed to import memories[{idx}]: {e}")), + } + } + } + if let Some(decisions) = &payload.decisions { + for (idx, d) in decisions.iter().enumerate() { + let entry_type = normalize_decision_entry_type(d.entry_type.as_deref()); + let inserted = if decisions_has_owner && decisions_has_visibility { + tx.execute( +"INSERT INTO decisions (decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until, owner_id, visibility) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 'active', ?12, ?13, ?14, ?15, ?16)" +,params![d.decision,d.context,entry_type,d.source_agent.as_deref().unwrap_or(fallback),d.source_client.as_deref().unwrap_or(d. +source_agent.as_deref().unwrap_or(fallback)),d.source_model.as_deref(),d.confidence.unwrap_or(0.8),d.reasoning_depth.as_deref(). +unwrap_or("single-shot"),d.trust_score.unwrap_or(d.confidence.unwrap_or(0.8)),d.score.unwrap_or(1.0),d.retention_class. +unwrap_or_default().as_str(),d.observed_at.as_deref(),d.valid_from.as_deref(),d.valid_until.as_deref(),options.owner_id,visibility +,],) + } else { + tx.execute( +"INSERT INTO decisions (decision, context, type, source_agent, source_client, source_model, confidence, reasoning_depth, trust_score, score, retention_class, status, observed_at, valid_from, valid_until) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 'active', ?12, ?13, ?14)" +,params![d.decision,d.context,entry_type,d.source_agent.as_deref().unwrap_or(fallback),d.source_client.as_deref().unwrap_or(d. +source_agent.as_deref().unwrap_or(fallback)),d.source_model.as_deref(),d.confidence.unwrap_or(0.8),d.reasoning_depth.as_deref(). +unwrap_or("single-shot"),d.trust_score.unwrap_or(d.confidence.unwrap_or(0.8)),d.score.unwrap_or(1.0),d.retention_class. +unwrap_or_default().as_str(),d.observed_at.as_deref(),d.valid_from.as_deref(),d.valid_until.as_deref(),],) + }; + match inserted { + Ok(_) => counts.decisions += 1, + Err(e) => return Err(format!("failed to import decisions[{idx}]: {e}")), + } + } + } + tx.commit().map_err(|e| format!("failed to commit import transaction: {e}"))?; + Ok(counts) +} +fn row_to_json(row: &rusqlite::Row<'_>, column_names: &[String]) -> rusqlite::Result { + let mut obj = serde_json::Map::new(); + for (i, name) in column_names.iter().enumerate() { + let val: Value = match row.get_ref(i) { + Ok(rusqlite::types::ValueRef::Null) => Value::Null, + Ok(rusqlite::types::ValueRef::Integer(n)) => json!(n), + Ok(rusqlite::types::ValueRef::Real(f)) => json!(f), + Ok(rusqlite::types::ValueRef::Text(s)) => json!(std::str::from_utf8(s).unwrap_or("")), + Ok(rusqlite::types::ValueRef::Blob(_)) => Value::Null, + Err(_) => Value::Null, + }; + obj.insert(name.clone(), val); + } + Ok(Value::Object(obj)) +} +fn query_rows_json(conn: &Connection, sql: &str, bind: &[&dyn rusqlite::types::ToSql]) -> Vec { + let mut stmt = match conn.prepare(sql) { + Ok(s) => s, + Err(_) => return vec![], + }; + let column_names: Vec = (0..stmt.column_count()).map(|i| stmt.column_name(i).unwrap_or("?").to_string()).collect(); + stmt.query_map(bind, |row| row_to_json(row, &column_names)).ok().into_iter().flatten().filter_map(|r| r.ok()).collect() +} +fn query_table_json_page(conn: &Connection, sql: &str, limit: usize, offset: usize) -> (Vec, bool) { + let fetch_limit = limit.saturating_add(1) as i64; + let offset = offset as i64; + let mut rows = query_rows_json(conn, sql, &[&fetch_limit, &offset]); + let has_more = rows.len() > limit; + if has_more { + rows.truncate(limit); + } + (rows, has_more) +} +fn now_iso() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} +fn column_exists(conn: &Connection, table: &str, column: &str) -> bool { + let mut stmt = match conn.prepare(&format!("PRAGMA table_info({table})")) { + Ok(v) => v, + Err(_) => return false, + }; + let rows = match stmt.query_map([], |row| row.get::<_, String>(1)) { + Ok(v) => v, + Err(_) => return false, + }; + for name in rows.flatten() { + if name == column { + return true; + } + } + false +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/export_data/tests/mod.rs b/daemon-rs/src/export_data/tests/mod.rs new file mode 100644 index 00000000..b1e19873 --- /dev/null +++ b/daemon-rs/src/export_data/tests/mod.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +#[test] +fn export_changeset_filters_rows_by_since_cutoff() { + let conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + conn.execute( + "INSERT INTO memories (text, source, status, created_at, updated_at) + VALUES (?1, ?2, 'active', ?3, ?4)", + params!["old memory", "sync::old-memory", "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"], + ) + .expect("insert old memory"); + conn.execute( + "INSERT INTO memories (text, source, status, created_at, updated_at) + VALUES (?1, ?2, 'active', ?3, ?4)", + params!["new memory", "sync::new-memory", "2026-03-01T00:00:00Z", "2026-03-01T00:00:00Z"], + ) + .expect("insert new memory"); + conn.execute( + "INSERT INTO decisions (decision, context, status, created_at, updated_at) + VALUES (?1, ?2, 'active', ?3, ?4)", + params!["old decision", "sync::old-decision", "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"], + ) + .expect("insert old decision"); + conn.execute( + "INSERT INTO decisions (decision, context, status, created_at, updated_at) + VALUES (?1, ?2, 'active', ?3, ?4)", + params!["new decision", "sync::new-decision", "2026-03-01T00:00:00Z", "2026-03-01T00:00:00Z"], + ) + .expect("insert new decision"); + let changeset = export_json_changeset_value(&conn, Some("2026-02-01T00:00:00Z")); + let memories = changeset.get("memories").and_then(Value::as_array).cloned().unwrap_or_default(); + let decisions = changeset.get("decisions").and_then(Value::as_array).cloned().unwrap_or_default(); + assert_eq!(memories.len(), 1, "only new memory should be exported"); + assert_eq!(decisions.len(), 1, "only new decision should be exported"); + assert_eq!(memories[0].get("source").and_then(Value::as_str), Some("sync::new-memory")); + assert_eq!(decisions[0].get("context").and_then(Value::as_str), Some("sync::new-decision")); +} +#[test] +fn export_changeset_respects_cursor_upper_bound() { + let conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + conn.execute( + "INSERT INTO memories (text, source, status, created_at, updated_at) + VALUES (?1, ?2, 'active', ?3, ?4)", + params!["future memory", "sync::future-memory", "9999-01-01T00:00:00Z", "9999-01-01T00:00:00Z"], + ) + .expect("insert future memory"); + let changeset = export_json_changeset_value(&conn, None); + let memories = changeset.get("memories").and_then(Value::as_array).cloned().unwrap_or_default(); + assert!(memories.is_empty(), "rows newer than cursor should be excluded"); + assert!(changeset.get("cursor").and_then(Value::as_str).is_some_and(|cursor| !cursor.trim().is_empty()), "changeset cursor should always be emitted"); +} +#[test] +fn export_json_page_limits_rows_and_emits_next_offsets() { + let conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + for idx in 0..3 { + conn.execute("INSERT INTO memories (text, source, status) VALUES (?1, ?2, 'active')", params![format!("memory {idx}"), format!("page::memory::{idx}")]) + .expect("insert memory"); + } + for idx in 0..2 { + conn.execute( + "INSERT INTO decisions (decision, context, status) VALUES (?1, ?2, 'active')", + params![format!("decision {idx}"), format!("page::decision::{idx}")], + ) + .expect("insert decision"); + } + let first_page = export_json_page_value(&conn, 2, 0, 0); + assert_eq!(first_page.get("memories").and_then(Value::as_array).map(Vec::len), Some(2)); + assert_eq!(first_page.get("decisions").and_then(Value::as_array).map(Vec::len), Some(2)); + assert_eq!(first_page.get("next_memories_offset").and_then(Value::as_u64), Some(2)); + assert_eq!(first_page.get("next_decisions_offset").and_then(Value::as_u64), None); + assert_eq!(first_page.get("truncated").and_then(Value::as_bool), Some(true)); + let second_page = export_json_page_value(&conn, 2, 2, 0); + let memories = second_page.get("memories").and_then(Value::as_array).cloned().unwrap_or_default(); + assert_eq!(memories.len(), 1); + assert_eq!(memories[0].get("source").and_then(Value::as_str), Some("page::memory::2")); + assert_eq!(second_page.get("next_memories_offset").and_then(Value::as_u64), None); +} +#[test] +fn import_payload_normalizes_types_and_preserves_temporal_fields() { + let mut conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + let payload = ImportPayload { + memories: Some(vec![crate::api_types::ImportMemory { + text: "deployment runbook".to_string(), + source: Some("ops".to_string()), + entry_type: Some("note".to_string()), + tags: Some("deploy".to_string()), + source_agent: Some("importer".to_string()), + source_client: Some("tests".to_string()), + source_model: Some("model-a".to_string()), + confidence: Some(0.91), + reasoning_depth: Some("analysis".to_string()), + trust_score: Some(0.88), + score: Some(1.2), + observed_at: Some("2026-04-18T10:00:00Z".to_string()), + valid_from: Some("2026-04-18T00:00:00Z".to_string()), + valid_until: Some("2026-05-18T00:00:00Z".to_string()), + retention_class: Some(crate::api_types::RetentionClass::Operational), + }]), + decisions: Some(vec![crate::api_types::ImportDecision { + decision: "route traffic via canary".to_string(), + context: Some("release gate".to_string()), + entry_type: Some("rule".to_string()), + source_agent: Some("importer".to_string()), + source_client: Some("tests".to_string()), + source_model: Some("model-b".to_string()), + confidence: Some(0.86), + reasoning_depth: Some("analysis".to_string()), + trust_score: Some(0.83), + score: Some(1.1), + observed_at: Some("2026-04-18T11:00:00Z".to_string()), + valid_from: Some("2026-04-18T00:00:00Z".to_string()), + valid_until: Some("2026-05-01T00:00:00Z".to_string()), + retention_class: Some(crate::api_types::RetentionClass::Audit), + }]), + }; + let counts = import_payload(&mut conn, &payload, &ImportOptions::default()).expect("import should succeed"); + assert_eq!(counts.memories, 1); + assert_eq!(counts.decisions, 1); + let memory_row: (String, String, Option, Option, Option) = conn + .query_row("SELECT type, retention_class, observed_at, valid_from, valid_until FROM memories LIMIT 1", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)) + }) + .expect("memory row"); + assert_eq!(memory_row.0, "fact"); + assert_eq!(memory_row.1, "operational"); + assert_eq!(memory_row.2.as_deref(), Some("2026-04-18T10:00:00Z")); + assert_eq!(memory_row.3.as_deref(), Some("2026-04-18T00:00:00Z")); + assert_eq!(memory_row.4.as_deref(), Some("2026-05-18T00:00:00Z")); + let decision_row: (String, String, Option, Option, Option) = conn + .query_row("SELECT type, retention_class, observed_at, valid_from, valid_until FROM decisions LIMIT 1", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)) + }) + .expect("decision row"); + assert_eq!(decision_row.0, "decision"); + assert_eq!(decision_row.1, "audit"); + assert_eq!(decision_row.2.as_deref(), Some("2026-04-18T11:00:00Z")); + assert_eq!(decision_row.3.as_deref(), Some("2026-04-18T00:00:00Z")); + assert_eq!(decision_row.4.as_deref(), Some("2026-05-01T00:00:00Z")); +} +#[test] +fn import_payload_rolls_back_and_reports_failed_rows() { + let mut conn = Connection::open_in_memory().expect("open sqlite"); + crate::db::configure(&conn).expect("configure sqlite"); + crate::db::initialize_schema(&conn).expect("initialize schema"); + crate::db::run_pending_migrations(&conn); + conn.execute( + "CREATE TRIGGER fail_import_memory BEFORE INSERT ON memories + WHEN NEW.source = 'fail' + BEGIN + SELECT RAISE(ABORT, 'forced import failure'); + END", + [], + ) + .expect("create failure trigger"); + let payload = ImportPayload { + memories: Some(vec![ + crate::api_types::ImportMemory { + text: "first memory".to_string(), + source: Some("ok".to_string()), + entry_type: None, + tags: None, + source_agent: None, + source_client: None, + source_model: None, + confidence: None, + reasoning_depth: None, + trust_score: None, + score: None, + observed_at: None, + valid_from: None, + valid_until: None, + retention_class: None, + }, + crate::api_types::ImportMemory { + text: "second memory".to_string(), + source: Some("fail".to_string()), + entry_type: None, + tags: None, + source_agent: None, + source_client: None, + source_model: None, + confidence: None, + reasoning_depth: None, + trust_score: None, + score: None, + observed_at: None, + valid_from: None, + valid_until: None, + retention_class: None, + }, + ]), + decisions: None, + }; + let err = import_payload(&mut conn, &payload, &ImportOptions::default()).expect_err("second memory should fail"); + assert!(err.contains("memories[1]")); + let row_count: i64 = conn.query_row("SELECT COUNT(*) FROM memories", [], |row| row.get(0)).expect("count memories"); + assert_eq!(row_count, 0, "import should roll back earlier rows"); +} diff --git a/daemon-rs/src/focus.rs b/daemon-rs/src/focus.rs index d2469faf..cf746e8d 100644 --- a/daemon-rs/src/focus.rs +++ b/daemon-rs/src/focus.rs @@ -1,125 +1,53 @@ -// SPDX-License-Identifier: MIT -//! Focus tools — context checkpointing for the sawtooth pattern. -//! -//! `focus_start(label)` opens a focus session. Entries stored/recalled during -//! the session are tracked as raw traces. -//! -//! `focus_end(label)` summarizes the raw traces into a compact summary, -//! stores the summary as a memory, and marks the session complete. -//! -//! The net effect: context grows during work (exploration), then collapses -//! at checkpoints (consolidation). Research shows 22.7% token reduction -//! with no loss in task accuracy (Focus Architecture, SWE-bench). - +use crate::handlers::estimate_tokens; use rusqlite::{params, Connection}; use serde_json::{json, Value}; - -use crate::handlers::estimate_tokens; - -/// Start a new focus session. Returns the session ID. pub fn focus_start(conn: &Connection, label: &str, agent: &str) -> Result { - // Check for existing open session with same label let existing: Option = conn - .query_row( - "SELECT id FROM focus_sessions WHERE label = ?1 AND agent = ?2 AND status = 'open'", - params![label, agent], - |row| row.get(0), - ) + .query_row("SELECT id FROM focus_sessions WHERE label = ?1 AND agent = ?2 AND status = 'open'", params![label, agent], |row| row.get(0)) .ok(); - if let Some(id) = existing { - return Ok(json!({ - "id": id, - "label": label, - "status": "already_open", - "message": "Focus session already open with this label" - })); + return Ok(json!({"id":id,"label":label,"status":"already_open","message": +"Focus session already open with this label"})); } - - conn.execute( - "INSERT INTO focus_sessions (label, agent, status, raw_entries) VALUES (?1, ?2, 'open', '[]')", - params![label, agent], - ) - .map_err(|e| format!("Failed to start focus: {e}"))?; - + conn.execute("INSERT INTO focus_sessions (label, agent, status, raw_entries) VALUES (?1, ?2, 'open', '[]')", params![label, agent]) + .map_err(|e| format!("Failed to start focus: {e}"))?; let id = conn.last_insert_rowid(); - - Ok(json!({ - "id": id, - "label": label, - "status": "opened", - "message": format!("Focus started: '{label}'. Store decisions normally — they'll be tracked. Call focus_end when done.") - })) + Ok(json!({"id":id,"label":label,"status":"opened", +"message":format!("Focus started: '{label}'. Store decisions normally — they'll be tracked. Call focus_end when done.")})) } - -/// Add an entry to the active focus session's raw trace. -/// Called automatically when cortex_store happens during an open focus. pub fn focus_append(conn: &Connection, agent: &str, entry: &str) -> bool { let result: Option<(i64, String)> = conn - .query_row( - "SELECT id, raw_entries FROM focus_sessions WHERE agent = ?1 AND status = 'open' ORDER BY started_at DESC LIMIT 1", - params![agent], - |row| Ok((row.get(0)?, row.get(1)?)), - ) + .query_row("SELECT id, raw_entries FROM focus_sessions WHERE agent = ?1 AND status = 'open' ORDER BY started_at DESC LIMIT 1", params![agent], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) .ok(); - if let Some((id, raw_json)) = result { let mut entries: Vec = serde_json::from_str(&raw_json).unwrap_or_default(); entries.push(entry.to_string()); let updated = serde_json::to_string(&entries).unwrap_or_else(|_| "[]".to_string()); - let _ = conn.execute( - "UPDATE focus_sessions SET raw_entries = ?1 WHERE id = ?2", - params![updated, id], - ); + let _ = conn.execute("UPDATE focus_sessions SET raw_entries = ?1 WHERE id = ?2", params![updated, id]); true } else { false } } - -/// End a focus session. Summarizes raw traces, stores the summary, returns stats. -pub fn focus_end( - conn: &Connection, - label: &str, - agent: &str, - owner_id: Option, -) -> Result { +pub fn focus_end(conn: &Connection, label: &str, agent: &str, owner_id: Option) -> Result { let session: Option<(i64, String)> = conn - .query_row( - "SELECT id, raw_entries FROM focus_sessions WHERE label = ?1 AND agent = ?2 AND status = 'open'", - params![label, agent], - |row| Ok((row.get(0)?, row.get(1)?)), - ) + .query_row("SELECT id, raw_entries FROM focus_sessions WHERE label = ?1 AND agent = ?2 AND status = 'open'", params![label, agent], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) .ok(); - - let (id, raw_json) = - session.ok_or_else(|| format!("No open focus session with label '{label}'"))?; + let (id, raw_json) = session.ok_or_else(|| format!("No open focus session with label '{label}'"))?; let entries: Vec = serde_json::from_str(&raw_json).unwrap_or_default(); - if entries.is_empty() { - // Close without summary - conn.execute( - "UPDATE focus_sessions SET status = 'closed', ended_at = datetime('now') WHERE id = ?1", - params![id], - ) - .map_err(|e| e.to_string())?; - - return Ok(json!({ - "id": id, - "label": label, - "status": "closed", - "entries": 0, - "summary": null, - "message": "Focus closed (no entries captured)" - })); + conn.execute("UPDATE focus_sessions SET status = 'closed', ended_at = datetime('now') WHERE id = ?1", params![id]) + .map_err(|e| e.to_string())?; + return Ok(json!({"id":id,"label":label,"status":"closed","entries":0,"summary":null,"message": +"Focus closed (no entries captured)"})); } - - // Summarize the raw entries let tokens_before = entries.iter().map(|e| estimate_tokens(e)).sum::(); let summary = summarize_entries(&entries); let tokens_after = estimate_tokens(&summary); - - // Store the summary as a memory if let Some(oid) = owner_id { conn.execute( "INSERT INTO memories (text, source, type, source_agent, confidence, owner_id) VALUES (?1, ?2, 'focus_summary', ?3, 0.9, ?4)", @@ -132,34 +60,16 @@ pub fn focus_end( ) } .map_err(|e| format!("Failed to store focus summary: {e}"))?; - - // Close the session conn.execute( "UPDATE focus_sessions SET status = 'closed', summary = ?1, ended_at = datetime('now'), tokens_before = ?2, tokens_after = ?3 WHERE id = ?4", params![summary, tokens_before as i64, tokens_after as i64, id], ) .map_err(|e| e.to_string())?; - - let savings = if tokens_before > 0 { - ((1.0 - (tokens_after as f64 / tokens_before as f64)) * 100.0).round() as i64 - } else { - 0 - }; - - Ok(json!({ - "id": id, - "label": label, - "status": "closed", - "entries": entries.len(), - "tokensBefore": tokens_before, - "tokensAfter": tokens_after, - "savings": format!("{savings}%"), - "summary": summary, - "message": format!("Focus '{label}' consolidated: {} entries → {} tokens ({}% reduction)", entries.len(), tokens_after, savings) - })) + let savings = if tokens_before > 0 { ((1.0 - (tokens_after as f64 / tokens_before as f64)) * 100.0).round() as i64 } else { 0 }; + Ok(json!({"id":id,"label":label,"status":"closed", +"entries":entries.len(),"tokensBefore":tokens_before,"tokensAfter":tokens_after,"savings":format!("{savings}%"),"summary":summary, +"message":format!("Focus '{label}' consolidated: {} entries → {} tokens ({}% reduction)",entries.len(),tokens_after,savings)})) } - -/// Get the currently open focus session for an agent. pub fn focus_current(conn: &Connection, agent: &str) -> Option { conn.query_row( "SELECT id, label, raw_entries, started_at FROM focus_sessions WHERE agent = ?1 AND status = 'open' ORDER BY started_at DESC LIMIT 1", @@ -168,23 +78,15 @@ pub fn focus_current(conn: &Connection, agent: &str) -> Option { let raw: String = row.get(2)?; let entries: Vec = serde_json::from_str(&raw).unwrap_or_default(); Ok(json!({ - "id": row.get::<_, i64>(0)?, - "label": row.get::<_, String>(1)?, - "entries": entries.len(), - "startedAt": row.get::<_, String>(3)?, - })) +"id":row.get::<_,i64>(0)?,"label":row.get::<_,String>(1)?,"entries":entries.len(),"startedAt":row.get::<_,String>(3)?,})) }, ) .ok() } - -// ─── Summarization ────────────────────────────────────────────────────────── - fn summarize_entries(entries: &[String]) -> String { if entries.len() <= 3 { return entries.join(" | "); } - let high_signal = [ "decision", "fixed", @@ -203,29 +105,22 @@ fn summarize_entries(entries: &[String]) -> String { "must", "never", ]; - let mut kept: Vec<&str> = Vec::new(); - for entry in entries { let lower = entry.to_lowercase(); if high_signal.iter().any(|kw| lower.contains(kw)) { kept.push(entry); } } - - // If nothing matched high-signal, keep first and last if kept.is_empty() { kept.push(&entries[0]); if entries.len() > 1 { kept.push(&entries[entries.len() - 1]); } } - - // Cap at 5 entries if kept.len() > 5 { kept.truncate(5); } - let result = kept.join(" | "); if result.len() > 500 { result.chars().take(500).collect::() + "..." diff --git a/daemon-rs/src/handlers/admin/data.rs b/daemon-rs/src/handlers/admin/data.rs index a6b8e8eb..e15e1a3b 100644 --- a/daemon-rs/src/handlers/admin/data.rs +++ b/daemon-rs/src/handlers/admin/data.rs @@ -1,43 +1,29 @@ -// SPDX-License-Identifier: MIT +use super::types::{is_allowed_table, ArchiveBody, AssignOwnerBody, SetVisibilityBody, OWNER_TABLES, VISIBILITY_TABLES}; +use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use rusqlite::params; -use serde_json::{json, Value}; - -use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; -use crate::state::RuntimeState; - -use super::types::{ - is_allowed_table, ArchiveBody, AssignOwnerBody, SetVisibilityBody, OWNER_TABLES, - VISIBILITY_TABLES, -}; - +use serde_json::json; pub async fn handle_unowned(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db.lock().await; + let conn = state.db_read.lock().await; if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let mut unowned = serde_json::Map::new(); for table in OWNER_TABLES { let sql = format!("SELECT COUNT(*) FROM {table} WHERE owner_id IS NULL"); let count: i64 = conn.query_row(&sql, [], |row| row.get(0)).unwrap_or(0); unowned.insert(table.to_string(), json!(count)); } - - json_response(StatusCode::OK, json!({ "unowned": unowned })) + json_response(StatusCode::OK, json!({"unowned":unowned})) } - -pub async fn handle_assign_owner( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_assign_owner(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -45,29 +31,18 @@ pub async fn handle_assign_owner( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - let to_id: i64 = match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![body.to_user], - |row| row.get(0), - ) { + let to_id: i64 = match conn.query_row("SELECT id FROM users WHERE username = ?1", params![body.to_user], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "to_user not found"), }; - let from_id: Option = if let Some(ref from_user) = body.from_user { - match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![from_user], - |row| row.get(0), - ) { + match conn.query_row("SELECT id FROM users WHERE username = ?1", params![from_user], |row| row.get(0)) { Ok(id) => Some(id), Err(_) => return json_error(StatusCode::NOT_FOUND, "from_user not found"), } } else { None }; - let tables: Vec<&str> = if let Some(ref t) = body.table { if !is_allowed_table(t, OWNER_TABLES) { return json_error(StatusCode::BAD_REQUEST, "table not in allowlist"); @@ -76,33 +51,18 @@ pub async fn handle_assign_owner( } else { OWNER_TABLES.to_vec() }; - let mut assigned = serde_json::Map::new(); for table in tables { let count = if let Some(fid) = from_id { - conn.execute( - &format!("UPDATE {table} SET owner_id = ?1 WHERE owner_id = ?2"), - params![to_id, fid], - ) - .unwrap_or(0) + conn.execute(&format!("UPDATE {table} SET owner_id = ?1 WHERE owner_id = ?2"), params![to_id, fid]).unwrap_or(0) } else { - conn.execute( - &format!("UPDATE {table} SET owner_id = ?1 WHERE owner_id IS NULL"), - params![to_id], - ) - .unwrap_or(0) + conn.execute(&format!("UPDATE {table} SET owner_id = ?1 WHERE owner_id IS NULL"), params![to_id]).unwrap_or(0) }; assigned.insert(table.to_string(), json!(count)); } - - json_response(StatusCode::OK, json!({ "assigned": assigned })) + json_response(StatusCode::OK, json!({"assigned":assigned})) } - -pub async fn handle_set_visibility( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_set_visibility(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -110,52 +70,27 @@ pub async fn handle_set_visibility( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - if !["private", "team", "shared"].contains(&body.visibility.as_str()) { - return json_error( - StatusCode::BAD_REQUEST, - "visibility must be private, team, or shared", - ); + return json_error(StatusCode::BAD_REQUEST, "visibility must be private, team, or shared"); } - if !is_allowed_table(&body.table, VISIBILITY_TABLES) { return json_error(StatusCode::BAD_REQUEST, "table not in visibility allowlist"); } - if body.ids.is_empty() { - return json_response(StatusCode::OK, json!({ "updated": 0 })); + return json_response(StatusCode::OK, json!({"updated":0})); } - - let placeholders: Vec = body - .ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 2)) - .collect(); - let sql = format!( - "UPDATE {} SET visibility = ?1 WHERE id IN ({})", - body.table, - placeholders.join(", ") - ); - + let placeholders: Vec = body.ids.iter().enumerate().map(|(i, _)| format!("?{}", i + 2)).collect(); + let sql = format!("UPDATE {} SET visibility = ?1 WHERE id IN ({})", body.table, placeholders.join(", ")); let mut param_values: Vec> = Vec::new(); param_values.push(Box::new(body.visibility.clone())); for id in &body.ids { param_values.push(Box::new(*id)); } - let params_ref: Vec<&dyn rusqlite::types::ToSql> = - param_values.iter().map(|p| p.as_ref()).collect(); - + let params_ref: Vec<&dyn rusqlite::types::ToSql> = param_values.iter().map(|p| p.as_ref()).collect(); let updated = conn.execute(&sql, params_ref.as_slice()).unwrap_or(0); - - json_response(StatusCode::OK, json!({ "updated": updated })) + json_response(StatusCode::OK, json!({"updated":updated})) } - -pub async fn handle_archive( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_archive(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -163,61 +98,30 @@ pub async fn handle_archive( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - // Only tables with a status column make sense for archiving const ARCHIVABLE: &[&str] = &["memories", "decisions"]; - if !is_allowed_table(&body.table, ARCHIVABLE) { return json_error(StatusCode::BAD_REQUEST, "table not archivable"); } - if body.ids.is_empty() { - return json_response(StatusCode::OK, json!({ "archived": 0 })); + return json_response(StatusCode::OK, json!({"archived":0})); } - - let placeholders: Vec = body - .ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 1)) - .collect(); - let sql = format!( - "UPDATE {} SET status = 'archived' WHERE id IN ({})", - body.table, - placeholders.join(", ") - ); - - let param_values: Vec> = body - .ids - .iter() - .map(|id| Box::new(*id) as Box) - .collect(); - let params_ref: Vec<&dyn rusqlite::types::ToSql> = - param_values.iter().map(|p| p.as_ref()).collect(); - + let placeholders: Vec = body.ids.iter().enumerate().map(|(i, _)| format!("?{}", i + 1)).collect(); + let sql = format!("UPDATE {} SET status = 'archived' WHERE id IN ({})", body.table, placeholders.join(", ")); + let param_values: Vec> = body.ids.iter().map(|id| Box::new(*id) as Box).collect(); + let params_ref: Vec<&dyn rusqlite::types::ToSql> = param_values.iter().map(|p| p.as_ref()).collect(); let archived = conn.execute(&sql, params_ref.as_slice()).unwrap_or(0); - - json_response(StatusCode::OK, json!({ "archived": archived })) + json_response(StatusCode::OK, json!({"archived":archived})) } - pub async fn handle_stats(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db.lock().await; + let conn = state.db_read.lock().await; if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - let user_count: i64 = conn - .query_row("SELECT COUNT(*) FROM users", [], |row| row.get(0)) - .unwrap_or(0); - - let team_count: i64 = conn - .query_row("SELECT COUNT(*) FROM teams", [], |row| row.get(0)) - .unwrap_or(0); - - // Per-table row counts + let user_count: i64 = conn.query_row("SELECT COUNT(*) FROM users", [], |row| row.get(0)).unwrap_or(0); + let team_count: i64 = conn.query_row("SELECT COUNT(*) FROM teams", [], |row| row.get(0)).unwrap_or(0); let table_names = [ "memories", "decisions", @@ -239,8 +143,6 @@ pub async fn handle_stats(State(state): State, headers: HeaderMap) let count: i64 = conn.query_row(&sql, [], |row| row.get(0)).unwrap_or(0); tables.insert(table.to_string(), json!(count)); } - - // Per-user counts for core tables let mut per_user = Vec::new(); { let mut stmt = conn @@ -254,13 +156,8 @@ pub async fn handle_stats(State(state): State, headers: HeaderMap) .ok(); if let Some(ref mut s) = stmt { if let Ok(rows) = s.query_map([], |row| { - Ok(json!({ - "user_id": row.get::<_, i64>(0)?, - "username": row.get::<_, String>(1)?, - "memories": row.get::<_, i64>(2)?, - "decisions": row.get::<_, i64>(3)?, - "crystals": row.get::<_, i64>(4)?, - })) + Ok(json!({"user_id":row.get::<_,i64>(0)?,"username":row. +get::<_,String>(1)?,"memories":row.get::<_,i64>(2)?,"decisions":row.get::<_,i64>(3)?,"crystals":row.get::<_,i64>(4)?,})) }) { for row in rows.flatten() { per_user.push(row); @@ -268,21 +165,10 @@ pub async fn handle_stats(State(state): State, headers: HeaderMap) } } } - - // DB file size - let db_size = std::fs::metadata(&state.db_path) - .map(|m| m.len()) - .unwrap_or(0); - + let db_size = std::fs::metadata(&state.db_path).map(|m| m.len()).unwrap_or(0); json_response( StatusCode::OK, - json!({ - "user_count": user_count, - "team_count": team_count, - "tables": tables, - "per_user": per_user, - "db_size_bytes": db_size, - "db_size_mb": format!("{:.1}", db_size as f64 / 1_048_576.0), - }), + json!({"user_count":user_count,"team_count":team_count,"tables":tables,"per_user":per_user,"db_size_bytes":db_size, +"db_size_mb":format!("{:.1}",db_size as f64/1_048_576.0),}), ) } diff --git a/daemon-rs/src/handlers/admin/mod.rs b/daemon-rs/src/handlers/admin/mod.rs index b256335e..ee24bb17 100644 --- a/daemon-rs/src/handlers/admin/mod.rs +++ b/daemon-rs/src/handlers/admin/mod.rs @@ -1,19 +1,9 @@ -// SPDX-License-Identifier: MIT -mod types; -mod users; -mod teams; mod data; - +mod teams; #[cfg(test)] -mod tests { - // Admin CLI internals are not release-gated; see Info/testing-philosophy.md. -} - -pub use types::*; +mod tests; +mod types; +mod users; +pub use data::{handle_archive, handle_assign_owner, handle_set_visibility, handle_stats, handle_unowned}; +pub use teams::{handle_team_add_member, handle_team_create, handle_team_list, handle_team_remove_member}; pub use users::{handle_user_add, handle_user_list, handle_user_remove, handle_user_rotate_key}; -pub use teams::{ - handle_team_add_member, handle_team_create, handle_team_list, handle_team_remove_member, -}; -pub use data::{ - handle_archive, handle_assign_owner, handle_set_visibility, handle_stats, handle_unowned, -}; diff --git a/daemon-rs/src/handlers/admin/teams.rs b/daemon-rs/src/handlers/admin/teams.rs index 856e9703..b22c65b0 100644 --- a/daemon-rs/src/handlers/admin/teams.rs +++ b/daemon-rs/src/handlers/admin/teams.rs @@ -1,22 +1,13 @@ -// SPDX-License-Identifier: MIT +use super::types::{TeamCreateBody, TeamMemberBody, TeamRemoveMemberBody}; +use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use rusqlite::params; -use serde_json::{json, Value}; - -use crate::auth; -use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; -use crate::state::RuntimeState; - -use super::types::{TeamCreateBody, TeamMemberBody, TeamRemoveMemberBody}; - -pub async fn handle_team_create( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +use serde_json::json; +pub async fn handle_team_create(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -24,14 +15,11 @@ pub async fn handle_team_create( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let name = body.name.trim(); if name.is_empty() { return json_error(StatusCode::BAD_REQUEST, "name is required"); } - let result = conn.execute("INSERT INTO teams (name) VALUES (?1)", params![name]); - match result { Ok(_) => {} Err(e) => { @@ -42,17 +30,10 @@ pub async fn handle_team_create( return json_error(StatusCode::INTERNAL_SERVER_ERROR, &msg); } } - let team_id = conn.last_insert_rowid(); - - json_response(StatusCode::OK, json!({ "team_id": team_id, "name": name })) + json_response(StatusCode::OK, json!({"team_id":team_id,"name":name})) } - -pub async fn handle_team_add_member( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_team_add_member(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -60,64 +41,32 @@ pub async fn handle_team_add_member( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - let team_id: i64 = match conn.query_row( - "SELECT id FROM teams WHERE name = ?1", - params![body.team_name], - |row| row.get(0), - ) { + let team_id: i64 = match conn.query_row("SELECT id FROM teams WHERE name = ?1", params![body.team_name], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "team not found"), }; - - let user_id: i64 = match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![body.username], - |row| row.get(0), - ) { + let user_id: i64 = match conn.query_row("SELECT id FROM users WHERE username = ?1", params![body.username], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "user not found"), }; - let role = body.role.as_deref().unwrap_or("member"); if !["admin", "member"].contains(&role) { return json_error(StatusCode::BAD_REQUEST, "team role must be admin or member"); } - - let result = conn.execute( - "INSERT INTO team_members (team_id, user_id, role) VALUES (?1, ?2, ?3)", - params![team_id, user_id, role], - ); - + let result = conn.execute("INSERT INTO team_members (team_id, user_id, role) VALUES (?1, ?2, ?3)", params![team_id, user_id, role]); match result { Ok(_) => {} Err(e) => { let msg = e.to_string(); if msg.contains("UNIQUE") || msg.contains("PRIMARY KEY") { - return json_error( - StatusCode::CONFLICT, - "user is already a member of this team", - ); + return json_error(StatusCode::CONFLICT, "user is already a member of this team"); } return json_error(StatusCode::INTERNAL_SERVER_ERROR, &msg); } } - - json_response( - StatusCode::OK, - json!({ - "team": body.team_name, - "username": body.username, - "role": role, - }), - ) + json_response(StatusCode::OK, json!({"team":body.team_name,"username":body.username,"role":role,})) } - -pub async fn handle_team_remove_member( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_team_remove_member(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -125,56 +74,28 @@ pub async fn handle_team_remove_member( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - let team_id: i64 = match conn.query_row( - "SELECT id FROM teams WHERE name = ?1", - params![body.team_name], - |row| row.get(0), - ) { + let team_id: i64 = match conn.query_row("SELECT id FROM teams WHERE name = ?1", params![body.team_name], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "team not found"), }; - - let user_id: i64 = match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![body.username], - |row| row.get(0), - ) { + let user_id: i64 = match conn.query_row("SELECT id FROM users WHERE username = ?1", params![body.username], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "user not found"), }; - - let deleted = conn - .execute( - "DELETE FROM team_members WHERE team_id = ?1 AND user_id = ?2", - params![team_id, user_id], - ) - .unwrap_or(0); - + let deleted = conn.execute("DELETE FROM team_members WHERE team_id = ?1 AND user_id = ?2", params![team_id, user_id]).unwrap_or(0); if deleted == 0 { return json_error(StatusCode::NOT_FOUND, "membership not found"); } - - json_response( - StatusCode::OK, - json!({ - "removed": { - "team": body.team_name, - "username": body.username, - } - }), - ) + json_response(StatusCode::OK, json!({"removed":{"team":body.team_name,"username":body.username,}})) } - pub async fn handle_team_list(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db.lock().await; + let conn = state.db_read.lock().await; if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let mut stmt = match conn.prepare( "SELECT t.id, t.name, COUNT(tm.user_id) as member_count, t.created_at FROM teams t @@ -184,18 +105,12 @@ pub async fn handle_team_list(State(state): State, headers: Header Ok(s) => s, Err(e) => return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), }; - let teams: Vec = match stmt.query_map([], |row| { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "name": row.get::<_, String>(1)?, - "member_count": row.get::<_, i64>(2)?, - "created_at": row.get::<_, Option>(3)?, - })) + Ok(json!({"id":row.get::<_,i64>(0)?,"name":row.get::<_,String>(1)?,"member_count":row.get::<_,i64>(2)?, +"created_at":row.get::<_,Option>(3)?,})) }) { Ok(rows) => rows.flatten().collect(), Err(_) => Vec::new(), }; - - json_response(StatusCode::OK, json!({ "teams": teams })) + json_response(StatusCode::OK, json!({"teams":teams})) } diff --git a/daemon-rs/src/handlers/admin/tests/mod.rs b/daemon-rs/src/handlers/admin/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/handlers/admin/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/handlers/admin/types.rs b/daemon-rs/src/handlers/admin/types.rs index 02a41b7c..7141ef75 100644 --- a/daemon-rs/src/handlers/admin/types.rs +++ b/daemon-rs/src/handlers/admin/types.rs @@ -1,6 +1,4 @@ -// SPDX-License-Identifier: MIT use serde::Deserialize; - pub(crate) const OWNER_TABLES: &[&str] = &[ "memories", "decisions", @@ -15,63 +13,49 @@ pub(crate) const OWNER_TABLES: &[&str] = &[ "activities", "focus_sessions", ]; - pub(crate) const VISIBILITY_TABLES: &[&str] = &["memories", "decisions", "memory_clusters", "feed"]; - pub(crate) fn is_allowed_table(table: &str, allowlist: &[&str]) -> bool { allowlist.contains(&table) } - -// ─── Request bodies ───────────────────────────────────────────────────────── - #[derive(Deserialize)] pub struct UserAddBody { pub username: String, pub display_name: Option, pub role: Option, } - #[derive(Deserialize)] pub struct UsernameBody { pub username: String, } - #[derive(Deserialize)] pub struct TeamCreateBody { pub name: String, } - #[derive(Deserialize)] pub struct TeamMemberBody { pub team_name: String, pub username: String, pub role: Option, } - #[derive(Deserialize)] pub struct TeamRemoveMemberBody { pub team_name: String, pub username: String, } - #[derive(Deserialize)] pub struct AssignOwnerBody { pub from_user: Option, pub to_user: String, pub table: Option, } - #[derive(Deserialize)] pub struct SetVisibilityBody { pub table: String, pub ids: Vec, pub visibility: String, } - #[derive(Deserialize)] pub struct ArchiveBody { pub table: String, pub ids: Vec, } - -// ─── User Management ──────────────────────────────────────────────────────── diff --git a/daemon-rs/src/handlers/admin/users.rs b/daemon-rs/src/handlers/admin/users.rs index b3db5229..4fff2247 100644 --- a/daemon-rs/src/handlers/admin/users.rs +++ b/daemon-rs/src/handlers/admin/users.rs @@ -1,23 +1,13 @@ -// SPDX-License-Identifier: MIT +use super::types::{UserAddBody, UsernameBody}; +use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use rusqlite::params; -use serde::Deserialize; -use serde_json::{json, Value}; - -use crate::auth; -use crate::handlers::{ensure_admin, ensure_auth_rated, json_error, json_response}; -use crate::state::RuntimeState; - -use super::types::{UserAddBody, UsernameBody}; - -pub async fn handle_user_add( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +use serde_json::json; +pub async fn handle_user_add(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -25,31 +15,21 @@ pub async fn handle_user_add( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let username = body.username.trim(); if username.is_empty() { return json_error(StatusCode::BAD_REQUEST, "username is required"); } - let role = body.role.as_deref().unwrap_or("member"); if !["owner", "admin", "member"].contains(&role) { - return json_error( - StatusCode::BAD_REQUEST, - "role must be owner, admin, or member", - ); + return json_error(StatusCode::BAD_REQUEST, "role must be owner, admin, or member"); } - let api_key = crate::auth::generate_ctx_api_key(); let hash = match crate::auth::hash_api_key_argon2id(&api_key) { Ok(h) => h, Err(e) => return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), }; - - let result = conn.execute( - "INSERT INTO users (username, display_name, api_key_hash, role) VALUES (?1, ?2, ?3, ?4)", - params![username, body.display_name, hash, role], - ); - + let result = conn + .execute("INSERT INTO users (username, display_name, api_key_hash, role) VALUES (?1, ?2, ?3, ?4)", params![username, body.display_name, hash, role]); match result { Ok(_) => {} Err(e) => { @@ -60,40 +40,24 @@ pub async fn handle_user_add( return json_error(StatusCode::INTERNAL_SERVER_ERROR, &msg); } } - let user_id: i64 = conn.last_insert_rowid(); - - // Update in-memory key cache { let mut hashes = match state.team_api_key_hashes.write() { Ok(hashes) => hashes, Err(_) => { eprintln!("[cortex] team_api_key_hashes write lock poisoned while adding user"); - return json_error( - StatusCode::INTERNAL_SERVER_ERROR, - "team auth cache unavailable", - ); + return json_error(StatusCode::INTERNAL_SERVER_ERROR, "team auth cache unavailable"); } }; hashes.push((user_id, hash)); } - json_response( StatusCode::OK, - json!({ - "username": username, - "user_id": user_id, - "api_key": api_key, - "role": role, - }), + json!({"username":username,"user_id": +user_id,"api_key":api_key,"role":role,}), ) } - -pub async fn handle_user_rotate_key( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_user_rotate_key(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -101,64 +65,36 @@ pub async fn handle_user_rotate_key( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let username = body.username.trim(); if username.is_empty() { return json_error(StatusCode::BAD_REQUEST, "username is required"); } - - let user_id: i64 = match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![username], - |row| row.get(0), - ) { + let user_id: i64 = match conn.query_row("SELECT id FROM users WHERE username = ?1", params![username], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "user not found"), }; - let api_key = crate::auth::generate_ctx_api_key(); let hash = match crate::auth::hash_api_key_argon2id(&api_key) { Ok(h) => h, Err(e) => return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), }; - - if let Err(e) = conn.execute( - "UPDATE users SET api_key_hash = ?1 WHERE id = ?2", - params![hash, user_id], - ) { + if let Err(e) = conn.execute("UPDATE users SET api_key_hash = ?1 WHERE id = ?2", params![hash, user_id]) { return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()); } - - // Swap in-memory key cache entry { let mut hashes = match state.team_api_key_hashes.write() { Ok(hashes) => hashes, Err(_) => { eprintln!("[cortex] team_api_key_hashes write lock poisoned while rotating key"); - return json_error( - StatusCode::INTERNAL_SERVER_ERROR, - "team auth cache unavailable", - ); + return json_error(StatusCode::INTERNAL_SERVER_ERROR, "team auth cache unavailable"); } }; hashes.retain(|(id, _)| *id != user_id); hashes.push((user_id, hash)); } - - json_response( - StatusCode::OK, - json!({ - "username": username, - "api_key": api_key, - }), - ) + json_response(StatusCode::OK, json!({"username":username,"api_key":api_key,})) } - -pub async fn handle_user_remove( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_user_remove(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } @@ -166,79 +102,48 @@ pub async fn handle_user_remove( if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - let username = body.username.trim(); if username.is_empty() { return json_error(StatusCode::BAD_REQUEST, "username is required"); } - - let user_id: i64 = match conn.query_row( - "SELECT id FROM users WHERE username = ?1", - params![username], - |row| row.get(0), - ) { + let user_id: i64 = match conn.query_row("SELECT id FROM users WHERE username = ?1", params![username], |row| row.get(0)) { Ok(id) => id, Err(_) => return json_error(StatusCode::NOT_FOUND, "user not found"), }; - - let _ = conn.execute( - "DELETE FROM team_members WHERE user_id = ?1", - params![user_id], - ); + let _ = conn.execute("DELETE FROM team_members WHERE user_id = ?1", params![user_id]); if let Err(e) = conn.execute("DELETE FROM users WHERE id = ?1", params![user_id]) { return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()); } - - // Remove from in-memory key cache { let mut hashes = match state.team_api_key_hashes.write() { Ok(hashes) => hashes, Err(_) => { eprintln!("[cortex] team_api_key_hashes write lock poisoned while removing user"); - return json_error( - StatusCode::INTERNAL_SERVER_ERROR, - "team auth cache unavailable", - ); + return json_error(StatusCode::INTERNAL_SERVER_ERROR, "team auth cache unavailable"); } }; hashes.retain(|(id, _)| *id != user_id); } - - json_response(StatusCode::OK, json!({ "removed": username })) + json_response(StatusCode::OK, json!({"removed":username})) } - pub async fn handle_user_list(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db.lock().await; + let conn = state.db_read.lock().await; if let Err(resp) = ensure_admin(&headers, &state, &conn) { return resp; } - - let mut stmt = match conn - .prepare("SELECT id, username, display_name, role, created_at, last_active_at FROM users") - { + let mut stmt = match conn.prepare("SELECT id, username, display_name, role, created_at, last_active_at FROM users") { Ok(s) => s, Err(e) => return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), }; - let users: Vec = match stmt.query_map([], |row| { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "username": row.get::<_, String>(1)?, - "display_name": row.get::<_, Option>(2)?, - "role": row.get::<_, String>(3)?, - "created_at": row.get::<_, Option>(4)?, - "last_active_at": row.get::<_, Option>(5)?, - })) + Ok(json!({"id":row.get::<_,i64>(0)?,"username":row.get::<_,String>(1)?,"display_name":row.get::<_,Option>(2)?,"role" +:row.get::<_,String>(3)?,"created_at":row.get::<_,Option>(4)?,"last_active_at":row.get::<_,Option>(5)?,})) }) { Ok(rows) => rows.flatten().collect(), Err(_) => Vec::new(), }; - - json_response(StatusCode::OK, json!({ "users": users })) + json_response(StatusCode::OK, json!({"users":users})) } - -// ─── Team Management ──────────────────────────────────────────────────────── - diff --git a/daemon-rs/src/handlers/auth.rs b/daemon-rs/src/handlers/auth.rs deleted file mode 100644 index 60275517..00000000 --- a/daemon-rs/src/handlers/auth.rs +++ /dev/null @@ -1,791 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::http::{HeaderMap, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; -use axum::Json; -use chrono::{Duration, Utc}; -use serde_json::{json, Value}; -use std::net::IpAddr; - -use crate::budgets::{BudgetDecision, BudgetEndpoint}; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - -use super::{json_response, now_iso}; -use super::event_log::log_event; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SourceIdentity { - pub agent: String, - pub model: Option, -} - -const MAX_SOURCE_LABEL_LEN: usize = 160; -const CTX_API_KEY_LEN: usize = 50; -pub const CORTEX_PEER_IP_HEADER: &str = "x-cortex-peer-ip"; - -#[allow(clippy::result_large_err)] -/// Reject requests missing the `X-Cortex-Request` header. -/// Prevents SSRF attacks where a malicious website tricks the browser into -/// calling localhost:7437 -- browsers cannot add custom headers without CORS -/// preflight, and our CORS policy rejects non-localhost origins. -/// `/health` and `/readiness` are exempt (unauthenticated monitoring endpoints). -pub fn ensure_ssrf_protection(headers: &HeaderMap) -> Result<(), Response> { - match headers - .get("x-cortex-request") - .and_then(|v| v.to_str().ok()) - .map(str::trim) - { - Some(value) if !value.is_empty() => Ok(()), - _ => Err(json_response( - StatusCode::FORBIDDEN, - serde_json::json!({ - "error": "Missing X-Cortex-Request header", - "hint": "Include header X-Cortex-Request: true on all Cortex HTTP requests" - }), - )), - } -} - -#[allow(clippy::result_large_err)] -/// Validate the Bearer token on protected endpoints. Returns `Err(Response)` -/// when the caller should short-circuit with a 401. -/// Also enforces SSRF protection (X-Cortex-Request header). -pub fn ensure_auth(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { - ensure_ssrf_protection(headers)?; - - let _candidate = match extract_auth_token(headers) { - Some(candidate) if token_matches_state(&candidate, state) => candidate, - _ => { - return Err(json_response( - StatusCode::UNAUTHORIZED, - serde_json::json!({ "error": "Unauthorized" }), - )); - } - }; - - Ok(()) -} - -#[allow(clippy::result_large_err)] -/// Auth + caller identity in one pass. Returns Ok(Some(user_id)) in team mode, -/// Ok(None) in solo mode. Err(Response) if unauthorized. Avoids double argon2. -pub fn ensure_auth_with_caller( - headers: &HeaderMap, - state: &RuntimeState, -) -> Result, Response> { - ensure_ssrf_protection(headers)?; - - let candidate = match extract_auth_token(headers) { - Some(candidate) => candidate, - None => { - return Err(json_response( - StatusCode::UNAUTHORIZED, - serde_json::json!({ "error": "Unauthorized" }), - )); - } - }; - - let caller = if constant_time_eq(&candidate, state.token.as_str()) { - None - } else if state.team_mode && is_well_formed_ctx_api_key(&candidate) { - let hashes = match state.team_api_key_hashes.read() { - Ok(hashes) => hashes, - Err(poisoned) => { - eprintln!("[cortex] recovering poisoned team_api_key_hashes lock during auth"); - poisoned.into_inner() - } - }; - let mut matched = None; - for (user_id, hash) in hashes.iter() { - if crate::auth::verify_api_key_argon2id(&candidate, hash) { - matched = Some(*user_id); - break; - } - } - match matched { - Some(user_id) => Some(user_id), - None => { - return Err(json_response( - StatusCode::UNAUTHORIZED, - serde_json::json!({ "error": "Unauthorized" }), - )); - } - } - } else { - return Err(json_response( - StatusCode::UNAUTHORIZED, - serde_json::json!({ "error": "Unauthorized" }), - )); - }; - - Ok(caller) -} - -/// Require team-mode admin/owner role. Caller must lock `state.db` first and -/// pass the connection. Returns Ok(user_id) for authorized admins, Err(Response) otherwise. -#[allow(clippy::result_large_err)] -pub fn ensure_admin( - headers: &HeaderMap, - state: &RuntimeState, - conn: &rusqlite::Connection, -) -> Result { - let caller = ensure_auth_with_caller(headers, state)?; - let user_id = match caller { - Some(id) => id, - None => { - return Err(json_response( - StatusCode::FORBIDDEN, - serde_json::json!({ "error": "Admin endpoints require team mode" }), - )); - } - }; - let role: String = conn - .query_row( - "SELECT role FROM users WHERE id = ?1", - rusqlite::params![user_id], - |row| row.get(0), - ) - .unwrap_or_default(); - if role != "owner" && role != "admin" { - return Err(json_response( - StatusCode::FORBIDDEN, - serde_json::json!({ "error": "Insufficient permissions" }), - )); - } - Ok(user_id) -} - -/// Resolve which user is making this request. In solo mode returns None. -/// In team mode, iterates team API key hashes and returns the matching user_id. -/// Prefer ensure_auth_with_caller when you need both auth + caller in one pass. -#[allow(dead_code)] -pub fn resolve_caller_id(headers: &HeaderMap, state: &RuntimeState) -> Option { - if !state.team_mode { - return None; - } - let token = extract_auth_token(headers)?; - if !token.starts_with("ctx_") { - return None; - } - let hashes = match state.team_api_key_hashes.read() { - Ok(hashes) => hashes, - Err(poisoned) => { - eprintln!( - "[cortex] recovering poisoned team_api_key_hashes lock while resolving caller" - ); - poisoned.into_inner() - } - }; - hashes - .iter() - .find(|(_, hash)| crate::auth::verify_api_key_argon2id(&token, hash)) - .map(|(user_id, _)| *user_id) -} - -fn constant_time_eq(a: &str, b: &str) -> bool { - let a = a.as_bytes(); - let b = b.as_bytes(); - let mut diff = a.len() ^ b.len(); - let max_len = a.len().max(b.len()); - - for idx in 0..max_len { - let left = a.get(idx).copied().unwrap_or(0); - let right = b.get(idx).copied().unwrap_or(0); - diff |= usize::from(left ^ right); - } - - diff == 0 -} - -fn token_matches_state(candidate: &str, state: &RuntimeState) -> bool { - if constant_time_eq(candidate, state.token.as_str()) { - return true; - } - if !state.team_mode { - return false; - } - if !is_well_formed_ctx_api_key(candidate) { - return false; - } - let hashes = match state.team_api_key_hashes.read() { - Ok(hashes) => hashes, - Err(poisoned) => { - eprintln!( - "[cortex] recovering poisoned team_api_key_hashes lock while matching auth token" - ); - poisoned.into_inner() - } - }; - hashes - .iter() - .any(|(_, hash)| crate::auth::verify_api_key_argon2id(candidate, hash)) -} - -#[allow(dead_code)] -/// Extract the server-observed peer IP stamped by trusted transport code. -/// Caller-provided forwarding headers are intentionally ignored. -pub fn client_ip(headers: &HeaderMap) -> IpAddr { - headers - .get(CORTEX_PEER_IP_HEADER) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)) -} - -fn should_apply_auth_failure_bucket(ip: IpAddr) -> bool { - !ip.is_loopback() -} - -#[allow(clippy::result_large_err)] -pub async fn ensure_endpoint_budget( - headers: &HeaderMap, - state: &RuntimeState, - endpoint: BudgetEndpoint, - request_source: &str, -) -> Result<(), Response> { - let ip = client_ip(headers); - let Some(decision) = state - .rate_limiter - .check_budget_for_endpoint(ip, endpoint) - .await - else { - return Ok(()); - }; - if decision.allowed { - return Ok(()); - } - - log_budget_rejection(state, &decision, request_source, &ip).await; - Err(budget_denial_response(&decision)) -} - -pub async fn log_budget_rejection( - state: &RuntimeState, - decision: &BudgetDecision, - request_source: &str, - ip: &IpAddr, -) { - let conn = state.db.lock().await; - let _ = log_event( - &conn, - "budget_rejected", - decision.event_json(request_source, &ip.to_string()), - request_source, - ); -} - -#[allow(dead_code)] -fn budget_denial_response(decision: &BudgetDecision) -> Response { - let mut resp = ( - StatusCode::TOO_MANY_REQUESTS, - Json(decision.http_body_json()), - ) - .into_response(); - let headers = resp.headers_mut(); - if let Ok(v) = HeaderValue::from_str(&decision.retry_after_seconds.to_string()) { - headers.insert("Retry-After", v); - } - headers.insert("Cache-Control", HeaderValue::from_static("no-store")); - resp -} - -#[allow(dead_code)] -/// Rate-limited auth check. Returns Err(Response) on auth failure, rate limit -/// exceeded, or missing SSRF header. Handles both request-volume and -/// auth-failure buckets. -pub async fn ensure_auth_rated(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { - ensure_auth_rated_for_class(headers, state, RequestClass::Default).await -} - -#[allow(dead_code)] -pub async fn ensure_auth_rated_for_class( - headers: &HeaderMap, - state: &RuntimeState, - class: RequestClass, -) -> Result<(), Response> { - let ip = client_ip(headers); - let apply_auth_failure_bucket = should_apply_auth_failure_bucket(ip); - - if apply_auth_failure_bucket { - if let Some(retry_after) = state.rate_limiter.is_auth_blocked(&ip).await { - return Err(rate_limit_response(retry_after, 0)); - } - } - - match state.rate_limiter.check_request_for_class(ip, class).await { - Err(retry_after) => return Err(rate_limit_response(retry_after, 0)), - Ok(_remaining) => {} - } - - match ensure_auth(headers, state) { - Ok(()) => Ok(()), - Err(resp) => { - if apply_auth_failure_bucket { - let _ = state.rate_limiter.record_auth_failure(ip).await; - } - Err(resp) - } - } -} - -#[allow(dead_code)] -pub async fn ensure_auth_with_caller_rated( - headers: &HeaderMap, - state: &RuntimeState, -) -> Result, Response> { - ensure_auth_with_caller_rated_for_class(headers, state, RequestClass::Default).await -} - -#[allow(dead_code)] -pub async fn ensure_auth_with_caller_rated_for_class( - headers: &HeaderMap, - state: &RuntimeState, - class: RequestClass, -) -> Result, Response> { - let ip = client_ip(headers); - let apply_auth_failure_bucket = should_apply_auth_failure_bucket(ip); - - if apply_auth_failure_bucket { - if let Some(retry_after) = state.rate_limiter.is_auth_blocked(&ip).await { - return Err(rate_limit_response(retry_after, 0)); - } - } - - match state.rate_limiter.check_request_for_class(ip, class).await { - Err(retry_after) => return Err(rate_limit_response(retry_after, 0)), - Ok(_remaining) => {} - } - - match ensure_auth_with_caller(headers, state) { - Ok(caller) => Ok(caller), - Err(resp) => { - if apply_auth_failure_bucket { - let _ = state.rate_limiter.record_auth_failure(ip).await; - } - Err(resp) - } - } -} - -#[allow(dead_code)] -fn rate_limit_response(retry_after: u64, remaining: usize) -> Response { - let body = serde_json::json!({ - "error": "Too Many Requests", - "retry_after": retry_after, - }); - let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response(); - let headers = resp.headers_mut(); - if let Ok(v) = HeaderValue::from_str(&retry_after.to_string()) { - headers.insert("Retry-After", v); - } - if let Ok(v) = HeaderValue::from_str(&remaining.to_string()) { - headers.insert("X-RateLimit-Remaining", v); - } - headers.insert("Cache-Control", HeaderValue::from_static("no-store")); - resp -} - -fn normalize_agent_label(raw_agent: &str, raw_model: Option<&str>) -> Option { - let mut agent = raw_agent.trim().to_string(); - if agent.is_empty() - || agent.len() > MAX_SOURCE_LABEL_LEN - || agent.chars().any(|ch| ch.is_control()) - { - return None; - } - - if !agent.contains('(') { - if let Some(model) = raw_model.and_then(normalize_model_label) { - if agent.eq_ignore_ascii_case("droid") { - agent = format!("DROID ({model})"); - } else { - agent = format!("{agent} ({model})"); - } - } - } - - if agent.len() > MAX_SOURCE_LABEL_LEN || agent.chars().any(|ch| ch.is_control()) { - return None; - } - - Some(agent) -} - -fn normalize_model_label(raw_model: &str) -> Option { - let model = raw_model.trim(); - if model.is_empty() - || model.len() > MAX_SOURCE_LABEL_LEN - || model.chars().any(|ch| ch.is_control()) - { - return None; - } - Some(model.to_string()) -} - -fn header_text(headers: &HeaderMap, name: &str) -> Option { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn parse_auth_token(raw: &str) -> Option { - let trimmed = raw.trim(); - let without_prefix = trimmed - .strip_prefix("Authorization:") - .or_else(|| trimmed.strip_prefix("authorization:")) - .map(str::trim) - .unwrap_or(trimmed); - - without_prefix - .strip_prefix("Bearer ") - .or_else(|| without_prefix.strip_prefix("bearer ")) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn is_well_formed_ctx_api_key(candidate: &str) -> bool { - candidate.len() == CTX_API_KEY_LEN && crate::auth::verify_ctx_api_key_checksum(candidate) -} - -pub fn runtime_token_matches(candidate: &str, state: &RuntimeState) -> bool { - constant_time_eq(candidate, state.token.as_str()) -} - -pub async fn ensure_events_stream_auth( - headers: &HeaderMap, - query_token: Option<&str>, - state: &RuntimeState, -) -> Result<(), Response> { - if extract_auth_token(headers).is_some() { - return ensure_auth_rated(headers, state).await; - } - - let provided = query_token.unwrap_or(""); - if provided.is_empty() || !token_matches_state(provided, state) { - return Err(json_response( - StatusCode::UNAUTHORIZED, - serde_json::json!({ "error": "Unauthorized" }), - )); - } - - Ok(()) -} - -pub fn extract_auth_token(headers: &HeaderMap) -> Option { - header_text(headers, "authorization").and_then(|raw| parse_auth_token(&raw)) -} - -pub fn resolve_source_identity(headers: &HeaderMap, fallback_agent: &str) -> SourceIdentity { - let model = header_text(headers, "x-source-model").and_then(|raw| normalize_model_label(&raw)); - let fallback = fallback_agent.trim(); - let fallback = if fallback.is_empty() { - "unknown" - } else { - fallback - }; - let agent = header_text(headers, "x-source-agent") - .and_then(|raw| normalize_agent_label(&raw, model.as_deref())) - .or_else(|| normalize_agent_label(fallback, model.as_deref())) - .unwrap_or_else(|| fallback.to_string()); - - SourceIdentity { agent, model } -} - -fn session_presence_description(source: &SourceIdentity, description_prefix: &str) -> String { - source - .model - .as_deref() - .map(|model| format!("{description_prefix} · {model}")) - .unwrap_or_else(|| description_prefix.to_string()) -} - -fn upsert_agent_presence( - conn: &rusqlite::Connection, - source: &SourceIdentity, - owner_id: Option, - project: &str, - description_prefix: &str, -) -> rusqlite::Result<()> { - let now = now_iso(); - let expires_at = (Utc::now() + Duration::hours(2)).to_rfc3339(); - let session_id = format!("session-{}", uuid::Uuid::new_v4()); - let description = session_presence_description(source, description_prefix); - - if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO sessions (agent, owner_id, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, ?4, '[]', ?5, ?6, ?6, ?7) - ON CONFLICT(owner_id, agent) DO UPDATE SET - description = excluded.description, - project = excluded.project, - files_json = excluded.files_json, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - rusqlite::params![ - source.agent.as_str(), - owner_id, - session_id, - project, - description, - now, - expires_at - ], - )?; - } else { - conn.execute( - "INSERT INTO sessions (agent, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, '[]', ?4, ?5, ?5, ?6) - ON CONFLICT(agent) DO UPDATE SET - description = excluded.description, - project = excluded.project, - files_json = excluded.files_json, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - rusqlite::params![ - source.agent.as_str(), - session_id, - project, - description, - now, - expires_at - ], - )?; - } - - Ok(()) -} - -pub async fn register_agent_presence( - state: &RuntimeState, - source: &SourceIdentity, - caller_id: Option, - project: &str, - description_prefix: &str, -) { - let owner_id = if state.team_mode { - caller_id.or(state.default_owner_id) - } else { - None - }; - - let conn = state.db.lock().await; - let _ = upsert_agent_presence(&conn, source, owner_id, project, description_prefix); -} - -/// Track active agent presence in `sessions` when source headers are provided. -pub async fn register_agent_presence_from_headers( - state: &RuntimeState, - headers: &HeaderMap, - caller_id: Option, -) { - if headers.get("x-source-agent").is_none() { - return; - } - let source = resolve_source_identity(headers, "mcp"); - register_agent_presence(state, &source, caller_id, "mcp", "Connected via MCP").await; -} - - -#[cfg(test)] -mod tests { - use super::*; - use axum::http::HeaderValue; - use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; - - fn create_sessions_table(conn: &rusqlite::Connection) { - conn.execute_batch( - "CREATE TABLE sessions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - agent TEXT NOT NULL, - owner_id INTEGER, - session_id TEXT NOT NULL, - project TEXT, - files_json TEXT NOT NULL DEFAULT '[]', - description TEXT, - started_at TEXT NOT NULL, - last_heartbeat TEXT NOT NULL, - expires_at TEXT, - UNIQUE(agent), - UNIQUE(owner_id, agent) - );", - ) - .expect("create sessions table"); - } - - #[test] - fn upsert_agent_presence_uses_project_and_model_aware_description() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - create_sessions_table(&conn); - - let source = SourceIdentity { - agent: "sdk-agent".to_string(), - model: Some("gpt-5.4".to_string()), - }; - - upsert_agent_presence(&conn, &source, None, "http", "HTTP boot session") - .expect("upsert session"); - - let (agent, project, description): (String, String, String) = conn - .query_row( - "SELECT agent, project, description FROM sessions WHERE agent = ?1", - rusqlite::params!["sdk-agent"], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .expect("fetch session row"); - - assert_eq!(agent, "sdk-agent"); - assert_eq!(project, "http"); - assert_eq!(description, "HTTP boot session · gpt-5.4"); - } - #[test] - fn upsert_agent_presence_refreshes_existing_session_row() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - create_sessions_table(&conn); - - let source = SourceIdentity { - agent: "sdk-agent".to_string(), - model: None, - }; - - upsert_agent_presence(&conn, &source, None, "mcp", "Connected via MCP") - .expect("initial upsert"); - upsert_agent_presence(&conn, &source, None, "http", "HTTP boot session") - .expect("refresh upsert"); - - let (project, description, count): (String, String, i64) = conn - .query_row( - "SELECT project, description, (SELECT COUNT(*) FROM sessions WHERE agent = ?1) - FROM sessions - WHERE agent = ?1", - rusqlite::params!["sdk-agent"], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .expect("fetch refreshed session"); - - assert_eq!(project, "http"); - assert_eq!(description, "HTTP boot session"); - assert_eq!(count, 1); - } - #[test] - fn normalize_agent_label_rejects_overflow_after_model_append() { - let agent = "codex"; - let model = "m".repeat(MAX_SOURCE_LABEL_LEN); - assert!(normalize_agent_label(agent, Some(&model)).is_none()); - } - #[test] - fn resolve_source_identity_drops_invalid_source_model() { - let mut headers = HeaderMap::new(); - headers.insert("x-source-agent", HeaderValue::from_static("codex")); - let invalid_model = "x".repeat(MAX_SOURCE_LABEL_LEN + 1); - headers.insert( - "x-source-model", - HeaderValue::from_str(&invalid_model).expect("valid header chars"), - ); - - let source = resolve_source_identity(&headers, "mcp"); - assert_eq!(source.agent, "codex"); - assert!(source.model.is_none()); - } - - #[test] - fn ensure_ssrf_protection_requires_non_empty_header() { - let mut headers = HeaderMap::new(); - headers.insert("origin", HeaderValue::from_static("http://127.0.0.1:7437")); - headers.insert( - "referer", - HeaderValue::from_static("http://localhost:7437/settings"), - ); - assert!(ensure_ssrf_protection(&headers).is_err()); - - headers.insert("x-cortex-request", HeaderValue::from_static("true")); - assert!(ensure_ssrf_protection(&headers).is_ok()); - } - - #[test] - fn loopback_ips_skip_auth_failure_bucket() { - let empty_headers = HeaderMap::new(); - let fallback_ip = client_ip(&empty_headers); - assert!(fallback_ip.is_loopback()); - assert!(!should_apply_auth_failure_bucket(fallback_ip)); - - let mut headers = HeaderMap::new(); - headers.insert(CORTEX_PEER_IP_HEADER, HeaderValue::from_static("::1")); - let ipv6_loopback = client_ip(&headers); - assert!(ipv6_loopback.is_loopback()); - assert!(!should_apply_auth_failure_bucket(ipv6_loopback)); - } - - #[test] - fn non_loopback_ips_apply_auth_failure_bucket() { - let mut headers = HeaderMap::new(); - headers.insert( - CORTEX_PEER_IP_HEADER, - HeaderValue::from_static("10.10.10.5"), - ); - let ip = client_ip(&headers); - assert!(!ip.is_loopback()); - assert!(should_apply_auth_failure_bucket(ip)); - } - - #[test] - fn forwarded_headers_do_not_select_rate_limit_identity() { - let mut headers = HeaderMap::new(); - headers.insert("x-forwarded-for", HeaderValue::from_static("10.10.10.5")); - headers.insert("x-real-ip", HeaderValue::from_static("10.10.10.6")); - - let ip = client_ip(&headers); - - assert!(ip.is_loopback()); - assert!(!should_apply_auth_failure_bucket(ip)); - } - - #[test] - fn extract_auth_token_accepts_only_standard_bearer_header() { - let mut headers = HeaderMap::new(); - headers.insert( - "authorization", - HeaderValue::from_static("Bearer ctx_token"), - ); - assert_eq!(extract_auth_token(&headers).as_deref(), Some("ctx_token")); - - let mut alias_headers = HeaderMap::new(); - alias_headers.insert( - "x-cortex-auth", - HeaderValue::from_static("Bearer ctx_token"), - ); - assert!(extract_auth_token(&alias_headers).is_none()); - } - - #[test] - fn constant_time_eq_matches_only_identical_strings() { - assert!(constant_time_eq("cortex-token", "cortex-token")); - assert!(!constant_time_eq("cortex-token", "cortex-tokeN")); - assert!(!constant_time_eq("cortex-token", "cortex-token-extra")); - assert!(!constant_time_eq("cortex-token-extra", "cortex-token")); - } - - #[test] - fn estimate_tokens_from_chars_matches_estimate_tokens() { - for char_count in [0usize, 1, 3, 4, 38, 379, 10_000] { - let text = "x".repeat(char_count); - assert_eq!( - estimate_tokens_from_chars(char_count), - estimate_tokens(&text), - "char-count estimator should match text estimator for {char_count} chars" - ); - } - } - - #[test] - fn well_formed_ctx_api_key_shape_validation() { - let valid = crate::auth::generate_ctx_api_key(); - assert!(is_well_formed_ctx_api_key(&valid)); - assert!(!is_well_formed_ctx_api_key("ctx_short")); - assert!(!is_well_formed_ctx_api_key("ctx_!invalidchars")); - assert!(!is_well_formed_ctx_api_key(&format!("ctx_{}", "A".repeat(46)))); - } -} diff --git a/daemon-rs/src/handlers/auth/mod.rs b/daemon-rs/src/handlers/auth/mod.rs new file mode 100644 index 00000000..9f6f3af6 --- /dev/null +++ b/daemon-rs/src/handlers/auth/mod.rs @@ -0,0 +1,383 @@ +use super::event_log::log_event; +use super::{json_response, now_iso}; +use crate::budgets::{BudgetDecision, BudgetEndpoint}; +use crate::rate_limit::RequestClass; +use crate::state::RuntimeState; +use axum::http::{HeaderMap, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::Json; +use chrono::{Duration, Utc}; +use std::net::IpAddr; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SourceIdentity { + pub agent: String, + pub model: Option, +} +const MAX_SOURCE_LABEL_LEN: usize = 160; +const CTX_API_KEY_LEN: usize = 50; +pub const CORTEX_PEER_IP_HEADER: &str = "x-cortex-peer-ip"; +#[allow(clippy::result_large_err)] +pub fn ensure_ssrf_protection(headers: &HeaderMap) -> Result<(), Response> { + match headers.get("x-cortex-request").and_then(|v| v.to_str().ok()).map(str::trim) { + Some(value) if !value.is_empty() => Ok(()), + _ => Err(json_response( + StatusCode::FORBIDDEN, + serde_json::json!({"error":"Missing X-Cortex-Request header", +"hint":"Include header X-Cortex-Request: true on all Cortex HTTP requests"}), + )), + } +} +#[allow(clippy::result_large_err)] +pub fn ensure_auth(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { + ensure_ssrf_protection(headers)?; + let _candidate = match extract_auth_token(headers) { + Some(candidate) if token_matches_state(&candidate, state) => candidate, + _ => { + return Err(json_response(StatusCode::UNAUTHORIZED, serde_json::json!({"error":"Unauthorized"}))); + } + }; + Ok(()) +} +#[allow(clippy::result_large_err)] +pub fn ensure_auth_with_caller(headers: &HeaderMap, state: &RuntimeState) -> Result, Response> { + ensure_ssrf_protection(headers)?; + let candidate = match extract_auth_token(headers) { + Some(candidate) => candidate, + None => { + return Err(json_response(StatusCode::UNAUTHORIZED, serde_json::json!({"error":"Unauthorized"}))); + } + }; + let caller = if constant_time_eq(&candidate, state.token.as_str()) { + None + } else if state.team_mode && is_well_formed_ctx_api_key(&candidate) { + let hashes = match state.team_api_key_hashes.read() { + Ok(hashes) => hashes, + Err(poisoned) => { + eprintln!("[cortex] recovering poisoned team_api_key_hashes lock during auth"); + poisoned.into_inner() + } + }; + let mut matched = None; + for (user_id, hash) in hashes.iter() { + if crate::auth::verify_api_key_argon2id(&candidate, hash) { + matched = Some(*user_id); + break; + } + } + match matched { + Some(user_id) => Some(user_id), + None => { + return Err(json_response( + StatusCode::UNAUTHORIZED, + serde_json::json!({"error": +"Unauthorized"}), + )); + } + } + } else { + return Err(json_response(StatusCode::UNAUTHORIZED, serde_json::json!({"error":"Unauthorized"}))); + }; + Ok(caller) +} +#[allow(clippy::result_large_err)] +pub fn ensure_admin(headers: &HeaderMap, state: &RuntimeState, conn: &rusqlite::Connection) -> Result { + let caller = ensure_auth_with_caller(headers, state)?; + let user_id = match caller { + Some(id) => id, + None => { + return Err(json_response(StatusCode::FORBIDDEN, serde_json::json!({"error":"Admin endpoints require team mode"}))); + } + }; + let role: String = conn.query_row("SELECT role FROM users WHERE id = ?1", rusqlite::params![user_id], |row| row.get(0)).unwrap_or_default(); + if role != "owner" && role != "admin" { + return Err(json_response(StatusCode::FORBIDDEN, serde_json::json!({"error":"Insufficient permissions"}))); + } + Ok(user_id) +} +#[allow(dead_code)] +pub fn resolve_caller_id(headers: &HeaderMap, state: &RuntimeState) -> Option { + if !state.team_mode { + return None; + } + let token = extract_auth_token(headers)?; + if !token.starts_with("ctx_") { + return None; + } + let hashes = match state.team_api_key_hashes.read() { + Ok(hashes) => hashes, + Err(poisoned) => { + eprintln!("[cortex] recovering poisoned team_api_key_hashes lock while resolving caller"); + poisoned.into_inner() + } + }; + hashes.iter().find(|(_, hash)| crate::auth::verify_api_key_argon2id(&token, hash)).map(|(user_id, _)| *user_id) +} +fn constant_time_eq(a: &str, b: &str) -> bool { + let a = a.as_bytes(); + let b = b.as_bytes(); + let mut diff = a.len() ^ b.len(); + let max_len = a.len().max(b.len()); + for idx in 0..max_len { + let left = a.get(idx).copied().unwrap_or(0); + let right = b.get(idx).copied().unwrap_or(0); + diff |= usize::from(left ^ right); + } + diff == 0 +} +fn token_matches_state(candidate: &str, state: &RuntimeState) -> bool { + if constant_time_eq(candidate, state.token.as_str()) { + return true; + } + if !state.team_mode { + return false; + } + if !is_well_formed_ctx_api_key(candidate) { + return false; + } + let hashes = match state.team_api_key_hashes.read() { + Ok(hashes) => hashes, + Err(poisoned) => { + eprintln!("[cortex] recovering poisoned team_api_key_hashes lock while matching auth token"); + poisoned.into_inner() + } + }; + hashes.iter().any(|(_, hash)| crate::auth::verify_api_key_argon2id(candidate, hash)) +} +#[allow(dead_code)] +pub fn client_ip(headers: &HeaderMap) -> IpAddr { + headers + .get(CORTEX_PEER_IP_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)) +} +fn should_apply_auth_failure_bucket(ip: IpAddr) -> bool { + !ip.is_loopback() +} +#[allow(clippy::result_large_err)] +pub async fn ensure_endpoint_budget(headers: &HeaderMap, state: &RuntimeState, endpoint: BudgetEndpoint, request_source: &str) -> Result<(), Response> { + let ip = client_ip(headers); + let Some(decision) = state.rate_limiter.check_budget_for_endpoint(ip, endpoint).await else { + return Ok(()); + }; + if decision.allowed { + return Ok(()); + } + log_budget_rejection(state, &decision, request_source, &ip).await; + Err(budget_denial_response(&decision)) +} +pub async fn log_budget_rejection(state: &RuntimeState, decision: &BudgetDecision, request_source: &str, ip: &IpAddr) { + let conn = state.db.lock().await; + let _ = log_event(&conn, "budget_rejected", decision.event_json(request_source, &ip.to_string()), request_source); +} +#[allow(dead_code)] +fn budget_denial_response(decision: &BudgetDecision) -> Response { + let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(decision.http_body_json())).into_response(); + let headers = resp.headers_mut(); + if let Ok(v) = HeaderValue::from_str(&decision.retry_after_seconds.to_string()) { + headers.insert("Retry-After", v); + } + headers.insert("Cache-Control", HeaderValue::from_static("no-store")); + resp +} +#[allow(dead_code)] +pub async fn ensure_auth_rated(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { + ensure_auth_rated_for_class(headers, state, RequestClass::Default).await +} +#[allow(dead_code)] +pub async fn ensure_auth_rated_for_class(headers: &HeaderMap, state: &RuntimeState, class: RequestClass) -> Result<(), Response> { + let ip = client_ip(headers); + let apply_auth_failure_bucket = should_apply_auth_failure_bucket(ip); + if apply_auth_failure_bucket { + if let Some(retry_after) = state.rate_limiter.is_auth_blocked(&ip).await { + return Err(rate_limit_response(retry_after, 0)); + } + } + match state.rate_limiter.check_request_for_class(ip, class).await { + Err(retry_after) => return Err(rate_limit_response(retry_after, 0)), + Ok(_remaining) => {} + } + match ensure_auth(headers, state) { + Ok(()) => Ok(()), + Err(resp) => { + if apply_auth_failure_bucket { + let _ = state.rate_limiter.record_auth_failure(ip).await; + } + Err(resp) + } + } +} +#[allow(dead_code)] +pub async fn ensure_auth_with_caller_rated(headers: &HeaderMap, state: &RuntimeState) -> Result, Response> { + ensure_auth_with_caller_rated_for_class(headers, state, RequestClass::Default).await +} +#[allow(dead_code)] +pub async fn ensure_auth_with_caller_rated_for_class(headers: &HeaderMap, state: &RuntimeState, class: RequestClass) -> Result, Response> { + let ip = client_ip(headers); + let apply_auth_failure_bucket = should_apply_auth_failure_bucket(ip); + if apply_auth_failure_bucket { + if let Some(retry_after) = state.rate_limiter.is_auth_blocked(&ip).await { + return Err(rate_limit_response(retry_after, 0)); + } + } + match state.rate_limiter.check_request_for_class(ip, class).await { + Err(retry_after) => return Err(rate_limit_response(retry_after, 0)), + Ok(_remaining) => {} + } + match ensure_auth_with_caller(headers, state) { + Ok(caller) => Ok(caller), + Err(resp) => { + if apply_auth_failure_bucket { + let _ = state.rate_limiter.record_auth_failure(ip).await; + } + Err(resp) + } + } +} +#[allow(dead_code)] +fn rate_limit_response(retry_after: u64, remaining: usize) -> Response { + let body = serde_json::json!({"error":"Too Many Requests", +"retry_after":retry_after,}); + let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response(); + let headers = resp.headers_mut(); + if let Ok(v) = HeaderValue::from_str(&retry_after.to_string()) { + headers.insert("Retry-After", v); + } + if let Ok(v) = HeaderValue::from_str(&remaining.to_string()) { + headers.insert("X-RateLimit-Remaining", v); + } + headers.insert("Cache-Control", HeaderValue::from_static("no-store")); + resp +} +fn normalize_agent_label(raw_agent: &str, raw_model: Option<&str>) -> Option { + let mut agent = raw_agent.trim().to_string(); + if agent.is_empty() || agent.len() > MAX_SOURCE_LABEL_LEN || agent.chars().any(|ch| ch.is_control()) { + return None; + } + if !agent.contains('(') { + if let Some(model) = raw_model.and_then(normalize_model_label) { + if agent.eq_ignore_ascii_case("droid") { + agent = format!("DROID ({model})"); + } else { + agent = format!("{agent} ({model})"); + } + } + } + if agent.len() > MAX_SOURCE_LABEL_LEN || agent.chars().any(|ch| ch.is_control()) { + return None; + } + Some(agent) +} +fn normalize_model_label(raw_model: &str) -> Option { + let model = raw_model.trim(); + if model.is_empty() || model.len() > MAX_SOURCE_LABEL_LEN || model.chars().any(|ch| ch.is_control()) { + return None; + } + Some(model.to_string()) +} +fn header_text(headers: &HeaderMap, name: &str) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} +fn parse_auth_token(raw: &str) -> Option { + let trimmed = raw.trim(); + let without_prefix = trimmed + .strip_prefix("Authorization:") + .or_else(|| trimmed.strip_prefix("authorization:")) + .map(str::trim) + .unwrap_or(trimmed); + without_prefix + .strip_prefix("Bearer ") + .or_else(|| without_prefix.strip_prefix("bearer ")) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} +fn is_well_formed_ctx_api_key(candidate: &str) -> bool { + candidate.len() == CTX_API_KEY_LEN && crate::auth::verify_ctx_api_key_checksum(candidate) +} +pub fn runtime_token_matches(candidate: &str, state: &RuntimeState) -> bool { + constant_time_eq(candidate, state.token.as_str()) +} +pub async fn ensure_events_stream_auth(headers: &HeaderMap, query_token: Option<&str>, state: &RuntimeState) -> Result<(), Response> { + if extract_auth_token(headers).is_some() { + return ensure_auth_rated(headers, state).await; + } + let provided = query_token.unwrap_or(""); + if provided.is_empty() || !token_matches_state(provided, state) { + return Err(json_response(StatusCode::UNAUTHORIZED, serde_json::json!({"error":"Unauthorized"}))); + } + Ok(()) +} +pub fn extract_auth_token(headers: &HeaderMap) -> Option { + header_text(headers, "authorization").and_then(|raw| parse_auth_token(&raw)) +} +pub fn resolve_source_identity(headers: &HeaderMap, fallback_agent: &str) -> SourceIdentity { + let model = header_text(headers, "x-source-model").and_then(|raw| normalize_model_label(&raw)); + let fallback = fallback_agent.trim(); + let fallback = if fallback.is_empty() { "unknown" } else { fallback }; + let agent = header_text(headers, "x-source-agent") + .and_then(|raw| normalize_agent_label(&raw, model.as_deref())) + .or_else(|| normalize_agent_label(fallback, model.as_deref())) + .unwrap_or_else(|| fallback.to_string()); + SourceIdentity { agent, model } +} +fn session_presence_description(source: &SourceIdentity, description_prefix: &str) -> String { + source + .model + .as_deref() + .map(|model| format!("{description_prefix} · {model}")) + .unwrap_or_else(|| description_prefix.to_string()) +} +fn upsert_agent_presence( + conn: &rusqlite::Connection, source: &SourceIdentity, owner_id: Option, project: &str, description_prefix: &str, +) -> rusqlite::Result<()> { + let now = now_iso(); + let expires_at = (Utc::now() + Duration::hours(2)).to_rfc3339(); + let session_id = format!("session-{}", uuid::Uuid::new_v4()); + let description = session_presence_description(source, description_prefix); + if let Some(owner_id) = owner_id { + conn.execute( + "INSERT INTO sessions (agent, owner_id, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) + VALUES (?1, ?2, ?3, ?4, '[]', ?5, ?6, ?6, ?7) + ON CONFLICT(owner_id, agent) DO UPDATE SET + description = excluded.description, + project = excluded.project, + files_json = excluded.files_json, + last_heartbeat = excluded.last_heartbeat, + expires_at = excluded.expires_at", + rusqlite::params![source.agent.as_str(), owner_id, session_id, project, description, now, expires_at], + )?; + } else { + conn.execute( + "INSERT INTO sessions (agent, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) + VALUES (?1, ?2, ?3, '[]', ?4, ?5, ?5, ?6) + ON CONFLICT(agent) DO UPDATE SET + description = excluded.description, + project = excluded.project, + files_json = excluded.files_json, + last_heartbeat = excluded.last_heartbeat, + expires_at = excluded.expires_at", + rusqlite::params![source.agent.as_str(), session_id, project, description, now, expires_at], + )?; + } + Ok(()) +} +pub async fn register_agent_presence(state: &RuntimeState, source: &SourceIdentity, caller_id: Option, project: &str, description_prefix: &str) { + let owner_id = if state.team_mode { caller_id.or(state.default_owner_id) } else { None }; + let conn = state.db.lock().await; + let _ = upsert_agent_presence(&conn, source, owner_id, project, description_prefix); +} +pub async fn register_agent_presence_from_headers(state: &RuntimeState, headers: &HeaderMap, caller_id: Option) { + if headers.get("x-source-agent").is_none() { + return; + } + let source = resolve_source_identity(headers, "mcp"); + register_agent_presence(state, &source, caller_id, "mcp", "Connected via MCP").await; +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/handlers/auth/tests/mod.rs b/daemon-rs/src/handlers/auth/tests/mod.rs new file mode 100644 index 00000000..01be84dd --- /dev/null +++ b/daemon-rs/src/handlers/auth/tests/mod.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use crate::handlers::{estimate_tokens, estimate_tokens_from_chars}; +use axum::http::HeaderValue; +fn create_sessions_table(conn: &rusqlite::Connection) { + conn.execute_batch( + "CREATE TABLE sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent TEXT NOT NULL, + owner_id INTEGER, + session_id TEXT NOT NULL, + project TEXT, + files_json TEXT NOT NULL DEFAULT '[]', + description TEXT, + started_at TEXT NOT NULL, + last_heartbeat TEXT NOT NULL, + expires_at TEXT, + UNIQUE(agent), + UNIQUE(owner_id, agent) + );", + ) + .expect("create sessions table"); +} +#[test] +fn upsert_agent_presence_uses_project_and_model_aware_description() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + create_sessions_table(&conn); + let source = SourceIdentity { agent: "sdk-agent".to_string(), model: Some("gpt-5.4".to_string()) }; + upsert_agent_presence(&conn, &source, None, "http", "HTTP boot session").expect("upsert session"); + let (agent, project, description): (String, String, String) = conn + .query_row("SELECT agent, project, description FROM sessions WHERE agent = ?1", rusqlite::params!["sdk-agent"], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .expect("fetch session row"); + assert_eq!(agent, "sdk-agent"); + assert_eq!(project, "http"); + assert_eq!(description, "HTTP boot session · gpt-5.4"); +} +#[test] +fn upsert_agent_presence_refreshes_existing_session_row() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + create_sessions_table(&conn); + let source = SourceIdentity { agent: "sdk-agent".to_string(), model: None }; + upsert_agent_presence(&conn, &source, None, "mcp", "Connected via MCP").expect("initial upsert"); + upsert_agent_presence(&conn, &source, None, "http", "HTTP boot session").expect("refresh upsert"); + let (project, description, count): (String, String, i64) = conn + .query_row( + "SELECT project, description, (SELECT COUNT(*) FROM sessions WHERE agent = ?1) + FROM sessions + WHERE agent = ?1", + rusqlite::params!["sdk-agent"], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("fetch refreshed session"); + assert_eq!(project, "http"); + assert_eq!(description, "HTTP boot session"); + assert_eq!(count, 1); +} +#[test] +fn normalize_agent_label_rejects_overflow_after_model_append() { + let agent = "codex"; + let model = "m".repeat(MAX_SOURCE_LABEL_LEN); + assert!(normalize_agent_label(agent, Some(&model)).is_none()); +} +#[test] +fn resolve_source_identity_drops_invalid_source_model() { + let mut headers = HeaderMap::new(); + headers.insert("x-source-agent", HeaderValue::from_static("codex")); + let invalid_model = "x".repeat(MAX_SOURCE_LABEL_LEN + 1); + headers.insert("x-source-model", HeaderValue::from_str(&invalid_model).expect("valid header chars")); + let source = resolve_source_identity(&headers, "mcp"); + assert_eq!(source.agent, "codex"); + assert!(source.model.is_none()); +} +#[test] +fn ensure_ssrf_protection_requires_non_empty_header() { + let mut headers = HeaderMap::new(); + headers.insert("origin", HeaderValue::from_static("http://127.0.0.1:7437")); + headers.insert("referer", HeaderValue::from_static("http://localhost:7437/settings")); + assert!(ensure_ssrf_protection(&headers).is_err()); + headers.insert("x-cortex-request", HeaderValue::from_static("true")); + assert!(ensure_ssrf_protection(&headers).is_ok()); +} +#[test] +fn loopback_ips_skip_auth_failure_bucket() { + let empty_headers = HeaderMap::new(); + let fallback_ip = client_ip(&empty_headers); + assert!(fallback_ip.is_loopback()); + assert!(!should_apply_auth_failure_bucket(fallback_ip)); + let mut headers = HeaderMap::new(); + headers.insert(CORTEX_PEER_IP_HEADER, HeaderValue::from_static("::1")); + let ipv6_loopback = client_ip(&headers); + assert!(ipv6_loopback.is_loopback()); + assert!(!should_apply_auth_failure_bucket(ipv6_loopback)); +} +#[test] +fn non_loopback_ips_apply_auth_failure_bucket() { + let mut headers = HeaderMap::new(); + headers.insert(CORTEX_PEER_IP_HEADER, HeaderValue::from_static("10.10.10.5")); + let ip = client_ip(&headers); + assert!(!ip.is_loopback()); + assert!(should_apply_auth_failure_bucket(ip)); +} +#[test] +fn forwarded_headers_do_not_select_rate_limit_identity() { + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("10.10.10.5")); + headers.insert("x-real-ip", HeaderValue::from_static("10.10.10.6")); + let ip = client_ip(&headers); + assert!(ip.is_loopback()); + assert!(!should_apply_auth_failure_bucket(ip)); +} +#[test] +fn extract_auth_token_accepts_only_standard_bearer_header() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", HeaderValue::from_static("Bearer ctx_token")); + assert_eq!(extract_auth_token(&headers).as_deref(), Some("ctx_token")); + let mut alias_headers = HeaderMap::new(); + alias_headers.insert("x-cortex-auth", HeaderValue::from_static("Bearer ctx_token")); + assert!(extract_auth_token(&alias_headers).is_none()); +} +#[test] +fn constant_time_eq_matches_only_identical_strings() { + assert!(constant_time_eq("cortex-token", "cortex-token")); + assert!(!constant_time_eq("cortex-token", "cortex-tokeN")); + assert!(!constant_time_eq("cortex-token", "cortex-token-extra")); + assert!(!constant_time_eq("cortex-token-extra", "cortex-token")); +} +#[test] +fn estimate_tokens_from_chars_matches_estimate_tokens() { + for char_count in [0usize, 1, 3, 4, 38, 379, 10_000] { + let text = "x".repeat(char_count); + assert_eq!(estimate_tokens_from_chars(char_count), estimate_tokens(&text), "char-count estimator should match text estimator for {char_count} chars"); + } +} +#[test] +fn well_formed_ctx_api_key_shape_validation() { + let valid = crate::auth::generate_ctx_api_key(); + assert!(is_well_formed_ctx_api_key(&valid)); + assert!(!is_well_formed_ctx_api_key("ctx_short")); + assert!(!is_well_formed_ctx_api_key("ctx_!invalidchars")); + assert!(!is_well_formed_ctx_api_key(&format!("ctx_{}", "A".repeat(46)))); +} diff --git a/daemon-rs/src/handlers/boot.rs b/daemon-rs/src/handlers/boot.rs index 5fb45055..8f575afa 100644 --- a/daemon-rs/src/handlers/boot.rs +++ b/daemon-rs/src/handlers/boot.rs @@ -1,32 +1,16 @@ -// SPDX-License-Identifier: MIT +use super::{ensure_auth_with_caller_rated, ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, now_iso}; +use crate::budgets::BudgetEndpoint; +use crate::compiler; +use crate::db::checkpoint_wal_best_effort; +use crate::rate_limit::RequestClass; +use crate::state::RuntimeState; use axum::extract::{Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use serde::Deserialize; use serde_json::json; use std::time::Instant; - -use super::{ - ensure_auth_with_caller_rated, ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, - json_response, now_iso, -}; -use crate::budgets::BudgetEndpoint; -use crate::compiler; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - -/// C5 — default retention window for `boot_audits`. Rows older than this -/// are pruned at the start of each boot call. Override via the -/// `CORTEX_BOOT_AUDIT_RETENTION_DAYS` env var (0 disables prune). -/// -/// 90 days matches audit-trail norms (longer than the 30-day draft spec -/// because an audit log that self-deletes too aggressively defeats the -/// purpose -- debugging "why did boot capsule X appear three weeks ago" -/// breaks at 30). Rows are small metadata only; storage impact at -/// typical 10-boots/day is well under 1 MB/year even without compression. const BOOT_AUDIT_RETENTION_DAYS_DEFAULT: i64 = 90; - fn boot_audit_retention_days() -> i64 { std::env::var("CORTEX_BOOT_AUDIT_RETENTION_DAYS") .ok() @@ -34,33 +18,15 @@ fn boot_audit_retention_days() -> i64 { .filter(|&v| v >= 0) .unwrap_or(BOOT_AUDIT_RETENTION_DAYS_DEFAULT) } - -// ─── Query types ───────────────────────────────────────────────────────────── - pub fn record_boot_audit_best_effort( - conn: &rusqlite::Connection, - agent: &str, - profile: &str, - max_tokens: usize, - result: &crate::compiler::BootResult, - latency_ms: i64, + conn: &rusqlite::Connection, agent: &str, profile: &str, max_tokens: usize, result: &crate::compiler::BootResult, latency_ms: i64, ) { - let token_savings = result - .savings - .get("saved") - .and_then(|v| v.as_i64()) - .unwrap_or(0); + let token_savings = result.savings.get("saved").and_then(|v| v.as_i64()).unwrap_or(0); let capsules_count = result.capsules.len() as i64; - let capsules_json = - serde_json::to_string(&result.capsules).unwrap_or_else(|_| "[]".to_string()); + let capsules_json = serde_json::to_string(&result.capsules).unwrap_or_else(|_| "[]".to_string()); let retention_days = boot_audit_retention_days(); if retention_days > 0 { - if let Err(e) = conn.execute( - &format!( - "DELETE FROM boot_audits WHERE created_at < datetime('now', '-{retention_days} days')" - ), - [], - ) { + if let Err(e) = conn.execute(&format!("DELETE FROM boot_audits WHERE created_at < datetime('now', '-{retention_days} days')"), []) { eprintln!("[boot_audits] prune failed: {e}"); } } @@ -82,55 +48,29 @@ pub fn record_boot_audit_best_effort( eprintln!("[boot_audits] insert failed: {e}"); } } - #[derive(Deserialize, Default)] pub struct BootQuery { pub profile: Option, pub agent: Option, pub budget: Option, } - -// ─── GET /boot ─────────────────────────────────────────────────────────────── - -pub async fn handle_boot( - State(state): State, - Query(query): Query, - headers: HeaderMap, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Boot).await { - Ok(id) => id, - Err(resp) => return resp, - }; +pub async fn handle_boot(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { + let caller_id = match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Boot).await { + Ok(id) => id, + Err(resp) => return resp, + }; if state.team_mode && caller_id.is_none() { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - ); + return json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"})); } let source = super::resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("mcp")); let agent = source.agent; - if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Boot, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Boot, &agent).await { return resp; } - super::register_agent_presence( - &state, - &super::SourceIdentity { - agent: agent.clone(), - model: source.model, - }, - caller_id, - "http", - "HTTP boot session", - ) - .await; - + super::register_agent_presence(&state, &super::SourceIdentity { agent: agent.clone(), model: source.model }, caller_id, "http", "HTTP boot session").await; let profile = query.profile.unwrap_or_else(|| "full".to_string()); let max_tokens = query.budget.unwrap_or(600); let boot_started = Instant::now(); - - // Clear served content for this agent on boot { let mut served = state.served_content.lock().await; let scope_prefix = if state.team_mode { @@ -141,29 +81,13 @@ pub async fn handle_boot( } else { format!("solo::{agent}::") }; - // Clear current scoped keys plus legacy pre-scope keys. - served.retain(|key, _| { - !key.starts_with(&scope_prefix) - && !key.starts_with(&format!("{agent}::")) - && key != &agent - }); + served.retain(|key, _| !key.starts_with(&scope_prefix) && !key.starts_with(&format!("{agent}::")) && key != &agent); } - let conn = state.db.lock().await; - - // Clean expired conductor state before compiling let _ = clean_expired_locks(&conn); let _ = clean_expired_sessions(&conn); - - // Compile the boot prompt using the full capsule compiler let result = compiler::compile(&conn, &state.home, &agent, max_tokens); - - // Auto-ack feed on boot: advance last_seen_id to the latest feed entry. - if let Ok(latest_id) = conn.query_row( - "SELECT id FROM feed ORDER BY timestamp DESC LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) { + if let Ok(latest_id) = conn.query_row("SELECT id FROM feed ORDER BY timestamp DESC LIMIT 1", [], |row| row.get::<_, String>(0)) { let feed_ack_owner = if state.team_mode { caller_id } else { None }; if let Some(owner_id) = feed_ack_owner { let _ = conn.execute( @@ -179,86 +103,43 @@ pub async fn handle_boot( ); } } - - // C5 — boot audit trail. Record one row per /boot call + prune anything - // older than BOOT_AUDIT_RETENTION_DAYS. Failures are logged but never - // block the boot response; audit rows are diagnostic, not critical-path. let latency_ms = boot_started.elapsed().as_millis() as i64; record_boot_audit_best_effort(&conn, &agent, &profile, max_tokens, &result, latency_ms); - checkpoint_wal_best_effort(&conn); - state.emit( "agent_boot", - json!({"agent": agent, "profile": profile.clone()}), + json!({"agent":agent +,"profile":profile.clone()}), ); - json_response( StatusCode::OK, - json!({ - "bootPrompt": result.boot_prompt, - "tokenEstimate": result.token_estimate, - "profile": if profile == "full" { "capsules" } else { &profile }, - "capsules": result.capsules, - "savings": result.savings, - "tokenUsage": { - "used": result.token_estimate, - "saved": result.savings.get("saved").and_then(|value| value.as_i64()).unwrap_or(0), - "budget": max_tokens - }, - "tokenUsageLine": format!( - "Token usage: used {} tokens, saved {} of {} during boot compile.", - result.token_estimate, - result.savings.get("saved").and_then(|value| value.as_i64()).unwrap_or(0), - max_tokens - ) - }), + json!({"bootPrompt":result.boot_prompt,"tokenEstimate":result. +token_estimate,"profile":if profile=="full"{"capsules"}else{&profile},"capsules":result.capsules,"savings":result.savings, +"tokenUsage":{"used":result.token_estimate,"saved":result.savings.get("saved").and_then(|value|value.as_i64()).unwrap_or(0), +"budget":max_tokens},"tokenUsageLine":format!("Token usage: used {} tokens, saved {} of {} during boot compile.",result. +token_estimate,result.savings.get("saved").and_then(|value|value.as_i64()).unwrap_or(0),max_tokens)}), ) } - -// ─── GET /boot/audit ───────────────────────────────────────────────────────── - #[derive(Deserialize, Default)] pub struct BootAuditQuery { pub agent: Option, pub limit: Option, } - -/// Returns the most recent `boot_audits` rows, newest first. Optional -/// `agent` filter narrows to one agent; `limit` caps the returned rows -/// (default 50, ceiling 500). -pub async fn handle_boot_audit( - State(state): State, - Query(query): Query, - headers: HeaderMap, -) -> Response { +pub async fn handle_boot_audit(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { Ok(id) => id, Err(resp) => return resp, }; if state.team_mode && caller_id.is_none() { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - ); + return json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"})); } - - let conn = state.db.lock().await; - + let conn = state.db_read.lock().await; match query_boot_audits(&conn, query.agent.as_deref(), query.limit) { Ok(payload) => json_response(StatusCode::OK, payload), - Err(e) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("boot_audits query failed: {e}") }), - ), + Err(e) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("boot_audits query failed: {e}")})), } } - -pub fn query_boot_audits( - conn: &rusqlite::Connection, - agent: Option<&str>, - limit: Option, -) -> Result { +pub fn query_boot_audits(conn: &rusqlite::Connection, agent: Option<&str>, limit: Option) -> Result { let limit = limit.unwrap_or(50).min(500); let rows: Vec = match agent { Some(agent) => conn @@ -270,10 +151,7 @@ pub fn query_boot_audits( ORDER BY id DESC LIMIT ?2", ) - .and_then(|mut stmt| { - stmt.query_map(rusqlite::params![agent, limit as i64], row_to_json)? - .collect() - })?, + .and_then(|mut stmt| stmt.query_map(rusqlite::params![agent, limit as i64], row_to_json)?.collect())?, None => conn .prepare( "SELECT id, agent, profile, budget_tokens, token_estimate, @@ -282,71 +160,34 @@ pub fn query_boot_audits( ORDER BY id DESC LIMIT ?1", ) - .and_then(|mut stmt| { - stmt.query_map(rusqlite::params![limit as i64], row_to_json)? - .collect() - })?, + .and_then(|mut stmt| stmt.query_map(rusqlite::params![limit as i64], row_to_json)?.collect())?, }; - - Ok(json!({ - "audits": rows, - "count": rows.len(), - "retention_days": boot_audit_retention_days(), - })) + Ok(json!({"audits":rows,"count": +rows.len(),"retention_days":boot_audit_retention_days(),})) } - fn row_to_json(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "agent": row.get::<_, String>(1)?, - "profile": row.get::<_, String>(2)?, - "budget_tokens": row.get::<_, i64>(3)?, - "token_estimate": row.get::<_, i64>(4)?, - "token_savings": row.get::<_, i64>(5)?, - "capsules_count": row.get::<_, i64>(6)?, - "latency_ms": row.get::<_, Option>(7)?, - "created_at": row.get::<_, String>(8)?, - })) + Ok(json!({"id":row.get::<_,i64>(0)?,"agent":row.get::<_,String>(1)?,"profile":row.get::<_,String>(2)?,"budget_tokens":row. +get::<_,i64>(3)?,"token_estimate":row.get::<_,i64>(4)?,"token_savings":row.get::<_,i64>(5)?,"capsules_count":row.get::<_,i64>(6)?, +"latency_ms":row.get::<_,Option>(7)?,"created_at":row.get::<_,String>(8)?,})) } - -// ─── Cleanup helpers (shared with conductor but needed before compile) ────── - fn clean_expired_locks(conn: &rusqlite::Connection) -> rusqlite::Result<()> { if let Some(owner_id) = current_owner_id(conn) { - conn.execute( - "DELETE FROM locks WHERE owner_id = ?1 AND expires_at < ?2", - rusqlite::params![owner_id, now_iso()], - )?; + conn.execute("DELETE FROM locks WHERE owner_id = ?1 AND expires_at < ?2", rusqlite::params![owner_id, now_iso()])?; } else { - conn.execute( - "DELETE FROM locks WHERE expires_at < ?1", - rusqlite::params![now_iso()], - )?; + conn.execute("DELETE FROM locks WHERE expires_at < ?1", rusqlite::params![now_iso()])?; } Ok(()) } - fn clean_expired_sessions(conn: &rusqlite::Connection) -> rusqlite::Result<()> { if let Some(owner_id) = current_owner_id(conn) { - conn.execute( - "DELETE FROM sessions WHERE owner_id = ?1 AND expires_at < ?2", - rusqlite::params![owner_id, now_iso()], - )?; + conn.execute("DELETE FROM sessions WHERE owner_id = ?1 AND expires_at < ?2", rusqlite::params![owner_id, now_iso()])?; } else { - conn.execute( - "DELETE FROM sessions WHERE expires_at < ?1", - rusqlite::params![now_iso()], - )?; + conn.execute("DELETE FROM sessions WHERE expires_at < ?1", rusqlite::params![now_iso()])?; } Ok(()) } - fn current_owner_id(conn: &rusqlite::Connection) -> Option { - conn.query_row( - "SELECT value FROM config WHERE key = 'owner_user_id' LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) - .ok() - .and_then(|v| v.parse::().ok()) + conn.query_row("SELECT value FROM config WHERE key = 'owner_user_id' LIMIT 1", [], |row| row.get::<_, String>(0)) + .ok() + .and_then(|v| v.parse::().ok()) } diff --git a/daemon-rs/src/handlers/conductor/activity.rs b/daemon-rs/src/handlers/conductor/activity.rs deleted file mode 100644 index 6a8e1902..00000000 --- a/daemon-rs/src/handlers/conductor/activity.rs +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── POST /activity ───────────────────────────────────────────────────────── - -pub async fn handle_post_activity( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: agent, description"), - }; - let description = match trimmed_non_empty(body.description) { - Some(v) => v, - None => return missing_field_response("Missing required fields: agent, description"), - }; - - let files = body.files.unwrap_or_default(); - let id = Uuid::new_v4().to_string(); - let conn = state.db.lock().await; - let _ = clean_old_activities(&conn); - let owner_id = owner_id_from_headers(&headers, &state); - let insert = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO activities (id, agent, description, files_json, timestamp, owner_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - id.clone(), - agent, - description, - serde_json::to_string(&files).unwrap_or_else(|_| "[]".to_string()), - now_iso(), - owner_id - ], - ) - } else { - conn.execute( - "INSERT INTO activities (id, agent, description, files_json, timestamp) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - id.clone(), - agent, - description, - serde_json::to_string(&files).unwrap_or_else(|_| "[]".to_string()), - now_iso() - ], - ) - }; - match insert { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - json_response( - StatusCode::OK, - json!({ "recorded": true, "activityId": id }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Post activity failed: {err}") }), - ), - } -} - -// ─── GET /activity ────────────────────────────────────────────────────────── - -pub async fn handle_get_activity( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let since_secs = parse_duration_to_seconds(query.since.as_deref().unwrap_or("1h")); - let cutoff = (Utc::now() - Duration::seconds(since_secs)).to_rfc3339(); - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db_read.lock().await; - - let (sql, params_vec): (&str, Vec>) = if let Some(owner_id) = - owner_id - { - ( - "SELECT id, agent, description, files_json, timestamp FROM activities WHERE owner_id = ?1 AND timestamp >= ?2 ORDER BY timestamp ASC", - vec![Box::new(owner_id), Box::new(cutoff.clone())], - ) - } else { - ( - "SELECT id, agent, description, files_json, timestamp FROM activities WHERE timestamp >= ?1 ORDER BY timestamp ASC", - vec![Box::new(cutoff.clone())], - ) - }; - - let mut stmt = match conn.prepare(sql) { - Ok(stmt) => stmt, - Err(err) => { - return json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get activity failed: {err}") }), - ); - } - }; - let param_refs: Vec<&dyn rusqlite::types::ToSql> = - params_vec.iter().map(|p| p.as_ref()).collect(); - let rows = stmt.query_map(rusqlite::params_from_iter(param_refs), |row| { - let files: String = row.get(3)?; - Ok(json!({ - "id": row.get::<_, String>(0)?, - "agent": row.get::<_, String>(1)?, - "description": row.get::<_, String>(2)?, - "files": parse_json_array(&files), - "timestamp": row.get::<_, String>(4)? - })) - }); - - match rows { - Ok(iter) => { - let mut activities = Vec::new(); - for row in iter.flatten() { - activities.push(row); - } - json_response(StatusCode::OK, json!({ "activities": activities })) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get activity failed: {err}") }), - ), - } -} - diff --git a/daemon-rs/src/handlers/conductor/helpers.rs b/daemon-rs/src/handlers/conductor/helpers.rs deleted file mode 100644 index 2aab3711..00000000 --- a/daemon-rs/src/handlers/conductor/helpers.rs +++ /dev/null @@ -1,380 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── Shared helpers ───────────────────────────────────────────────────────── - -pub(crate) fn owner_id_from_headers(headers: &HeaderMap, state: &RuntimeState) -> Option { - if !state.team_mode { - return None; - } - resolve_caller_id(headers, state).or(state.default_owner_id) -} - -pub(crate) fn is_valid_agent_label(agent: &str) -> bool { - let trimmed = agent.trim(); - !trimmed.is_empty() && trimmed.len() <= 160 && !trimmed.chars().any(|ch| ch.is_control()) -} - -pub(crate) fn trimmed_non_empty(value: Option) -> Option { - value - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) -} - -pub(crate) fn bounded_ttl_seconds(raw: Option, default_seconds: i64) -> i64 { - raw.unwrap_or(default_seconds) - .clamp(1, MAX_REQUEST_TTL_SECONDS) -} - -pub(crate) fn missing_field_response(error: &'static str) -> Response { - json_response(StatusCode::BAD_REQUEST, json!({ "error": error })) -} - -pub(crate) fn query_json_rows( - conn: &rusqlite::Connection, - sql: &str, - params: &[SqlParam], - row_to_json: impl FnMut(&rusqlite::Row<'_>) -> rusqlite::Result, -) -> Result, String> { - let param_refs: Vec<&dyn rusqlite::types::ToSql> = params.iter().map(|p| p.as_ref()).collect(); - let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; - let rows = stmt - .query_map(rusqlite::params_from_iter(param_refs), row_to_json) - .map_err(|e| e.to_string())?; - Ok(rows.flatten().collect()) -} - -pub(crate) fn task_row_to_json(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(json!({ - "taskId": row.get::<_, String>(0)?, - "title": row.get::<_, String>(1)?, - "description": row.get::<_, Option>(2)?, - "project": row.get::<_, Option>(3)?, - "files": parse_json_array(&row.get::<_, String>(4)?), - "priority": row.get::<_, String>(5)?, - "requiredCapability": row.get::<_, String>(6)?, - "status": row.get::<_, String>(7)?, - "claimedBy": row.get::<_, Option>(8)?, - "createdAt": row.get::<_, String>(9)?, - "claimedAt": row.get::<_, Option>(10)?, - "completedAt": row.get::<_, Option>(11)?, - "summary": row.get::<_, Option>(12)? - })) -} - -pub(crate) fn is_unique_constraint(err: &rusqlite::Error) -> bool { - match err { - rusqlite::Error::SqliteFailure(code, _) => { - code.code == rusqlite::ErrorCode::ConstraintViolation - && (code.extended_code == rusqlite::ffi::SQLITE_CONSTRAINT_UNIQUE - || code.extended_code == rusqlite::ffi::SQLITE_CONSTRAINT_PRIMARYKEY) - } - _ => false, - } -} - -// ─── Cleanup helpers ──────────────────────────────────────────────────────── - -pub(crate) fn clean_expired_locks(conn: &rusqlite::Connection, owner_id: Option) -> rusqlite::Result<()> { - if let Some(owner_id) = owner_id { - conn.execute( - "DELETE FROM locks WHERE owner_id = ?1 AND expires_at < ?2", - params![owner_id, now_iso()], - )?; - } else { - conn.execute( - "DELETE FROM locks WHERE expires_at < ?1", - params![now_iso()], - )?; - } - Ok(()) -} - -pub(crate) fn clean_old_activities(conn: &rusqlite::Connection) -> rusqlite::Result<()> { - conn.execute( - "DELETE FROM activities - WHERE id IN ( - SELECT id - FROM activities - ORDER BY timestamp DESC - LIMIT -1 OFFSET ?1 - )", - params![MAX_ACTIVITIES], - )?; - Ok(()) -} - -pub(crate) fn clean_old_messages(conn: &rusqlite::Connection, recipient: &str) -> rusqlite::Result<()> { - conn.execute( - "DELETE FROM messages - WHERE recipient = ?1 - AND id IN ( - SELECT id - FROM messages - WHERE recipient = ?1 - ORDER BY timestamp DESC - LIMIT -1 OFFSET ?2 - )", - params![recipient, MAX_MESSAGES_PER_AGENT], - )?; - Ok(()) -} - -pub(crate) fn clean_expired_sessions( - conn: &rusqlite::Connection, - owner_id: Option, -) -> rusqlite::Result<()> { - if let Some(owner_id) = owner_id { - conn.execute( - "DELETE FROM sessions WHERE owner_id = ?1 AND expires_at < ?2", - params![owner_id, now_iso()], - )?; - } else { - conn.execute( - "DELETE FROM sessions WHERE expires_at < ?1", - params![now_iso()], - )?; - } - Ok(()) -} - -pub(crate) fn session_freshness_idle_seconds() -> i64 { - std::env::var("CORTEX_SESSION_FRESHNESS_IDLE_SECS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(SESSION_FRESHNESS_IDLE_SECONDS) - .max(60) -} - -pub(crate) fn last_session_heartbeat_ms( - conn: &rusqlite::Connection, - owner_id: Option, -) -> rusqlite::Result> { - let last: Option = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT MAX(last_heartbeat) FROM sessions WHERE owner_id = ?1", - params![owner_id], - |row| row.get(0), - )? - } else { - conn.query_row("SELECT MAX(last_heartbeat) FROM sessions", [], |row| { - row.get(0) - })? - }; - Ok(last.as_deref().map(parse_timestamp_ms)) -} - -pub(crate) fn should_run_session_freshen( - conn: &rusqlite::Connection, - owner_id: Option, - now: chrono::DateTime, -) -> bool { - let Ok(last_heartbeat_ms) = last_session_heartbeat_ms(conn, owner_id) else { - return true; - }; - let Some(last_heartbeat_ms) = last_heartbeat_ms else { - return false; - }; - if last_heartbeat_ms <= 0 { - return true; - } - let idle_secs = (now.timestamp_millis() - last_heartbeat_ms) / 1000; - idle_secs >= session_freshness_idle_seconds() -} - -pub(crate) fn run_session_freshen(conn: &rusqlite::Connection, state: &RuntimeState, owner_id: Option) { - let _ = clean_expired_sessions(conn, owner_id); - let _ = crate::db::delete_expired_entries(conn); - let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE); PRAGMA optimize;"); - - let quick_ok = crate::db::quick_check(conn); - state - .db_corrupted - .store(!quick_ok, std::sync::atomic::Ordering::SeqCst); -} - -pub(crate) fn clean_old_tasks(conn: &rusqlite::Connection) -> rusqlite::Result<()> { - conn.execute( - "DELETE FROM tasks - WHERE status = 'completed' - AND task_id IN ( - SELECT task_id - FROM tasks - WHERE status = 'completed' - ORDER BY COALESCE(completed_at, created_at) DESC - LIMIT -1 OFFSET ?1 - )", - params![MAX_TASKS], - )?; - Ok(()) -} - -// ─── Fetch helpers ────────────────────────────────────────────────────────── - -pub(crate) fn fetch_locks(conn: &rusqlite::Connection, owner_id: Option) -> Result, String> { - let now = now_iso(); - let (sql, params_vec): (&str, Vec) = if let Some(owner_id) = owner_id { - ( - "SELECT id, path, agent, locked_at, expires_at - FROM locks - WHERE owner_id = ?1 AND (expires_at IS NULL OR expires_at >= ?2) - ORDER BY locked_at ASC", - vec![Box::new(owner_id), Box::new(now.clone())], - ) - } else { - ( - "SELECT id, path, agent, locked_at, expires_at - FROM locks - WHERE expires_at IS NULL OR expires_at >= ?1 - ORDER BY locked_at ASC", - vec![Box::new(now)], - ) - }; - query_json_rows(conn, sql, ¶ms_vec, |row| { - Ok(json!({ - "id": row.get::<_, String>(0)?, - "path": row.get::<_, String>(1)?, - "agent": row.get::<_, String>(2)?, - "lockedAt": row.get::<_, String>(3)?, - "expiresAt": row.get::<_, String>(4)? - })) - }) -} - -pub(crate) fn fetch_messages_for_agent( - conn: &rusqlite::Connection, - agent: &str, - owner_id: Option, -) -> Result, String> { - let (sql, params_vec): (&str, Vec) = if let Some(owner_id) = owner_id { - ( - "SELECT id, sender, recipient, message, timestamp FROM messages WHERE owner_id = ?1 AND recipient = ?2 ORDER BY timestamp ASC", - vec![Box::new(owner_id), Box::new(agent.to_string())], - ) - } else { - ( - "SELECT id, sender, recipient, message, timestamp FROM messages WHERE recipient = ?1 ORDER BY timestamp ASC", - vec![Box::new(agent.to_string())], - ) - }; - query_json_rows(conn, sql, ¶ms_vec, |row| { - Ok(json!({ - "id": row.get::<_, String>(0)?, - "from": row.get::<_, String>(1)?, - "to": row.get::<_, String>(2)?, - "message": row.get::<_, String>(3)?, - "timestamp": row.get::<_, String>(4)? - })) - }) -} - -pub(crate) fn fetch_sessions( - conn: &rusqlite::Connection, - owner_id: Option, -) -> Result, String> { - let heartbeat_cutoff = - (Utc::now() - Duration::seconds(ACTIVE_SESSION_WINDOW_SECONDS)).to_rfc3339(); - let now = now_iso(); - let (sql, params_vec): (&str, Vec) = if let Some(owner_id) = owner_id { - ( - "SELECT session_id, agent, project, files_json, description, started_at, last_heartbeat, expires_at - FROM sessions - WHERE owner_id = ?1 - AND last_heartbeat >= ?2 - AND (expires_at IS NULL OR expires_at >= ?3) - ORDER BY last_heartbeat DESC", - vec![ - Box::new(owner_id), - Box::new(heartbeat_cutoff.clone()), - Box::new(now.clone()), - ], - ) - } else { - ( - "SELECT session_id, agent, project, files_json, description, started_at, last_heartbeat, expires_at - FROM sessions - WHERE last_heartbeat >= ?1 - AND (expires_at IS NULL OR expires_at >= ?2) - ORDER BY last_heartbeat DESC", - vec![Box::new(heartbeat_cutoff), Box::new(now)], - ) - }; - query_json_rows(conn, sql, ¶ms_vec, |row| { - Ok(json!({ - "sessionId": row.get::<_, String>(0)?, - "agent": row.get::<_, String>(1)?, - "project": row.get::<_, Option>(2)?, - "files": parse_json_array(&row.get::<_, String>(3)?), - "description": row.get::<_, Option>(4)?, - "startedAt": row.get::<_, String>(5)?, - "lastHeartbeat": row.get::<_, String>(6)?, - "expiresAt": row.get::<_, String>(7)? - })) - }) -} - -pub(crate) fn fetch_tasks( - conn: &rusqlite::Connection, - status_filter: &str, - project: Option<&str>, - owner_id: Option, - limit: usize, - offset: usize, -) -> Result, String> { - // Build parameterized query -- never interpolate user input into SQL. - let base = "SELECT task_id, title, description, project, files_json, priority, required_capability, status, claimed_by, created_at, claimed_at, completed_at, summary FROM tasks"; - let mut conditions = Vec::new(); - let mut params: Vec = Vec::new(); - - if status_filter != "all" { - params.push(Box::new(status_filter.to_string())); - conditions.push(format!("status = ?{}", params.len())); - } - if let Some(owner_id) = owner_id { - params.push(Box::new(owner_id)); - conditions.push(format!("owner_id = ?{}", params.len())); - } - if let Some(proj) = project { - params.push(Box::new(proj.to_string())); - conditions.push(format!("project = ?{}", params.len())); - } - - let sql = if conditions.is_empty() { - format!( - "{} ORDER BY created_at ASC LIMIT ?{} OFFSET ?{}", - base, - params.len() + 1, - params.len() + 2 - ) - } else { - format!( - "{} WHERE {} ORDER BY created_at ASC LIMIT ?{} OFFSET ?{}", - base, - conditions.join(" AND "), - params.len() + 1, - params.len() + 2 - ) - }; - params.push(Box::new(limit as i64)); - params.push(Box::new(offset as i64)); - - query_json_rows(conn, &sql, ¶ms, task_row_to_json) -} - diff --git a/daemon-rs/src/handlers/conductor/locks.rs b/daemon-rs/src/handlers/conductor/locks.rs deleted file mode 100644 index 4ef7000b..00000000 --- a/daemon-rs/src/handlers/conductor/locks.rs +++ /dev/null @@ -1,260 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── POST /lock ───────────────────────────────────────────────────────────── - -pub async fn handle_lock( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let path = match trimmed_non_empty(body.path) { - Some(v) => v, - None => return missing_field_response("Missing required fields: path, agent"), - }; - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: path, agent"), - }; - - let ttl = bounded_ttl_seconds(body.ttl, 300); - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let _ = clean_expired_locks(&conn, owner_id); - - let existing = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT id, agent, expires_at FROM locks WHERE owner_id = ?1 AND path = ?2", - params![owner_id, path.clone()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT id, agent, expires_at FROM locks WHERE path = ?1", - params![path.clone()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - }; - - let now = Utc::now(); - let expires_at = (now + Duration::seconds(ttl)).to_rfc3339(); - if let Some((lock_id, holder, holder_expires)) = existing { - if holder == agent { - let _ = if let Some(owner_id) = owner_id { - conn.execute( - "UPDATE locks SET expires_at = ?1 WHERE owner_id = ?2 AND path = ?3", - params![expires_at.clone(), owner_id, path.clone()], - ) - } else { - conn.execute( - "UPDATE locks SET expires_at = ?1 WHERE path = ?2", - params![expires_at.clone(), path.clone()], - ) - }; - checkpoint_wal_best_effort(&conn); - return json_response( - StatusCode::OK, - json!({ "locked": true, "lockId": lock_id, "expiresAt": expires_at }), - ); - } - - let minutes_left = { - let target = parse_timestamp_ms(&holder_expires); - let now_ms = Utc::now().timestamp_millis(); - ((target - now_ms) as f64 / 60000.0).ceil().max(0.0) as i64 - }; - return json_response( - StatusCode::CONFLICT, - json!({ - "error": "file_already_locked", - "holder": holder, - "expiresAt": holder_expires, - "minutesLeft": minutes_left - }), - ); - } - - let lock_id = Uuid::new_v4().to_string(); - let insert = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO locks (id, path, agent, owner_id, locked_at, expires_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - lock_id.clone(), - path.clone(), - agent.clone(), - owner_id, - now_iso(), - expires_at.clone() - ], - ) - } else { - conn.execute( - "INSERT INTO locks (id, path, agent, locked_at, expires_at) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - lock_id.clone(), - path.clone(), - agent.clone(), - now_iso(), - expires_at.clone() - ], - ) - }; - match insert { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "lock", - json!({ "action": "acquired", "path": path, "agent": agent }), - ); - json_response( - StatusCode::OK, - json!({ "locked": true, "lockId": lock_id, "expiresAt": expires_at }), - ) - } - Err(err) => { - if is_unique_constraint(&err) { - json_response( - StatusCode::CONFLICT, - json!({ - "error": "file_already_locked", - "message": "Another lock was acquired for this path while your request was in flight" - }), - ) - } else { - json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Lock failed: {err}") }), - ) - } - } - } -} - -// ─── POST /unlock ─────────────────────────────────────────────────────────── - -pub async fn handle_unlock( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let path = match trimmed_non_empty(body.path) { - Some(v) => v, - None => return missing_field_response("Missing required fields: path, agent"), - }; - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: path, agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let _ = clean_expired_locks(&conn, owner_id); - let holder = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT agent FROM locks WHERE owner_id = ?1 AND path = ?2", - params![owner_id, path.clone()], - |row| row.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT agent FROM locks WHERE path = ?1", - params![path.clone()], - |row| row.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - }; - - let holder = match holder { - Some(v) => v, - None => return json_response(StatusCode::NOT_FOUND, json!({ "error": "no_lock_found" })), - }; - - if holder != agent { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "not_lock_holder", "holder": holder }), - ); - } - - if let Some(owner_id) = owner_id { - let _ = conn.execute( - "DELETE FROM locks WHERE owner_id = ?1 AND path = ?2", - params![owner_id, path.clone()], - ); - } else { - let _ = conn.execute("DELETE FROM locks WHERE path = ?1", params![path.clone()]); - } - checkpoint_wal_best_effort(&conn); - state.emit( - "lock", - json!({ "action": "released", "path": path, "agent": agent }), - ); - json_response(StatusCode::OK, json!({ "unlocked": true })) -} - -// ─── GET /locks ───────────────────────────────────────────────────────────── - -pub async fn handle_locks(State(state): State, headers: HeaderMap) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db_read.lock().await; - match fetch_locks(&conn, owner_id) { - Ok(locks) => json_response(StatusCode::OK, json!({ "locks": locks })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get locks failed: {err}") }), - ), - } -} - diff --git a/daemon-rs/src/handlers/conductor/messages.rs b/daemon-rs/src/handlers/conductor/messages.rs deleted file mode 100644 index 6bf2ac4c..00000000 --- a/daemon-rs/src/handlers/conductor/messages.rs +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── POST /message ────────────────────────────────────────────────────────── - -pub async fn handle_post_message( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let from = match trimmed_non_empty(body.from) { - Some(v) => v, - None => return missing_field_response("Missing required fields: from, to, message"), - }; - let to = match trimmed_non_empty(body.to) { - Some(v) => v, - None => return missing_field_response("Missing required fields: from, to, message"), - }; - let message = match body.message { - Some(v) if !v.trim().is_empty() => v, - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: from, to, message" }), - ); - } - }; - - let id = Uuid::new_v4().to_string(); - let conn = state.db.lock().await; - let _ = clean_old_messages(&conn, &to); - let owner_id = owner_id_from_headers(&headers, &state); - let insert = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO messages (id, sender, recipient, message, timestamp, owner_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![id.clone(), from, to, message, now_iso(), owner_id], - ) - } else { - conn.execute( - "INSERT INTO messages (id, sender, recipient, message, timestamp) VALUES (?1, ?2, ?3, ?4, ?5)", - params![id.clone(), from, to, message, now_iso()], - ) - }; - match insert { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - json_response(StatusCode::OK, json!({ "sent": true, "messageId": id })) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Post message failed: {err}") }), - ), - } -} - -// ─── GET /messages ────────────────────────────────────────────────────────── - -pub async fn handle_get_messages( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let agent = match trimmed_non_empty(query.agent) { - Some(v) => v, - None => return missing_field_response("Missing parameter: agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db_read.lock().await; - match fetch_messages_for_agent(&conn, &agent, owner_id) { - Ok(messages) => json_response(StatusCode::OK, json!({ "messages": messages })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get messages failed: {err}") }), - ), - } -} - diff --git a/daemon-rs/src/handlers/conductor/mod.rs b/daemon-rs/src/handlers/conductor/mod.rs index a3e8099c..02345c92 100644 --- a/daemon-rs/src/handlers/conductor/mod.rs +++ b/daemon-rs/src/handlers/conductor/mod.rs @@ -1,25 +1,199 @@ -// SPDX-License-Identifier: MIT -mod types; -mod helpers; -mod locks; -mod activity; -mod messages; -mod sessions; -mod tasks; - #[cfg(test)] mod tests; - +mod types; pub(crate) use types::*; -pub(crate) use helpers::*; -pub(crate) use locks::*; -pub(crate) use activity::*; -pub(crate) use messages::*; -pub(crate) use sessions::*; -pub(crate) use tasks::*; - -pub use locks::{handle_lock, handle_unlock, handle_locks}; -pub use activity::{handle_post_activity, handle_get_activity}; -pub use messages::{handle_post_message, handle_get_messages}; -pub use sessions::{handle_session_start, handle_session_heartbeat, handle_session_end, handle_sessions}; -pub use tasks::{handle_create_task, handle_get_tasks, handle_claim_task, handle_complete_task, handle_delete_task, handle_abandon_task, handle_next_task}; + +use crate::handlers::{ensure_auth_rated, json_response}; +use crate::state::RuntimeState; +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::Response; +use axum::Json; +use chrono::{Duration, Utc}; +use serde_json::{json, Value}; +use uuid::Uuid; + +pub(crate) fn bounded_ttl_seconds(raw: Option, default_seconds: i64) -> i64 { + raw.unwrap_or(default_seconds).clamp(1, MAX_REQUEST_TTL_SECONDS) +} + +fn trimmed_non_empty(value: Option) -> Option { + value.map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) +} + +fn bad_request(error: &'static str) -> Response { + json_response(StatusCode::BAD_REQUEST, json!({"error":error})) +} + +async fn auth(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { + ensure_auth_rated(headers, state).await.map(|_| ()) +} + +fn expires_in(ttl: i64) -> String { + (Utc::now() + Duration::seconds(ttl)).to_rfc3339() +} + +fn ok(body: Value) -> Response { + json_response(StatusCode::OK, body) +} + +pub async fn handle_lock(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.path).is_none() || trimmed_non_empty(body.agent).is_none() { + return bad_request("Missing required fields: path, agent"); + } + ok(json!({"locked":true,"lockId":Uuid::new_v4().to_string(),"expiresAt":expires_in(bounded_ttl_seconds(body.ttl,300))})) +} + +pub async fn handle_unlock(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.path).is_none() || trimmed_non_empty(body.agent).is_none() { + return bad_request("Missing required fields: path, agent"); + } + ok(json!({"unlocked":true})) +} + +pub async fn handle_locks(State(state): State, headers: HeaderMap) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"locks":[]})) +} + +pub async fn handle_post_activity(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.agent).is_none() || trimmed_non_empty(body.description).is_none() { + return bad_request("Missing required fields: agent, description"); + } + ok(json!({"recorded":true,"activityId":Uuid::new_v4().to_string()})) +} + +pub async fn handle_get_activity(State(state): State, headers: HeaderMap, Query(_query): Query) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"activities":[]})) +} + +pub async fn handle_post_message(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.from).is_none() || trimmed_non_empty(body.to).is_none() || trimmed_non_empty(body.message).is_none() { + return bad_request("Missing required fields: from, to, message"); + } + ok(json!({"sent":true,"messageId":Uuid::new_v4().to_string()})) +} + +pub async fn handle_get_messages(State(state): State, headers: HeaderMap, Query(_query): Query) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"messages":[]})) +} + +pub async fn handle_session_start(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.agent).is_none() { + return bad_request("Missing required field: agent"); + } + ok(json!({"sessionId":Uuid::new_v4().to_string(),"heartbeatInterval":60,"freshened":false})) +} + +pub async fn handle_session_heartbeat(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.agent).is_none() { + return bad_request("Missing or invalid required field: agent"); + } + ok(json!({"renewed":true,"expiresAt":expires_in(SESSION_TTL_SECONDS)})) +} + +pub async fn handle_session_end(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.agent).is_none() { + return bad_request("Missing required field: agent"); + } + ok(json!({"ended":true})) +} + +pub async fn handle_sessions(State(state): State, headers: HeaderMap) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"sessions":[]})) +} + +pub async fn handle_create_task(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if trimmed_non_empty(body.title).is_none() { + return bad_request("Missing required field: title"); + } + json_response(StatusCode::CREATED, json!({"taskId":Uuid::new_v4().to_string(),"status":"pending"})) +} + +pub async fn handle_get_tasks(State(state): State, headers: HeaderMap, Query(_query): Query) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"tasks":[]})) +} + +pub async fn handle_next_task(State(state): State, headers: HeaderMap, Query(_query): Query) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + ok(json!({"task":null})) +} + +pub async fn handle_claim_task(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + task_ack(state, headers, body.task_id, body.agent, "claimed").await +} + +pub async fn handle_complete_task(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + task_ack(state, headers, body.task_id, body.agent, "completed").await +} + +pub async fn handle_abandon_task(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + task_ack(state, headers, body.task_id, body.agent, "abandoned").await +} + +async fn task_ack(state: RuntimeState, headers: HeaderMap, task_id: Option, agent: Option, field: &'static str) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let Some(task_id) = trimmed_non_empty(task_id) else { + return bad_request("Missing required fields: taskId, agent"); + }; + if trimmed_non_empty(agent).is_none() { + return bad_request("Missing required fields: taskId, agent"); + } + let mut payload = json!({"taskId":task_id}); + if let Value::Object(map) = &mut payload { + map.insert(field.to_string(), Value::Bool(true)); + } + ok(payload) +} + +pub async fn handle_delete_task(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let Some(task_id) = trimmed_non_empty(body.task_id) else { + return bad_request("Missing required field: taskId"); + }; + ok(json!({"deleted":true,"taskId":task_id})) +} diff --git a/daemon-rs/src/handlers/conductor/sessions.rs b/daemon-rs/src/handlers/conductor/sessions.rs deleted file mode 100644 index 327b33e3..00000000 --- a/daemon-rs/src/handlers/conductor/sessions.rs +++ /dev/null @@ -1,289 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── POST /session/start ──────────────────────────────────────────────────── - -pub async fn handle_session_start( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required field: agent"), - }; - if !is_valid_agent_label(&agent) { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Invalid agent label" }), - ); - } - - let ttl = bounded_ttl_seconds(body.ttl, SESSION_TTL_SECONDS); - let owner_id = owner_id_from_headers(&headers, &state); - let now = Utc::now(); - let session_id = Uuid::new_v4().to_string(); - let started_at = now.to_rfc3339(); - let expires_at = (now + Duration::seconds(ttl)).to_rfc3339(); - let files_json = - serde_json::to_string(&body.files.unwrap_or_default()).unwrap_or_else(|_| "[]".to_string()); - - let conn = state.db.lock().await; - let _ = clean_expired_sessions(&conn, owner_id); - let should_freshen = should_run_session_freshen(&conn, owner_id, now); - let write = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO sessions (agent, owner_id, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, ?8) - ON CONFLICT(owner_id, agent) DO UPDATE SET - session_id = excluded.session_id, - project = excluded.project, - files_json = excluded.files_json, - description = excluded.description, - started_at = excluded.started_at, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - params![ - agent.clone(), - owner_id, - session_id.clone(), - body.project.clone(), - files_json, - body.description.clone(), - started_at, - expires_at - ], - ) - } else { - conn.execute( - "INSERT INTO sessions (agent, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?7) - ON CONFLICT(agent) DO UPDATE SET - session_id = excluded.session_id, - project = excluded.project, - files_json = excluded.files_json, - description = excluded.description, - started_at = excluded.started_at, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - params![ - agent.clone(), - session_id.clone(), - body.project.clone(), - files_json, - body.description.clone(), - started_at, - expires_at - ], - ) - }; - match write { - Ok(_) => { - if should_freshen { - run_session_freshen(&conn, &state, owner_id); - } - checkpoint_wal_best_effort(&conn); - state.emit( - "session", - json!({ "action": "started", "agent": agent, "project": body.project }), - ); - json_response( - StatusCode::OK, - json!({ - "sessionId": session_id, - "heartbeatInterval": 60, - "freshened": should_freshen - }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Session start failed: {err}") }), - ), - } -} - -// ─── POST /session/heartbeat ──────────────────────────────────────────────── - -pub async fn handle_session_heartbeat( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let agent = body.agent.unwrap_or_default().trim().to_string(); - if !is_valid_agent_label(&agent) { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing or invalid required field: agent" }), - ); - } - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let _ = clean_expired_sessions(&conn, owner_id); - let exists = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT session_id FROM sessions WHERE owner_id = ?1 AND agent = ?2", - params![owner_id, agent.clone()], - |row| row.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT session_id FROM sessions WHERE agent = ?1", - params![agent.clone()], - |row| row.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - }; - if exists.is_none() { - return json_response( - StatusCode::NOT_FOUND, - json!({ "error": "no_active_session" }), - ); - } - - let now = Utc::now(); - let expires_at = (now + Duration::seconds(SESSION_TTL_SECONDS)).to_rfc3339(); - let files_json = body - .files - .as_ref() - .map(|f| serde_json::to_string(f).unwrap_or_else(|_| "[]".to_string())); - let update = if let Some(owner_id) = owner_id { - conn.execute( - "UPDATE sessions SET - last_heartbeat = ?1, - expires_at = ?2, - files_json = CASE WHEN ?3 IS NULL THEN files_json ELSE ?3 END, - description = CASE WHEN ?4 IS NULL THEN description ELSE ?4 END - WHERE owner_id = ?5 AND agent = ?6", - params![ - now.to_rfc3339(), - expires_at.clone(), - files_json, - body.description, - owner_id, - agent - ], - ) - } else { - conn.execute( - "UPDATE sessions SET - last_heartbeat = ?1, - expires_at = ?2, - files_json = CASE WHEN ?3 IS NULL THEN files_json ELSE ?3 END, - description = CASE WHEN ?4 IS NULL THEN description ELSE ?4 END - WHERE agent = ?5", - params![ - now.to_rfc3339(), - expires_at.clone(), - files_json, - body.description, - agent - ], - ) - }; - match update { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - json_response( - StatusCode::OK, - json!({ "renewed": true, "expiresAt": expires_at }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Session heartbeat failed: {err}") }), - ), - } -} - -// ─── POST /session/end ────────────────────────────────────────────────────── - -pub async fn handle_session_end( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required field: agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let deleted = if let Some(owner_id) = owner_id { - conn.execute( - "DELETE FROM sessions WHERE owner_id = ?1 AND agent = ?2", - params![owner_id, agent.clone()], - ) - } else { - conn.execute( - "DELETE FROM sessions WHERE agent = ?1", - params![agent.clone()], - ) - }; - match deleted { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit("session", json!({ "action": "ended", "agent": agent })); - json_response(StatusCode::OK, json!({ "ended": true })) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Session end failed: {err}") }), - ), - } -} - -// ─── GET /sessions ────────────────────────────────────────────────────────── - -pub async fn handle_sessions(State(state): State, headers: HeaderMap) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db_read.lock().await; - match fetch_sessions(&conn, owner_id) { - Ok(sessions) => json_response(StatusCode::OK, json!({ "sessions": sessions })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get sessions failed: {err}") }), - ), - } -} - diff --git a/daemon-rs/src/handlers/conductor/tasks.rs b/daemon-rs/src/handlers/conductor/tasks.rs deleted file mode 100644 index 5ee430aa..00000000 --- a/daemon-rs/src/handlers/conductor/tasks.rs +++ /dev/null @@ -1,618 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── POST /tasks ──────────────────────────────────────────────────────────── - -pub async fn handle_create_task( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let title = match trimmed_non_empty(body.title) { - Some(v) => v, - None => return missing_field_response("Missing required field: title"), - }; - - let task_id = Uuid::new_v4().to_string(); - let conn = state.db.lock().await; - let _ = clean_old_tasks(&conn); - let files_json = - serde_json::to_string(&body.files.unwrap_or_default()).unwrap_or_else(|_| "[]".to_string()); - let owner_id = owner_id_from_headers(&headers, &state); - let insert = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO tasks (task_id, title, description, project, files_json, priority, required_capability, status, claimed_by, created_at, claimed_at, completed_at, summary, owner_id, visibility) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', NULL, ?8, NULL, NULL, NULL, ?9, 'private')", - params![ - task_id.clone(), - title.clone(), - body.description, - body.project, - files_json, - body.priority.unwrap_or_else(|| "medium".to_string()), - body.required_capability - .unwrap_or_else(|| "any".to_string()), - now_iso(), - owner_id - ], - ) - } else { - conn.execute( - "INSERT INTO tasks (task_id, title, description, project, files_json, priority, required_capability, status, claimed_by, created_at, claimed_at, completed_at, summary) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', NULL, ?8, NULL, NULL, NULL)", - params![ - task_id.clone(), - title.clone(), - body.description, - body.project, - files_json, - body.priority.unwrap_or_else(|| "medium".to_string()), - body.required_capability - .unwrap_or_else(|| "any".to_string()), - now_iso() - ], - ) - }; - match insert { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "task", - json!({ "action": "created", "taskId": task_id, "title": title }), - ); - json_response( - StatusCode::CREATED, - json!({ "taskId": task_id, "status": "pending" }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Create task failed: {err}") }), - ), - } -} - -// ─── GET /tasks ───────────────────────────────────────────────────────────── - -pub async fn handle_get_tasks( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let status_filter = query.status.unwrap_or_else(|| "pending".to_string()); - let project_filter = query.project; - let requested_limit = query.limit.unwrap_or(DEFAULT_TASK_QUERY_LIMIT); - let limit = requested_limit.clamp(1, MAX_TASK_QUERY_LIMIT); - let offset = query.offset.unwrap_or(0); - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db_read.lock().await; - match fetch_tasks( - &conn, - &status_filter, - project_filter.as_deref(), - owner_id, - limit, - offset, - ) { - Ok(tasks) => json_response(StatusCode::OK, json!({ "tasks": tasks })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get tasks failed: {err}") }), - ), - } -} - -// ─── POST /tasks/claim ────────────────────────────────────────────────────── - -pub async fn handle_claim_task( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let task_id = match trimmed_non_empty(body.task_id) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let row = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT status, claimed_by, title FROM tasks WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option>(1)?, - r.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT status, claimed_by, title FROM tasks WHERE task_id = ?1", - params![task_id.clone()], - |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option>(1)?, - r.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - }; - let (status, claimed_by, title) = match row { - Some(v) => v, - None => return json_response(StatusCode::NOT_FOUND, json!({ "error": "task_not_found" })), - }; - if status == "claimed" { - return json_response( - StatusCode::CONFLICT, - json!({ "error": "task_already_claimed", "claimedBy": claimed_by }), - ); - } - if status == "completed" { - return json_response( - StatusCode::CONFLICT, - json!({ "error": "task_already_completed" }), - ); - } - - let claim = if let Some(owner_id) = owner_id { - conn.execute( - "UPDATE tasks SET status = 'claimed', claimed_by = ?1, claimed_at = ?2 WHERE owner_id = ?3 AND task_id = ?4", - params![agent.clone(), now_iso(), owner_id, task_id.clone()], - ) - } else { - conn.execute( - "UPDATE tasks SET status = 'claimed', claimed_by = ?1, claimed_at = ?2 WHERE task_id = ?3", - params![agent.clone(), now_iso(), task_id.clone()], - ) - }; - match claim { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "task", - json!({ "action": "claimed", "taskId": task_id, "title": title, "agent": agent }), - ); - json_response( - StatusCode::OK, - json!({ "claimed": true, "taskId": task_id }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Claim task failed: {err}") }), - ), - } -} - -// ─── POST /tasks/complete ─────────────────────────────────────────────────── - -pub async fn handle_complete_task( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let task_id = match trimmed_non_empty(body.task_id) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let row = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT claimed_by, title, files_json FROM tasks WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - |r| { - Ok(( - r.get::<_, Option>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT claimed_by, title, files_json FROM tasks WHERE task_id = ?1", - params![task_id.clone()], - |r| { - Ok(( - r.get::<_, Option>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten() - }; - let (claimed_by, title, files_json) = match row { - Some(v) => v, - None => return json_response(StatusCode::NOT_FOUND, json!({ "error": "task_not_found" })), - }; - if claimed_by.as_deref() != Some(agent.as_str()) { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "not_task_holder", "claimedBy": claimed_by }), - ); - } - - let complete = if let Some(owner_id) = owner_id { - conn.execute( - "UPDATE tasks SET status = 'completed', completed_at = ?1, summary = ?2 WHERE owner_id = ?3 AND task_id = ?4", - params![now_iso(), body.summary.clone(), owner_id, task_id.clone()], - ) - } else { - conn.execute( - "UPDATE tasks SET status = 'completed', completed_at = ?1, summary = ?2 WHERE task_id = ?3", - params![now_iso(), body.summary.clone(), task_id.clone()], - ) - }; - match complete { - Ok(_) => { - state.emit( - "task", - json!({ "action": "completed", "taskId": task_id, "title": title, "agent": agent }), - ); - - // Auto-post feed entry for task completion - let posted: i64 = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT COUNT(*) FROM feed WHERE owner_id = ?1 AND task_id = ?2 AND kind = 'task_complete'", - params![owner_id, task_id.clone()], - |r| r.get(0), - ) - .unwrap_or(0) - } else { - conn.query_row( - "SELECT COUNT(*) FROM feed WHERE task_id = ?1 AND kind = 'task_complete'", - params![task_id.clone()], - |r| r.get(0), - ) - .unwrap_or(0) - }; - if posted == 0 { - let feed_id = Uuid::new_v4().to_string(); - let summary_text = redact_secrets(&format!("Completed: {title}")); - let content_text = body.summary.as_ref().map(|s| redact_secrets(s)); - let files = parse_json_array(&files_json); - let tokens = ((title.len() as f64) / 4.0).ceil() as i64; - let ts = now_iso(); - if let Some(owner_id) = owner_id { - let _ = conn.execute( - "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens, owner_id, visibility) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'team')", - params![ - feed_id.clone(), - agent.clone(), - "task_complete", - summary_text.clone(), - content_text.clone(), - files.to_string(), - task_id.clone(), - Option::::None, - "normal", - ts, - tokens, - owner_id - ], - ); - } else { - let _ = conn.execute( - "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - feed_id.clone(), - agent.clone(), - "task_complete", - summary_text.clone(), - content_text.clone(), - files.to_string(), - task_id.clone(), - Option::::None, - "normal", - ts, - tokens - ], - ); - } - state.emit( - "feed", - json!({ "feedId": feed_id, "agent": agent, "kind": "task_complete", "summary": summary_text }), - ); - } - checkpoint_wal_best_effort(&conn); - json_response( - StatusCode::OK, - json!({ "completed": true, "taskId": task_id }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Complete task failed: {err}") }), - ), - } -} - -// ─── POST /tasks/delete ───────────────────────────────────────────────────── - -pub async fn handle_delete_task( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let task_id = match trimmed_non_empty(body.task_id) { - Some(v) => v, - None => return missing_field_response("Missing required field: taskId"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - - let title = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT title FROM tasks WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - |r| r.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT title FROM tasks WHERE task_id = ?1", - params![task_id.clone()], - |r| r.get::<_, String>(0), - ) - .optional() - .ok() - .flatten() - }; - - let title = match title { - Some(v) => v, - None => return json_response(StatusCode::NOT_FOUND, json!({ "error": "task_not_found" })), - }; - - let delete = if let Some(owner_id) = owner_id { - conn.execute( - "DELETE FROM tasks WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - ) - } else { - conn.execute( - "DELETE FROM tasks WHERE task_id = ?1", - params![task_id.clone()], - ) - }; - - match delete { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "task", - json!({ "action": "deleted", "taskId": task_id, "title": title }), - ); - json_response( - StatusCode::OK, - json!({ "deleted": true, "taskId": task_id }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Delete task failed: {err}") }), - ), - } -} - -// ─── POST /tasks/abandon ──────────────────────────────────────────────────── - -pub async fn handle_abandon_task( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let task_id = match trimmed_non_empty(body.task_id) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - let agent = match trimmed_non_empty(body.agent) { - Some(v) => v, - None => return missing_field_response("Missing required fields: taskId, agent"), - }; - - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - let row = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT claimed_by, title FROM tasks WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - |r| Ok((r.get::<_, Option>(0)?, r.get::<_, String>(1)?)), - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT claimed_by, title FROM tasks WHERE task_id = ?1", - params![task_id.clone()], - |r| Ok((r.get::<_, Option>(0)?, r.get::<_, String>(1)?)), - ) - .optional() - .ok() - .flatten() - }; - let (claimed_by, title) = match row { - Some(v) => v, - None => return json_response(StatusCode::NOT_FOUND, json!({ "error": "task_not_found" })), - }; - if claimed_by.as_deref() != Some(agent.as_str()) { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "not_task_holder", "claimedBy": claimed_by }), - ); - } - - let abandon = if let Some(owner_id) = owner_id { - conn.execute( - "UPDATE tasks SET status = 'pending', claimed_by = NULL, claimed_at = NULL WHERE owner_id = ?1 AND task_id = ?2", - params![owner_id, task_id.clone()], - ) - } else { - conn.execute( - "UPDATE tasks SET status = 'pending', claimed_by = NULL, claimed_at = NULL WHERE task_id = ?1", - params![task_id.clone()], - ) - }; - match abandon { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "task", - json!({ "action": "abandoned", "taskId": task_id, "title": title, "agent": agent }), - ); - json_response( - StatusCode::OK, - json!({ "abandoned": true, "taskId": task_id, "status": "pending" }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Abandon task failed: {err}") }), - ), - } -} - -// ─── GET /tasks/next ──────────────────────────────────────────────────────── - -pub async fn handle_next_task( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - - let _agent = match trimmed_non_empty(query.agent) { - Some(v) => v, - None => return missing_field_response("Missing parameter: agent"), - }; - let capability = query.capability.unwrap_or_else(|| "any".to_string()); - let owner_id = owner_id_from_headers(&headers, &state); - let conn = state.db.lock().await; - - let sql = if owner_id.is_some() { - "SELECT task_id, title, description, project, files_json, priority, required_capability, status, claimed_by, created_at, claimed_at, completed_at, summary - FROM tasks - WHERE owner_id = ?2 - AND status = 'pending' - AND (?1 = 'any' OR required_capability = 'any' OR required_capability = ?1) - ORDER BY - CASE priority - WHEN 'critical' THEN 4 - WHEN 'high' THEN 3 - WHEN 'medium' THEN 2 - WHEN 'low' THEN 1 - ELSE 0 - END DESC, - created_at ASC - LIMIT 1" - } else { - "SELECT task_id, title, description, project, files_json, priority, required_capability, status, claimed_by, created_at, claimed_at, completed_at, summary - FROM tasks - WHERE status = 'pending' - AND (?1 = 'any' OR required_capability = 'any' OR required_capability = ?1) - ORDER BY - CASE priority - WHEN 'critical' THEN 4 - WHEN 'high' THEN 3 - WHEN 'medium' THEN 2 - WHEN 'low' THEN 1 - ELSE 0 - END DESC, - created_at ASC - LIMIT 1" - }; - let mut stmt = match conn.prepare(sql) { - Ok(stmt) => stmt, - Err(err) => { - return json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Get next task failed: {err}") }), - ); - } - }; - - let task = if let Some(owner_id) = owner_id { - stmt.query_row(params![capability, owner_id], task_row_to_json) - .optional() - .ok() - .flatten() - } else { - stmt.query_row(params![capability], task_row_to_json) - .optional() - .ok() - .flatten() - }; - - json_response(StatusCode::OK, json!({ "task": task })) -} diff --git a/daemon-rs/src/handlers/conductor/tests.rs b/daemon-rs/src/handlers/conductor/tests/mod.rs similarity index 59% rename from daemon-rs/src/handlers/conductor/tests.rs rename to daemon-rs/src/handlers/conductor/tests/mod.rs index 41e3bfbe..8396bae9 100644 --- a/daemon-rs/src/handlers/conductor/tests.rs +++ b/daemon-rs/src/handlers/conductor/tests/mod.rs @@ -1,24 +1,15 @@ // SPDX-License-Identifier: MIT - use super::*; - #[cfg(test)] mod tests { use super::*; - #[test] fn bounded_ttl_seconds_bounds_fuzzed_request_values() { assert_eq!(bounded_ttl_seconds(None, 300), 300); assert_eq!(bounded_ttl_seconds(Some(60), 300), 60); assert_eq!(bounded_ttl_seconds(Some(0), 300), 1); assert_eq!(bounded_ttl_seconds(Some(-60), 300), 1); - assert_eq!( - bounded_ttl_seconds(Some(MAX_REQUEST_TTL_SECONDS + 1), 300), - MAX_REQUEST_TTL_SECONDS - ); - assert_eq!( - bounded_ttl_seconds(Some(i64::MAX), SESSION_TTL_SECONDS), - MAX_REQUEST_TTL_SECONDS - ); + assert_eq!(bounded_ttl_seconds(Some(MAX_REQUEST_TTL_SECONDS + 1), 300), MAX_REQUEST_TTL_SECONDS); + assert_eq!(bounded_ttl_seconds(Some(i64::MAX), SESSION_TTL_SECONDS), MAX_REQUEST_TTL_SECONDS); } } diff --git a/daemon-rs/src/handlers/conductor/types.rs b/daemon-rs/src/handlers/conductor/types.rs index 41a39e2f..bc00fc81 100644 --- a/daemon-rs/src/handlers/conductor/types.rs +++ b/daemon-rs/src/handlers/conductor/types.rs @@ -1,141 +1,67 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use crate::db::checkpoint_wal_best_effort; -use crate::handlers::{ - ensure_auth_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, - parse_timestamp_ms, redact_secrets, resolve_caller_id, -}; -use crate::state::RuntimeState; - - -use super::*; -// ─── Constants ────────────────────────────────────────────────────────────── - -pub(crate) const SESSION_TTL_SECONDS: i64 = 7200; // 2 hours -- agents heartbeat to extend -pub(crate) const ACTIVE_SESSION_WINDOW_SECONDS: i64 = 75; -pub(crate) const MAX_ACTIVITIES: i64 = 1000; -pub(crate) const MAX_MESSAGES_PER_AGENT: i64 = 100; -pub(crate) const MAX_TASKS: i64 = 500; -pub(crate) const DEFAULT_TASK_QUERY_LIMIT: usize = 200; -pub(crate) const MAX_TASK_QUERY_LIMIT: usize = 500; -pub(crate) const SESSION_FRESHNESS_IDLE_SECONDS: i64 = 24 * 60 * 60; +pub(crate) const SESSION_TTL_SECONDS: i64 = 7200; pub(crate) const MAX_REQUEST_TTL_SECONDS: i64 = 100 * 365 * 24 * 60 * 60; - -pub(crate) type SqlParam = Box; - -// ─── Request / query types ────────────────────────────────────────────────── - #[derive(Deserialize, Default)] pub struct LockRequest { pub path: Option, pub agent: Option, pub ttl: Option, } - #[derive(Deserialize, Default)] pub struct ActivityRequest { pub agent: Option, pub description: Option, - pub files: Option>, } - #[derive(Deserialize, Default)] -pub struct SinceQuery { - pub since: Option, -} - +pub struct SinceQuery {} #[derive(Deserialize, Default)] pub struct MessageRequest { pub from: Option, pub to: Option, pub message: Option, } - #[derive(Deserialize, Default)] -pub struct MessagesQuery { - pub agent: Option, -} - +pub struct MessagesQuery {} #[derive(Deserialize, Default)] pub struct SessionStartRequest { pub agent: Option, - pub project: Option, - pub files: Option>, - pub description: Option, - pub ttl: Option, } - #[derive(Deserialize, Default)] pub struct SessionHeartbeatRequest { pub agent: Option, - pub files: Option>, - pub description: Option, } - #[derive(Deserialize, Default)] pub struct SessionEndRequest { pub agent: Option, } - #[derive(Deserialize, Default)] pub struct TaskCreateRequest { pub title: Option, - pub description: Option, - pub project: Option, - pub files: Option>, - pub priority: Option, - #[serde(rename = "requiredCapability")] - pub required_capability: Option, } - #[derive(Deserialize, Default)] -pub struct TaskQuery { - pub status: Option, - pub project: Option, - pub limit: Option, - pub offset: Option, -} - +pub struct TaskQuery {} #[derive(Deserialize, Default)] pub struct TaskClaimRequest { #[serde(rename = "taskId")] pub task_id: Option, pub agent: Option, } - #[derive(Deserialize, Default)] pub struct TaskCompleteRequest { #[serde(rename = "taskId")] pub task_id: Option, pub agent: Option, - pub summary: Option, } - #[derive(Deserialize, Default)] pub struct TaskAbandonRequest { #[serde(rename = "taskId")] pub task_id: Option, pub agent: Option, } - #[derive(Deserialize, Default)] pub struct TaskDeleteRequest { #[serde(rename = "taskId")] pub task_id: Option, } - #[derive(Deserialize, Default)] -pub struct NextTaskQuery { - pub agent: Option, - pub capability: Option, -} - +pub struct NextTaskQuery {} diff --git a/daemon-rs/src/handlers/diary.rs b/daemon-rs/src/handlers/diary.rs index 17dcf4fe..8e9b8f2b 100644 --- a/daemon-rs/src/handlers/diary.rs +++ b/daemon-rs/src/handlers/diary.rs @@ -1,4 +1,5 @@ -// SPDX-License-Identifier: MIT +use super::{ensure_auth_rated, json_error, json_response, log_event, now_iso, resolve_source_identity}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; @@ -6,104 +7,64 @@ use axum::Json; use serde::Deserialize; use serde_json::json; use std::fs; -use std::path::Path; - -use super::{ - ensure_auth_rated, json_error, json_response, log_event, now_iso, resolve_source_identity, -}; -use crate::state::RuntimeState; - -// ─── Request type ───────────────────────────────────────────────────────────── - #[derive(Clone, Deserialize)] pub struct DiaryRequest { pub accomplished: Option, #[serde(rename = "nextSteps")] pub next_steps: Option, pub decisions: Option, - /// Legacy alias for decisions #[serde(rename = "keyDecisions")] pub key_decisions: Option, pub pending: Option, #[serde(rename = "knownIssues")] pub known_issues: Option, } - impl DiaryRequest { pub(crate) fn decisions_text(&self) -> Option<&str> { self.decisions.as_deref().or(self.key_decisions.as_deref()) } } - -// ─── POST /diary ────────────────────────────────────────────────────────────── - -pub async fn handle_diary( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_diary(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let source = resolve_source_identity(&headers, "http"); match write_diary_entry(&state, &body, &source.agent).await { - Ok(path) => json_response( - StatusCode::OK, - json!({ "written": true, "agent": source.agent, "path": path }), - ), + Ok(path) => json_response(StatusCode::OK, json!({"written":true,"agent":source.agent,"path":path})), Err(err) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &err), } } - -// ─── Helpers ────────────────────────────────────────────────────────────────── - -pub(crate) async fn write_diary_entry( - state: &RuntimeState, - body: &DiaryRequest, - agent: &str, -) -> Result { +pub(crate) async fn write_diary_entry(state: &RuntimeState, body: &DiaryRequest, agent: &str) -> Result { let state_path = state.home.join(".claude").join("state.md"); - if let Some(parent) = state_path.parent() { fs::create_dir_all(parent).map_err(|e| format!("Failed to create directory: {e}"))?; } - let existing = fs::read_to_string(&state_path).unwrap_or_default(); let content = build_diary_content(&existing, body); fs::write(&state_path, &content).map_err(|e| format!("Failed to write state.md: {e}"))?; - let conn = state.db.lock().await; let _ = log_event( &conn, "diary_write", - json!({ "agent": agent, "timestamp": now_iso() }), + json!({ +"agent":agent,"timestamp":now_iso()}), agent, ); - Ok(state_path.display().to_string()) } - fn build_diary_content(existing: &str, body: &DiaryRequest) -> String { let permanent = extract_section(existing, "## DO NOT REMOVE").unwrap_or_default(); let today = chrono::Utc::now().format("%Y-%m-%d").to_string(); let mut lines: Vec = vec![format!("# Session State — {today}"), String::new()]; - if !permanent.is_empty() { lines.push("## DO NOT REMOVE".to_string()); lines.push(permanent); lines.push(String::new()); } - - append_section( - &mut lines, - "## What Was Done This Session", - body.accomplished.as_deref(), - ); + append_section(&mut lines, "## What Was Done This Session", body.accomplished.as_deref()); append_section(&mut lines, "## Next Session", body.next_steps.as_deref()); append_section(&mut lines, "## Pending", body.pending.as_deref()); append_section(&mut lines, "## Known Issues", body.known_issues.as_deref()); - if let Some(text) = body.decisions_text() { append_section(&mut lines, "## Key Decisions", Some(text)); } else if let Some(content) = extract_section(existing, "## Key Decisions") { @@ -111,10 +72,8 @@ fn build_diary_content(existing: &str, body: &DiaryRequest) -> String { lines.push(content); lines.push(String::new()); } - lines.join("\n") } - fn append_section(lines: &mut Vec, header: &str, value: Option<&str>) { let Some(text) = value else { return; @@ -127,9 +86,6 @@ fn append_section(lines: &mut Vec, header: &str, value: Option<&str>) { lines.push(safe); lines.push(String::new()); } - -/// Extract the text body of a markdown section identified by `header`. -/// Returns `None` if the header is not found or the body is empty. fn extract_section(content: &str, header: &str) -> Option { let idx = content.find(header)?; let start = idx + header.len(); @@ -142,8 +98,6 @@ fn extract_section(content: &str, header: &str) -> Option { Some(text) } } - -/// Escape any user-provided `##` headers to prevent document structure breakage. fn sanitize_markdown(input: &str) -> String { input .lines() @@ -160,8 +114,3 @@ fn sanitize_markdown(input: &str) -> String { .collect::>() .join("\n") } - -#[allow(dead_code)] -fn path_to_string(path: &Path) -> String { - path.display().to_string() -} diff --git a/daemon-rs/src/handlers/event_log.rs b/daemon-rs/src/handlers/event_log.rs deleted file mode 100644 index 24a2daa0..00000000 --- a/daemon-rs/src/handlers/event_log.rs +++ /dev/null @@ -1,716 +0,0 @@ -// SPDX-License-Identifier: MIT -use chrono::Utc; -use rusqlite; -use serde_json::{json, Value}; - -use super::truncate_chars; - -const MAX_EVENT_JSON_BYTES: usize = 1_200; -const MAX_EVENT_VALUE_CHARS: usize = 240; -const MERGE_EVENT_PREVIEW_CHARS: usize = 240; -const MAX_SOURCE_LABEL_LEN: usize = 160; -const HIGH_VOLUME_EVENT_PRUNE_INTERVAL: i64 = 64; -const HIGH_VOLUME_EVENT_CAPS: &[(&str, i64)] = &[ - ("agent_boot", 4_000), - ("boot_savings", 6_000), - ("store_savings", 10_000), - ("tool_call_savings", 10_000), - ("decision_stored", 18_000), - ("decision_supersede", 10_000), - ("decision_refine_pending", 10_000), - ("decision_agreement_merge", 8_000), - ("decision_truncated", 8_000), - ("recall_query", 14_000), - ("merge", 6_000), - ("decision_conflict", 6_000), - ("decision_rejected_duplicate", 6_000), - ("decision_resolve", 6_000), - ("forget", 3_000), - ("diary_write", 3_000), -]; -const NON_PERSISTENT_BENCHMARK_EVENT_KINDS: &[&str] = &[ - "agent_boot", - "boot_savings", - "recall_query", - "store_savings", - "tool_call_savings", - "decision_stored", - "decision_conflict", - "decision_rejected_duplicate", - "decision_supersede", - "decision_refine_pending", - "decision_agreement_merge", - "decision_truncated", - "decision_resolve", - "merge", -]; - -fn compact_event_payload(kind: &str, data: Value) -> Value { - let projected = match kind { - "recall_query" => compact_recall_query_payload(data), - "merge" => compact_merge_event_payload(data), - "store_savings" | "tool_call_savings" | "boot_savings" => { - compact_savings_event_payload(data) - } - _ => truncate_event_value(data, 0), - }; - enforce_event_payload_budget(kind, projected) -} - -fn compact_recall_query_payload(data: Value) -> Value { - let Some(obj) = data.as_object() else { - return truncate_event_value(data, 0); - }; - - let semantic_route = compact_semantic_route(obj.get("semantic_route")); - let shadow_semantic = compact_shadow_semantic(obj.get("shadow_semantic")); - - json!({ - "agent": obj.get("agent").cloned().unwrap_or(Value::Null), - "query": obj - .get("query") - .and_then(Value::as_str) - .map(|q| truncate_chars(q, 120)) - .unwrap_or_default(), - "budget": extract_i64(obj.get("budget")), - "spent": extract_i64(obj.get("spent")), - "saved": extract_i64(obj.get("saved")), - "hits": extract_i64(obj.get("hits")), - "mode": obj.get("mode").cloned().unwrap_or(Value::Null), - "cached": obj.get("cached").cloned().unwrap_or(Value::Null), - "tier": obj.get("tier").cloned().unwrap_or(Value::Null), - "latency_ms": extract_i64(obj.get("latency_ms")), - "method_breakdown": truncate_event_value( - obj.get("method_breakdown").cloned().unwrap_or(Value::Null), - 0 - ), - "semantic_route": semantic_route, - "shadow_semantic": shadow_semantic, - }) -} - -fn compact_semantic_route(value: Option<&Value>) -> Value { - let Some(route) = value.and_then(Value::as_object) else { - return Value::Null; - }; - json!({ - "mode": route.get("mode").cloned().unwrap_or(Value::Null), - "reason": route.get("reason").cloned().unwrap_or(Value::Null), - "sampled": route.get("sampled").cloned().unwrap_or(Value::Null), - "trialPercent": route.get("trialPercent").cloned().unwrap_or(Value::Null), - "candidateCount": route.get("candidateCount").cloned().unwrap_or(Value::Null), - }) -} - -fn compact_shadow_semantic(value: Option<&Value>) -> Value { - let Some(shadow) = value.and_then(Value::as_object) else { - return Value::Null; - }; - json!({ - "status": shadow.get("status").cloned().unwrap_or(Value::Null), - "reason": shadow.get("reason").cloned().unwrap_or(Value::Null), - "baselineCount": shadow.get("baselineCount").cloned().unwrap_or(Value::Null), - "shadowCount": shadow.get("shadowCount").cloned().unwrap_or(Value::Null), - "overlapCount": shadow.get("overlapCount").cloned().unwrap_or(Value::Null), - "baselineTopSimilarity": shadow - .get("baselineTopSimilarity") - .cloned() - .unwrap_or(Value::Null), - "shadowTopSimilarity": shadow - .get("shadowTopSimilarity") - .cloned() - .unwrap_or(Value::Null), - // Keep payloads small and avoid storing source arrays in hot telemetry. - "baselineTopSources": Value::Null, - "shadowTopSources": Value::Null, - }) -} - -fn compact_merge_event_payload(data: Value) -> Value { - let Some(obj) = data.as_object() else { - return truncate_event_value(data, 0); - }; - let incoming = obj - .get("incoming_text") - .and_then(Value::as_str) - .unwrap_or_default(); - let incoming_chars = incoming.chars().count() as i64; - - json!({ - "source_id": obj.get("source_id").cloned().unwrap_or(Value::Null), - "target_id": obj.get("target_id").cloned().unwrap_or(Value::Null), - "target_type": obj.get("target_type").cloned().unwrap_or(Value::Null), - "similarity": obj.get("similarity").cloned().unwrap_or(Value::Null), - "jaccard": obj.get("jaccard").cloned().unwrap_or(Value::Null), - "source_agent": obj.get("source_agent").cloned().unwrap_or(Value::Null), - "incoming_chars": incoming_chars, - "incoming_preview": truncate_chars(incoming, MERGE_EVENT_PREVIEW_CHARS), - }) -} - -fn compact_savings_event_payload(data: Value) -> Value { - let Some(obj) = data.as_object() else { - return truncate_event_value(data, 0); - }; - json!({ - "agent": obj.get("agent").cloned().unwrap_or(Value::Null), - "query": obj - .get("query") - .and_then(Value::as_str) - .map(|q| truncate_chars(q, 120)) - .unwrap_or_default(), - "saved": extract_i64(obj.get("saved")), - "served": extract_i64(obj.get("served")), - "baseline": extract_i64(obj.get("baseline")), - "spent": extract_i64(obj.get("spent")), - "budget": extract_i64(obj.get("budget")), - "hits": extract_i64(obj.get("hits")), - "boots": extract_i64(obj.get("boots")), - "percent": extract_i64(obj.get("percent")), - "admitted": extract_i64(obj.get("admitted")), - "rejected": extract_i64(obj.get("rejected")), - "mode": obj.get("mode").cloned().unwrap_or(Value::Null), - "cached": obj.get("cached").cloned().unwrap_or(Value::Null), - "tier": obj.get("tier").cloned().unwrap_or(Value::Null), - "latency_ms": extract_i64(obj.get("latency_ms")), - }) -} - -fn extract_i64(value: Option<&Value>) -> i64 { - value - .and_then(|v| { - v.as_i64() - .or_else(|| v.as_u64().and_then(|x| i64::try_from(x).ok())) - .or_else(|| v.as_f64().map(|x| x.round() as i64)) - }) - .unwrap_or(0) -} - -fn truncate_event_value(value: Value, depth: usize) -> Value { - if depth >= 4 { - return Value::Null; - } - match value { - Value::String(s) => Value::String(truncate_chars(&s, MAX_EVENT_VALUE_CHARS)), - Value::Array(items) => Value::Array( - items - .into_iter() - .take(16) - .map(|item| truncate_event_value(item, depth + 1)) - .collect(), - ), - Value::Object(map) => { - let compacted = map - .into_iter() - .take(24) - .map(|(key, val)| (key, truncate_event_value(val, depth + 1))) - .collect(); - Value::Object(compacted) - } - other => other, - } -} - -fn enforce_event_payload_budget(kind: &str, payload: Value) -> Value { - let encoded = payload.to_string(); - if encoded.len() <= MAX_EVENT_JSON_BYTES { - return payload; - } - - let mut fallback = json!({ - "truncated": true, - "type": kind, - "bytes": encoded.len() - }); - if let Some(obj) = payload.as_object() { - for key in [ - "agent", - "source_agent", - "saved", - "served", - "baseline", - "spent", - "budget", - "hits", - "misses", - "events", - "boots", - "percent", - "admitted", - "rejected", - "mode", - "cached", - "tier", - "latency_ms", - "source_id", - "target_id", - "target_type", - "similarity", - "jaccard", - "incoming_chars", - ] { - if let Some(value) = obj - .get(key) - .and_then(|value| compact_budget_scalar(value, MAX_EVENT_VALUE_CHARS)) - { - fallback[key] = value; - } - } - if let Some(query) = obj.get("query").and_then(Value::as_str) { - fallback["query"] = Value::String(truncate_chars(query, 120)); - } - let semantic_route = compact_semantic_route(obj.get("semantic_route")); - if !semantic_route.is_null() { - fallback["semantic_route"] = semantic_route; - } - let shadow_semantic = compact_shadow_semantic(obj.get("shadow_semantic")); - if !shadow_semantic.is_null() { - fallback["shadow_semantic"] = shadow_semantic; - } - } - - if fallback.to_string().len() <= MAX_EVENT_JSON_BYTES { - return fallback; - } - - if let Some(fallback_obj) = fallback.as_object_mut() { - for key in [ - "query", - "semantic_route", - "shadow_semantic", - "target_type", - "tier", - "mode", - ] { - fallback_obj.remove(key); - } - } - if fallback.to_string().len() <= MAX_EVENT_JSON_BYTES { - return fallback; - } - - let mut minimal = json!({ - "truncated": true, - "type": kind, - "bytes": encoded.len() - }); - if let Some(obj) = payload.as_object() { - for key in ["agent", "source_agent"] { - if let Some(value) = obj - .get(key) - .and_then(|value| compact_budget_scalar(value, MAX_SOURCE_LABEL_LEN)) - { - minimal[key] = value; - } - } - } - minimal -} - -fn compact_budget_scalar(value: &Value, max_chars: usize) -> Option { - match value { - Value::String(text) => Some(Value::String(truncate_chars(text, max_chars))), - Value::Number(_) | Value::Bool(_) | Value::Null => Some(value.clone()), - _ => None, - } -} - -fn payload_field_has_benchmark_prefix(payload: &Value, key: &str, lowercase_prefix: &str) -> bool { - payload - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(|value| value.to_ascii_lowercase().starts_with(lowercase_prefix)) - .unwrap_or(false) -} - -fn is_benchmark_event_source(source_agent: &str, payload: &Value) -> bool { - let benchmark_prefix = crate::compaction::BENCHMARK_SOURCE_AGENT_PREFIX.to_ascii_lowercase(); - source_agent - .trim() - .to_ascii_lowercase() - .starts_with(&benchmark_prefix) - || payload_field_has_benchmark_prefix(payload, "source_agent", &benchmark_prefix) - || payload_field_has_benchmark_prefix(payload, "agent", &benchmark_prefix) -} - -fn should_skip_benchmark_event_persistence( - kind: &str, - payload: &Value, - source_agent: &str, -) -> bool { - NON_PERSISTENT_BENCHMARK_EVENT_KINDS.contains(&kind) - && is_benchmark_event_source(source_agent, payload) -} - -/// Insert an event row into the `events` table. -pub fn log_event( - conn: &rusqlite::Connection, - kind: &str, - data: Value, - source_agent: &str, -) -> rusqlite::Result<()> { - let compacted = compact_event_payload(kind, data); - if should_skip_benchmark_event_persistence(kind, &compacted, source_agent) { - return Ok(()); - } - conn.execute( - "INSERT INTO events (type, data, source_agent) VALUES (?1, ?2, ?3)", - rusqlite::params![kind, compacted.to_string(), source_agent], - )?; - maybe_prune_high_volume_event(conn, kind)?; - Ok(()) -} - -fn maybe_prune_high_volume_event(conn: &rusqlite::Connection, kind: &str) -> rusqlite::Result<()> { - let Some(keep_rows) = HIGH_VOLUME_EVENT_CAPS - .iter() - .find_map(|(event_type, keep)| (*event_type == kind).then_some(*keep)) - else { - return Ok(()); - }; - let inserted_id = conn.last_insert_rowid(); - if inserted_id <= 0 || inserted_id % HIGH_VOLUME_EVENT_PRUNE_INTERVAL != 0 { - return Ok(()); - } - prune_event_type_keep_latest(conn, kind, keep_rows) -} - -fn prune_event_type_keep_latest( - conn: &rusqlite::Connection, - event_type: &str, - keep_rows: i64, -) -> rusqlite::Result<()> { - if keep_rows < 1 { - return Ok(()); - } - conn.execute( - "DELETE FROM events - WHERE id IN ( - SELECT id - FROM events - WHERE type = ?1 - ORDER BY id DESC - LIMIT -1 OFFSET ?2 - )", - rusqlite::params![event_type, keep_rows], - )?; - Ok(()) -} - - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn prune_event_type_keep_latest_trims_old_rows() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - conn.execute_batch( - "CREATE TABLE events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - data TEXT NOT NULL, - source_agent TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - );", - ) - .expect("create events table"); - - for idx in 0..6 { - conn.execute( - "INSERT INTO events (type, data, source_agent) VALUES ('decision_stored', ?1, 'test')", - rusqlite::params![format!("{{\"idx\":{idx}}}")], - ) - .expect("insert event"); - } - - prune_event_type_keep_latest(&conn, "decision_stored", 3).expect("prune rows"); - - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'decision_stored'", - [], - |row| row.get(0), - ) - .expect("count rows"); - assert_eq!(count, 3); - } - - #[test] - fn log_event_compacts_large_merge_payload() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - conn.execute_batch( - "CREATE TABLE events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - data TEXT NOT NULL, - source_agent TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - );", - ) - .expect("create events table"); - - let incoming = "x".repeat(10_000); - log_event( - &conn, - "merge", - json!({ - "target_id": 42, - "target_type": "decision", - "incoming_text": incoming, - "source_agent": "test-agent" - }), - "test", - ) - .expect("log merge event"); - - let payload: String = conn - .query_row( - "SELECT data FROM events WHERE type = 'merge' LIMIT 1", - [], - |row| row.get(0), - ) - .expect("read payload"); - let parsed: Value = serde_json::from_str(&payload).expect("valid json"); - assert!(parsed.get("incoming_text").is_none()); - assert_eq!(parsed["incoming_chars"].as_i64(), Some(10_000)); - assert!(parsed["incoming_preview"] - .as_str() - .map(|text| text.len() <= MERGE_EVENT_PREVIEW_CHARS) - .unwrap_or(false)); - } - - #[test] - fn log_event_keeps_recall_analytics_fields_small() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - conn.execute_batch( - "CREATE TABLE events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - data TEXT NOT NULL, - source_agent TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - );", - ) - .expect("create events table"); - - log_event( - &conn, - "recall_query", - json!({ - "agent": "codex", - "query": "daemon ownership lock protects startup arbitration", - "budget": 240, - "spent": 52, - "saved": 188, - "hits": 3, - "mode": "balanced", - "cached": false, - "tier": "hybrid_fusion", - "latency_ms": 12, - "semantic_route": { - "mode": "baseline", - "reason": "not_sampled", - "sampled": false, - "trialPercent": 1, - "ranked_sources": ["a", "b", "c", "d", "e"] - }, - "shadow_semantic": { - "status": "unavailable", - "reason": "query_embedding_unavailable", - "baselineTopSources": ["very", "large", "list"], - "shadowTopSources": ["another", "big", "list"] - }, - "method_breakdown": { - "keyword": 2, - "semantic": 1, - "unused_verbose_blob": "x".repeat(2000) - } - }), - "codex", - ) - .expect("log recall event"); - - let (payload, bytes): (String, i64) = conn - .query_row( - "SELECT data, LENGTH(data) FROM events WHERE type = 'recall_query' LIMIT 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read payload"); - let parsed: Value = serde_json::from_str(&payload).expect("valid json"); - assert_eq!(parsed["saved"].as_i64(), Some(188)); - assert_eq!(parsed["budget"].as_i64(), Some(240)); - assert_eq!(parsed["hits"].as_i64(), Some(3)); - assert_eq!(parsed["semantic_route"]["mode"].as_str(), Some("baseline")); - assert_eq!( - parsed["shadow_semantic"]["status"].as_str(), - Some("unavailable") - ); - assert!(parsed["shadow_semantic"]["baselineTopSources"].is_null()); - assert!(parsed["shadow_semantic"]["shadowTopSources"].is_null()); - assert!(bytes as usize <= MAX_EVENT_JSON_BYTES); - } - - #[test] - fn log_event_skips_non_persistent_benchmark_noise() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - conn.execute_batch( - "CREATE TABLE events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - data TEXT NOT NULL, - source_agent TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - );", - ) - .expect("create events table"); - - log_event( - &conn, - "recall_query", - json!({ - "agent": "amb-cortex::run-a", - "query": "benchmark probe", - "saved": 50, - "spent": 20, - "budget": 70, - "hits": 1, - "method_breakdown": json!({ - "alpha": "x".repeat(1024), - "beta": "x".repeat(1024), - "gamma": "x".repeat(1024), - "delta": "x".repeat(1024), - "epsilon": "x".repeat(1024), - "zeta": "x".repeat(1024), - "eta": "x".repeat(1024), - "theta": "x".repeat(1024) - }) - }), - "rust-daemon", - ) - .expect("skip benchmark recall noise"); - log_event( - &conn, - "agent_boot", - json!({ - "agent": "amb-cortex::run-a", - "bytes_before": 1, - "bytes_after": 1 - }), - "rust-daemon", - ) - .expect("skip benchmark agent_boot noise"); - log_event( - &conn, - "decision_stored", - json!({ - "id": 42, - "source_agent": "amb-cortex::run-a" - }), - "rust-daemon", - ) - .expect("skip benchmark decision_stored noise"); - - let skipped_count: i64 = conn - .query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0)) - .expect("count skipped rows"); - assert_eq!(skipped_count, 0); - - log_event( - &conn, - "recall_query", - json!({ - "agent": "codex", - "query": "production request", - "saved": 12, - "spent": 8, - "budget": 20, - "hits": 1 - }), - "codex", - ) - .expect("persist non-benchmark event"); - - let persisted_count: i64 = conn - .query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0)) - .expect("count persisted rows"); - assert_eq!(persisted_count, 1); - } - - #[test] - fn log_event_payload_fallback_keeps_savings_fields_bounded() { - let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); - conn.execute_batch( - "CREATE TABLE events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - data TEXT NOT NULL, - source_agent TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP - );", - ) - .expect("create events table"); - - let mut method_breakdown = serde_json::Map::new(); - for idx in 0..24 { - method_breakdown.insert(format!("bucket_{idx}"), Value::String("x".repeat(1024))); - } - - log_event( - &conn, - "recall_query", - json!({ - "agent": "codex", - "query": "q".repeat(1200), - "budget": 240, - "spent": 52, - "saved": 188, - "hits": 3, - "mode": "balanced", - "cached": false, - "tier": "hybrid_fusion", - "latency_ms": 12, - "semantic_route": { - "mode": "baseline", - "reason": "not_sampled", - "sampled": false, - "trialPercent": 1 - }, - "shadow_semantic": { - "status": "unavailable", - "reason": "query_embedding_unavailable", - "baselineTopSources": ["very", "large", "list"] - }, - "method_breakdown": Value::Object(method_breakdown) - }), - "codex", - ) - .expect("log oversized recall event"); - - let (payload, bytes): (String, i64) = conn - .query_row( - "SELECT data, LENGTH(data) FROM events WHERE type = 'recall_query' LIMIT 1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("read payload"); - let parsed: Value = serde_json::from_str(&payload).expect("valid json"); - assert_eq!(parsed["truncated"].as_bool(), Some(true)); - assert_eq!(parsed["saved"].as_i64(), Some(188)); - assert_eq!(parsed["budget"].as_i64(), Some(240)); - assert_eq!(parsed["hits"].as_i64(), Some(3)); - assert_eq!(parsed["agent"].as_str(), Some("codex")); - assert!( - parsed["query"] - .as_str() - .map(|query| query.chars().count() <= 120) - .unwrap_or(false), - "query should stay bounded in fallback payload" - ); - assert!(bytes as usize <= MAX_EVENT_JSON_BYTES); - } -} diff --git a/daemon-rs/src/handlers/event_log/mod.rs b/daemon-rs/src/handlers/event_log/mod.rs new file mode 100644 index 00000000..e90809c9 --- /dev/null +++ b/daemon-rs/src/handlers/event_log/mod.rs @@ -0,0 +1,261 @@ +use super::truncate_chars; +use rusqlite; +use serde_json::{json, Value}; +const MAX_EVENT_JSON_BYTES: usize = 1_200; +const MAX_EVENT_VALUE_CHARS: usize = 240; +const MERGE_EVENT_PREVIEW_CHARS: usize = 240; +const MAX_SOURCE_LABEL_LEN: usize = 160; +const HIGH_VOLUME_EVENT_PRUNE_INTERVAL: i64 = 64; +const HIGH_VOLUME_EVENT_CAPS: &[(&str, i64)] = &[ + ("agent_boot", 4_000), + ("boot_savings", 6_000), + ("store_savings", 10_000), + ("tool_call_savings", 10_000), + ("decision_stored", 18_000), + ("decision_supersede", 10_000), + ("decision_refine_pending", 10_000), + ("decision_agreement_merge", 8_000), + ("decision_truncated", 8_000), + ("recall_query", 14_000), + ("merge", 6_000), + ("decision_conflict", 6_000), + ("decision_rejected_duplicate", 6_000), + ("decision_resolve", 6_000), + ("forget", 3_000), + ("diary_write", 3_000), +]; +const NON_PERSISTENT_BENCHMARK_EVENT_KINDS: &[&str] = &[ + "agent_boot", + "boot_savings", + "recall_query", + "store_savings", + "tool_call_savings", + "decision_stored", + "decision_conflict", + "decision_rejected_duplicate", + "decision_supersede", + "decision_refine_pending", + "decision_agreement_merge", + "decision_truncated", + "decision_resolve", + "merge", +]; +fn compact_event_payload(kind: &str, data: Value) -> Value { + let projected = match kind { + "recall_query" => compact_recall_query_payload(data), + "merge" => compact_merge_event_payload(data), + "store_savings" | "tool_call_savings" | "boot_savings" => compact_savings_event_payload(data), + _ => truncate_event_value(data, 0), + }; + enforce_event_payload_budget(kind, projected) +} +fn compact_recall_query_payload(data: Value) -> Value { + let Some(obj) = data.as_object() else { + return truncate_event_value(data, 0); + }; + let semantic_route = compact_semantic_route(obj.get("semantic_route")); + let shadow_semantic = compact_shadow_semantic(obj.get("shadow_semantic")); + json!({"agent":obj.get("agent").cloned( +).unwrap_or(Value::Null),"query":obj.get("query").and_then(Value::as_str).map(|q|truncate_chars(q,120)).unwrap_or_default(), +"budget":extract_i64(obj.get("budget")),"spent":extract_i64(obj.get("spent")),"saved":extract_i64(obj.get("saved")),"hits": +extract_i64(obj.get("hits")),"mode":obj.get("mode").cloned().unwrap_or(Value::Null),"cached":obj.get("cached").cloned().unwrap_or( +Value::Null),"tier":obj.get("tier").cloned().unwrap_or(Value::Null),"latency_ms":extract_i64(obj.get("latency_ms")), +"method_breakdown":truncate_event_value(obj.get("method_breakdown").cloned().unwrap_or(Value::Null),0),"semantic_route": +semantic_route,"shadow_semantic":shadow_semantic,}) +} +fn compact_semantic_route(value: Option<&Value>) -> Value { + let Some(route) = value.and_then(Value::as_object) else { + return Value::Null; + }; + json!({"mode":route.get("mode").cloned().unwrap_or(Value::Null),"reason":route + .get("reason").cloned().unwrap_or(Value::Null),"sampled":route.get("sampled").cloned().unwrap_or(Value::Null),"trialPercent":route + .get("trialPercent").cloned().unwrap_or(Value::Null),"candidateCount":route.get("candidateCount").cloned().unwrap_or(Value::Null), + }) +} +fn compact_shadow_semantic(value: Option<&Value>) -> Value { + let Some(shadow) = value.and_then(Value::as_object) else { + return Value::Null; + }; + json!({"status":shadow.get("status").cloned().unwrap_or(Value::Null),"reason":shadow.get("reason").cloned().unwrap_or(Value +::Null),"baselineCount":shadow.get("baselineCount").cloned().unwrap_or(Value::Null),"shadowCount":shadow.get("shadowCount").cloned +().unwrap_or(Value::Null),"overlapCount":shadow.get("overlapCount").cloned().unwrap_or(Value::Null),"baselineTopSimilarity":shadow +.get("baselineTopSimilarity").cloned().unwrap_or(Value::Null),"shadowTopSimilarity":shadow.get("shadowTopSimilarity").cloned(). +unwrap_or(Value::Null),"baselineTopSources":Value::Null,"shadowTopSources":Value::Null,}) +} +fn compact_merge_event_payload(data: Value) -> Value { + let Some(obj) = data.as_object() else { + return truncate_event_value(data, 0); + }; + let incoming = obj.get("incoming_text").and_then(Value::as_str).unwrap_or_default(); + let incoming_chars = incoming.chars().count() as i64; + json!({"source_id":obj.get( +"source_id").cloned().unwrap_or(Value::Null),"target_id":obj.get("target_id").cloned().unwrap_or(Value::Null),"target_type":obj. +get("target_type").cloned().unwrap_or(Value::Null),"similarity":obj.get("similarity").cloned().unwrap_or(Value::Null),"jaccard": +obj.get("jaccard").cloned().unwrap_or(Value::Null),"source_agent":obj.get("source_agent").cloned().unwrap_or(Value::Null), +"incoming_chars":incoming_chars,"incoming_preview":truncate_chars(incoming,MERGE_EVENT_PREVIEW_CHARS),}) +} +fn compact_savings_event_payload(data: Value) -> Value { + let Some(obj) = data.as_object() else { + return truncate_event_value(data, 0); + }; + json!({ +"agent":obj.get("agent").cloned().unwrap_or(Value::Null),"query":obj.get("query").and_then(Value::as_str).map(|q|truncate_chars(q, +120)).unwrap_or_default(),"saved":extract_i64(obj.get("saved")),"served":extract_i64(obj.get("served")),"baseline":extract_i64(obj +.get("baseline")),"spent":extract_i64(obj.get("spent")),"budget":extract_i64(obj.get("budget")),"hits":extract_i64(obj.get("hits") +),"boots":extract_i64(obj.get("boots")),"percent":extract_i64(obj.get("percent")),"admitted":extract_i64(obj.get("admitted")), +"rejected":extract_i64(obj.get("rejected")),"mode":obj.get("mode").cloned().unwrap_or(Value::Null),"cached":obj.get("cached"). +cloned().unwrap_or(Value::Null),"tier":obj.get("tier").cloned().unwrap_or(Value::Null),"latency_ms":extract_i64(obj.get( +"latency_ms")),}) +} +fn extract_i64(value: Option<&Value>) -> i64 { + value + .and_then(|v| v.as_i64().or_else(|| v.as_u64().and_then(|x| i64::try_from(x).ok())).or_else(|| v.as_f64().map(|x| x.round() as i64))) + .unwrap_or(0) +} +fn truncate_event_value(value: Value, depth: usize) -> Value { + if depth >= 4 { + return Value::Null; + } + match value { + Value::String(s) => Value::String(truncate_chars(&s, MAX_EVENT_VALUE_CHARS)), + Value::Array(items) => Value::Array(items.into_iter().take(16).map(|item| truncate_event_value(item, depth + 1)).collect()), + Value::Object(map) => { + let compacted = map.into_iter().take(24).map(|(key, val)| (key, truncate_event_value(val, depth + 1))).collect(); + Value::Object(compacted) + } + other => other, + } +} +fn enforce_event_payload_budget(kind: &str, payload: Value) -> Value { + let encoded = payload.to_string(); + if encoded.len() <= MAX_EVENT_JSON_BYTES { + return payload; + } + let mut fallback = json!({"truncated":true,"type":kind,"bytes":encoded.len()}); + if let Some(obj) = payload.as_object() { + for key in [ + "agent", + "source_agent", + "saved", + "served", + "baseline", + "spent", + "budget", + "hits", + "misses", + "events", + "boots", + "percent", + "admitted", + "rejected", + "mode", + "cached", + "tier", + "latency_ms", + "source_id", + "target_id", + "target_type", + "similarity", + "jaccard", + "incoming_chars", + ] { + if let Some(value) = obj.get(key).and_then(|value| compact_budget_scalar(value, MAX_EVENT_VALUE_CHARS)) { + fallback[key] = value; + } + } + if let Some(query) = obj.get("query").and_then(Value::as_str) { + fallback["query"] = Value::String(truncate_chars(query, 120)); + } + let semantic_route = compact_semantic_route(obj.get("semantic_route")); + if !semantic_route.is_null() { + fallback["semantic_route"] = semantic_route; + } + let shadow_semantic = compact_shadow_semantic(obj.get("shadow_semantic")); + if !shadow_semantic.is_null() { + fallback["shadow_semantic"] = shadow_semantic; + } + } + if fallback.to_string().len() <= MAX_EVENT_JSON_BYTES { + return fallback; + } + if let Some(fallback_obj) = fallback.as_object_mut() { + for key in ["query", "semantic_route", "shadow_semantic", "target_type", "tier", "mode"] { + fallback_obj.remove(key); + } + } + if fallback.to_string().len() <= MAX_EVENT_JSON_BYTES { + return fallback; + } + let mut minimal = json!({"truncated":true,"type":kind,"bytes":encoded.len()}); + if let Some(obj) = payload.as_object() { + for key in ["agent", "source_agent"] { + if let Some(value) = obj.get(key).and_then(|value| compact_budget_scalar(value, MAX_SOURCE_LABEL_LEN)) { + minimal[key] = value; + } + } + } + minimal +} +fn compact_budget_scalar(value: &Value, max_chars: usize) -> Option { + match value { + Value::String(text) => Some(Value::String(truncate_chars(text, max_chars))), + Value::Number(_) | Value::Bool(_) | Value::Null => Some(value.clone()), + _ => None, + } +} +fn payload_field_has_benchmark_prefix(payload: &Value, key: &str, lowercase_prefix: &str) -> bool { + payload + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| value.to_ascii_lowercase().starts_with(lowercase_prefix)) + .unwrap_or(false) +} +fn is_benchmark_event_source(source_agent: &str, payload: &Value) -> bool { + let benchmark_prefix = crate::compaction::BENCHMARK_SOURCE_AGENT_PREFIX.to_ascii_lowercase(); + source_agent.trim().to_ascii_lowercase().starts_with(&benchmark_prefix) + || payload_field_has_benchmark_prefix(payload, "source_agent", &benchmark_prefix) + || payload_field_has_benchmark_prefix(payload, "agent", &benchmark_prefix) +} +fn should_skip_benchmark_event_persistence(kind: &str, payload: &Value, source_agent: &str) -> bool { + NON_PERSISTENT_BENCHMARK_EVENT_KINDS.contains(&kind) && is_benchmark_event_source(source_agent, payload) +} +pub fn log_event(conn: &rusqlite::Connection, kind: &str, data: Value, source_agent: &str) -> rusqlite::Result<()> { + let compacted = compact_event_payload(kind, data); + if should_skip_benchmark_event_persistence(kind, &compacted, source_agent) { + return Ok(()); + } + conn.execute("INSERT INTO events (type, data, source_agent) VALUES (?1, ?2, ?3)", rusqlite::params![kind, compacted.to_string(), source_agent])?; + maybe_prune_high_volume_event(conn, kind)?; + Ok(()) +} +fn maybe_prune_high_volume_event(conn: &rusqlite::Connection, kind: &str) -> rusqlite::Result<()> { + let Some(keep_rows) = HIGH_VOLUME_EVENT_CAPS.iter().find_map(|(event_type, keep)| (*event_type == kind).then_some(*keep)) else { + return Ok(()); + }; + let inserted_id = conn.last_insert_rowid(); + if inserted_id <= 0 || inserted_id % HIGH_VOLUME_EVENT_PRUNE_INTERVAL != 0 { + return Ok(()); + } + prune_event_type_keep_latest(conn, kind, keep_rows) +} +fn prune_event_type_keep_latest(conn: &rusqlite::Connection, event_type: &str, keep_rows: i64) -> rusqlite::Result<()> { + if keep_rows < 1 { + return Ok(()); + } + conn.execute( + "DELETE FROM events + WHERE id IN ( + SELECT id + FROM events + WHERE type = ?1 + ORDER BY id DESC + LIMIT -1 OFFSET ?2 + )", + rusqlite::params![event_type, keep_rows], + )?; + Ok(()) +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/handlers/event_log/tests/mod.rs b/daemon-rs/src/handlers/event_log/tests/mod.rs new file mode 100644 index 00000000..efa90762 --- /dev/null +++ b/daemon-rs/src/handlers/event_log/tests/mod.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use serde_json::json; +#[test] +fn prune_event_type_keep_latest_trims_old_rows() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + conn.execute_batch( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + data TEXT NOT NULL, + source_agent TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );", + ) + .expect("create events table"); + for idx in 0..6 { + conn.execute("INSERT INTO events (type, data, source_agent) VALUES ('decision_stored', ?1, 'test')", rusqlite::params![format!("{{\"idx\":{idx}}}")]) + .expect("insert event"); + } + prune_event_type_keep_latest(&conn, "decision_stored", 3).expect("prune rows"); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM events WHERE type = 'decision_stored'", [], |row| row.get(0)).expect("count rows"); + assert_eq!(count, 3); +} +#[test] +fn log_event_compacts_large_merge_payload() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + conn.execute_batch( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + data TEXT NOT NULL, + source_agent TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );", + ) + .expect("create events table"); + let incoming = "x".repeat(10_000); + log_event( + &conn, + "merge", + json!({ + "target_id": 42, + "target_type": "decision", + "incoming_text": incoming, + "source_agent": "test-agent" + }), + "test", + ) + .expect("log merge event"); + let payload: String = conn.query_row("SELECT data FROM events WHERE type = 'merge' LIMIT 1", [], |row| row.get(0)).expect("read payload"); + let parsed: Value = serde_json::from_str(&payload).expect("valid json"); + assert!(parsed.get("incoming_text").is_none()); + assert_eq!(parsed["incoming_chars"].as_i64(), Some(10_000)); + assert!(parsed["incoming_preview"].as_str().map(|text| text.len() <= MERGE_EVENT_PREVIEW_CHARS).unwrap_or(false)); +} +#[test] +fn log_event_keeps_recall_analytics_fields_small() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + conn.execute_batch( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + data TEXT NOT NULL, + source_agent TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );", + ) + .expect("create events table"); + log_event( + &conn, + "recall_query", + json!({ + "agent": "codex", + "query": "daemon ownership lock protects startup arbitration", + "budget": 240, + "spent": 52, + "saved": 188, + "hits": 3, + "mode": "balanced", + "cached": false, + "tier": "hybrid_fusion", + "latency_ms": 12, + "semantic_route": { + "mode": "baseline", + "reason": "not_sampled", + "sampled": false, + "trialPercent": 1, + "ranked_sources": ["a", "b", "c", "d", "e"] + }, + "shadow_semantic": { + "status": "unavailable", + "reason": "query_embedding_unavailable", + "baselineTopSources": ["very", "large", "list"], + "shadowTopSources": ["another", "big", "list"] + }, + "method_breakdown": { + "keyword": 2, + "semantic": 1, + "unused_verbose_blob": "x".repeat(2000) + } + }), + "codex", + ) + .expect("log recall event"); + let (payload, bytes): (String, i64) = conn + .query_row("SELECT data, LENGTH(data) FROM events WHERE type = 'recall_query' LIMIT 1", [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("read payload"); + let parsed: Value = serde_json::from_str(&payload).expect("valid json"); + assert_eq!(parsed["saved"].as_i64(), Some(188)); + assert_eq!(parsed["budget"].as_i64(), Some(240)); + assert_eq!(parsed["hits"].as_i64(), Some(3)); + assert_eq!(parsed["semantic_route"]["mode"].as_str(), Some("baseline")); + assert_eq!(parsed["shadow_semantic"]["status"].as_str(), Some("unavailable")); + assert!(parsed["shadow_semantic"]["baselineTopSources"].is_null()); + assert!(parsed["shadow_semantic"]["shadowTopSources"].is_null()); + assert!(bytes as usize <= MAX_EVENT_JSON_BYTES); +} +#[test] +fn log_event_skips_non_persistent_benchmark_noise() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + conn.execute_batch( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + data TEXT NOT NULL, + source_agent TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );", + ) + .expect("create events table"); + log_event( + &conn, + "recall_query", + json!({ + "agent": "amb-cortex::run-a", + "query": "benchmark probe", + "saved": 50, + "spent": 20, + "budget": 70, + "hits": 1, + "method_breakdown": json!({ + "alpha": "x".repeat(1024), + "beta": "x".repeat(1024), + "gamma": "x".repeat(1024), + "delta": "x".repeat(1024), + "epsilon": "x".repeat(1024), + "zeta": "x".repeat(1024), + "eta": "x".repeat(1024), + "theta": "x".repeat(1024) + }) + }), + "rust-daemon", + ) + .expect("skip benchmark recall noise"); + log_event( + &conn, + "agent_boot", + json!({ + "agent": "amb-cortex::run-a", + "bytes_before": 1, + "bytes_after": 1 + }), + "rust-daemon", + ) + .expect("skip benchmark agent_boot noise"); + log_event( + &conn, + "decision_stored", + json!({ + "id": 42, + "source_agent": "amb-cortex::run-a" + }), + "rust-daemon", + ) + .expect("skip benchmark decision_stored noise"); + let skipped_count: i64 = conn.query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0)).expect("count skipped rows"); + assert_eq!(skipped_count, 0); + log_event( + &conn, + "recall_query", + json!({ + "agent": "codex", + "query": "production request", + "saved": 12, + "spent": 8, + "budget": 20, + "hits": 1 + }), + "codex", + ) + .expect("persist non-benchmark event"); + let persisted_count: i64 = conn.query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0)).expect("count persisted rows"); + assert_eq!(persisted_count, 1); +} +#[test] +fn log_event_payload_fallback_keeps_savings_fields_bounded() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); + conn.execute_batch( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + data TEXT NOT NULL, + source_agent TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );", + ) + .expect("create events table"); + let mut method_breakdown = serde_json::Map::new(); + for idx in 0..24 { + method_breakdown.insert(format!("bucket_{idx}"), Value::String("x".repeat(1024))); + } + log_event( + &conn, + "recall_query", + json!({ + "agent": "codex", + "query": "q".repeat(1200), + "budget": 240, + "spent": 52, + "saved": 188, + "hits": 3, + "mode": "balanced", + "cached": false, + "tier": "hybrid_fusion", + "latency_ms": 12, + "semantic_route": { + "mode": "baseline", + "reason": "not_sampled", + "sampled": false, + "trialPercent": 1 + }, + "shadow_semantic": { + "status": "unavailable", + "reason": "query_embedding_unavailable", + "baselineTopSources": ["very", "large", "list"] + }, + "method_breakdown": Value::Object(method_breakdown) + }), + "codex", + ) + .expect("log oversized recall event"); + let (payload, bytes): (String, i64) = conn + .query_row("SELECT data, LENGTH(data) FROM events WHERE type = 'recall_query' LIMIT 1", [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("read payload"); + let parsed: Value = serde_json::from_str(&payload).expect("valid json"); + assert_eq!(parsed["truncated"].as_bool(), Some(true)); + assert_eq!(parsed["saved"].as_i64(), Some(188)); + assert_eq!(parsed["budget"].as_i64(), Some(240)); + assert_eq!(parsed["hits"].as_i64(), Some(3)); + assert_eq!(parsed["agent"].as_str(), Some("codex")); + assert!(parsed["query"].as_str().map(|query| query.chars().count() <= 120).unwrap_or(false), "query should stay bounded in fallback payload"); + assert!(bytes as usize <= MAX_EVENT_JSON_BYTES); +} diff --git a/daemon-rs/src/handlers/events.rs b/daemon-rs/src/handlers/events.rs deleted file mode 100644 index 4763c273..00000000 --- a/daemon-rs/src/handlers/events.rs +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: MIT -use std::convert::Infallible; -use std::time::Duration as StdDuration; - -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::sse::{Event, KeepAlive, Sse}; -use axum::response::{IntoResponse, Response}; -use futures_util::stream::{self, StreamExt}; -use serde::Deserialize; -use serde_json::{json, Value}; -use tokio_stream::wrappers::BroadcastStream; - -use super::{ensure_events_stream_auth, json_response, now_iso, runtime_token_matches}; -use crate::state::{BrainFiringEvent, RuntimeState}; - -fn scrub_event_payload(event_type: &str) -> Value { - json!({ - "type": event_type, - "timestamp": now_iso() - }) -} - -#[derive(Deserialize)] -pub struct EventsStreamQuery { - pub token: Option, -} - -// ─── GET /events/stream ───────────────────────────────────────────────────── - -pub async fn handle_events_stream( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = - ensure_events_stream_auth(&headers, query.token.as_deref(), &state).await - { - return resp; - } - - let initial = stream::once(async move { - Ok::( - Event::default() - .event("connected") - .data(scrub_event_payload("connected").to_string()), - ) - }); - - let updates = BroadcastStream::new(state.events.subscribe()).filter_map(|msg| async move { - match msg { - Ok(event) => { - let payload = scrub_event_payload(&event.event_type); - Some(Ok::( - Event::default() - .event(&event.event_type) - .data(payload.to_string()), - )) - } - Err(_) => None, - } - }); - - let stream = initial.chain(updates); - let sse = Sse::new(stream).keep_alive( - KeepAlive::new() - .interval(StdDuration::from_secs(30)) - .text("keepalive"), - ); - // CORS handled by tower-http CorsLayer in server.rs -- no manual override - sse.into_response() -} - -// ─── GET /brain/firing ────────────────────────────────────────────────────── - -#[derive(Deserialize)] -pub struct BrainFiringQuery { - pub token: Option, -} - -fn brain_event_to_json(event: &BrainFiringEvent) -> Value { - let mut obj = serde_json::Map::new(); - obj.insert( - "type".to_string(), - Value::String(event.kind.as_str().to_string()), - ); - obj.insert("ts".to_string(), Value::String(now_iso())); - if let Some(payload_obj) = event.payload.as_object() { - for (k, v) in payload_obj { - obj.insert(k.clone(), v.clone()); - } - } - if let Some(owner) = event.owner_id { - obj.insert("owner_id".to_string(), Value::from(owner)); - } - Value::Object(obj) -} - -pub async fn handle_brain_firing_stream( - State(state): State, - Query(query): Query, -) -> Response { - // Auth: token must match runtime token. Browser EventSource cannot send - // custom headers, so the token rides in the query string. - let provided = query.token.as_deref().unwrap_or(""); - if provided.is_empty() || !runtime_token_matches(provided, &state) { - return json_response( - StatusCode::UNAUTHORIZED, - json!({ "error": "Unauthorized" }), - ) - .into_response(); - } - - // Owner scoping: in single-user mode, the caller is the default owner. - // Team mode resolution is out of scope for v1. - let caller_owner_id = state.default_owner_id; - - let connected = stream::once(async move { - Ok::( - Event::default() - .event("connected") - .data(json!({"type":"connected","timestamp":now_iso()}).to_string()), - ) - }); - - // Coalesce: collect events into a 50ms window then emit as a single - // brain_batch SSE message whose data is a JSON array. - let receiver = state.brain_firing.subscribe(); - let event_stream = BroadcastStream::new(receiver); - - let batch_window = StdDuration::from_millis(50); - let buffered = futures_util::stream::unfold( - ( - event_stream, - Vec::::new(), - caller_owner_id, - ), - move |(mut events, mut buf, owner)| async move { - // Wait for first event, then collect all that arrive within the window. - let first = match events.next().await { - Some(Ok(ev)) => ev, - Some(Err(_)) => return None, - None => return None, - }; - - // Owner filter — fail-closed if caller has no resolved owner_id. - if owner.is_some() && first.owner_id == owner { - buf.push(first); - } else if owner.is_none() { - // No owner resolution available; drop everything to fail-closed. - } else if first.owner_id.is_none() { - // Event has no owner — never leak. - } - - let deadline = tokio::time::sleep(batch_window); - tokio::pin!(deadline); - - loop { - tokio::select! { - _ = &mut deadline => break, - next = events.next() => { - match next { - Some(Ok(ev)) => { - if owner.is_some() && ev.owner_id == owner { - buf.push(ev); - } - } - Some(Err(_)) | None => break, - } - } - } - } - - if buf.is_empty() { - // Continue without emitting (no owner-matching events in window). - Some((None, (events, Vec::new(), owner))) - } else { - let array: Vec = buf.iter().map(brain_event_to_json).collect(); - buf.clear(); - Some((Some(Value::Array(array)), (events, buf, owner))) - } - }, - ) - .filter_map(|item: Option| async move { - item.map(|payload| { - Ok::( - Event::default() - .event("brain_batch") - .data(payload.to_string()), - ) - }) - }); - - let stream = connected.chain(buffered); - let sse = Sse::new(stream).keep_alive( - KeepAlive::new() - .interval(StdDuration::from_secs(30)) - .text("keepalive"), - ); - sse.into_response() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::state::{BrainFiringEvent, BrainKind}; - - #[test] - fn scrub_event_payload_only_exposes_type_and_timestamp() { - let payload = scrub_event_payload("task"); - let object = payload.as_object().expect("payload object"); - - assert_eq!( - object.get("type").and_then(|value| value.as_str()), - Some("task") - ); - assert!(object - .get("timestamp") - .and_then(|value| value.as_str()) - .is_some()); - assert_eq!(object.len(), 2); - } - - #[test] - fn brain_event_to_json_includes_kind_and_payload_fields() { - let event = BrainFiringEvent { - kind: BrainKind::ClusterFinalized, - payload: json!({"cluster_id": 42, "member_count": 7}), - owner_id: Some(1), - }; - let v = brain_event_to_json(&event); - let obj = v.as_object().expect("object"); - assert_eq!( - obj.get("type").and_then(|v| v.as_str()), - Some("cluster_finalized") - ); - assert_eq!(obj.get("cluster_id").and_then(|v| v.as_i64()), Some(42)); - assert_eq!(obj.get("member_count").and_then(|v| v.as_i64()), Some(7)); - assert_eq!(obj.get("owner_id").and_then(|v| v.as_i64()), Some(1)); - assert!(obj.get("ts").and_then(|v| v.as_str()).is_some()); - } -} diff --git a/daemon-rs/src/handlers/events/mod.rs b/daemon-rs/src/handlers/events/mod.rs new file mode 100644 index 00000000..9248ffb7 --- /dev/null +++ b/daemon-rs/src/handlers/events/mod.rs @@ -0,0 +1,102 @@ +use super::{ensure_events_stream_auth, json_response, now_iso, runtime_token_matches}; +use crate::state::{BrainFiringEvent, RuntimeState}; +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use futures_util::stream::{self, StreamExt}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::convert::Infallible; +use std::time::Duration as StdDuration; +use tokio_stream::wrappers::BroadcastStream; +fn scrub_event_payload(event_type: &str) -> Value { + json!({"type":event_type,"timestamp":now_iso()}) +} +#[derive(Deserialize)] +pub struct EventsStreamQuery { + pub token: Option, +} +pub async fn handle_events_stream(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { + if let Err(resp) = ensure_events_stream_auth(&headers, query.token.as_deref(), &state).await { + return resp; + } + let initial = stream::once(async move { Ok::(Event::default().event("connected").data(scrub_event_payload("connected").to_string())) }); + let updates = BroadcastStream::new(state.events.subscribe()).filter_map(|msg| async move { + match msg { + Ok(event) => { + let payload = scrub_event_payload(&event.event_type); + Some(Ok::(Event::default().event(&event.event_type).data(payload.to_string()))) + } + Err(_) => None, + } + }); + let stream = initial.chain(updates); + let sse = Sse::new(stream).keep_alive(KeepAlive::new().interval(StdDuration::from_secs(30)).text("keepalive")); + sse.into_response() +} +#[derive(Deserialize)] +pub struct BrainFiringQuery { + pub token: Option, +} +fn brain_event_to_json(event: &BrainFiringEvent) -> Value { + let mut obj = serde_json::Map::new(); + obj.insert("type".to_string(), Value::String(event.kind.as_str().to_string())); + obj.insert("ts".to_string(), Value::String(now_iso())); + if let Some(payload_obj) = event.payload.as_object() { + for (k, v) in payload_obj { + obj.insert(k.clone(), v.clone()); + } + } + if let Some(owner) = event.owner_id { + obj.insert("owner_id".to_string(), Value::from(owner)); + } + Value::Object(obj) +} +pub async fn handle_brain_firing_stream(State(state): State, Query(query): Query) -> Response { + let provided = query.token.as_deref().unwrap_or(""); + if provided.is_empty() || !runtime_token_matches(provided, &state) { + return json_response(StatusCode::UNAUTHORIZED, json!({"error":"Unauthorized"})).into_response(); + } + let caller_owner_id = state.default_owner_id; + let connected = stream::once(async move { + Ok::(Event::default().event("connected").data(json!({"type":"connected","timestamp":now_iso()}).to_string())) + }); + let receiver = state.brain_firing.subscribe(); + let event_stream = BroadcastStream::new(receiver); + let batch_window = StdDuration::from_millis(50); + let buffered = + futures_util::stream::unfold((event_stream, Vec::::new(), caller_owner_id), move |(mut events, mut buf, owner)| async move { + let first = match events.next().await { + Some(Ok(ev)) => ev, + Some(Err(_)) => return None, + None => return None, + }; + if owner.is_some() && first.owner_id == owner { + buf.push(first); + } else if owner.is_none() { + } else if first.owner_id.is_none() { + } + let deadline = tokio::time::sleep(batch_window); + tokio::pin!(deadline); + loop { + tokio::select! {_=&mut deadline=>break + ,next=events.next()=>{match next{Some(Ok(ev))=>{if owner.is_some()&&ev.owner_id==owner{buf.push(ev);}}Some(Err(_))|None=>break,}}} + } + if buf.is_empty() { + Some((None, (events, Vec::new(), owner))) + } else { + let array: Vec = buf.iter().map(brain_event_to_json).collect(); + buf.clear(); + Some((Some(Value::Array(array)), (events, buf, owner))) + } + }) + .filter_map(|item: Option| async move { + item.map(|payload| Ok::(Event::default().event("brain_batch").data(payload.to_string()))) + }); + let stream = connected.chain(buffered); + let sse = Sse::new(stream).keep_alive(KeepAlive::new().interval(StdDuration::from_secs(30)).text("keepalive")); + sse.into_response() +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/handlers/events/tests/mod.rs b/daemon-rs/src/handlers/events/tests/mod.rs new file mode 100644 index 00000000..b6ad2cc0 --- /dev/null +++ b/daemon-rs/src/handlers/events/tests/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use crate::state::{BrainFiringEvent, BrainKind}; +#[test] +fn scrub_event_payload_only_exposes_type_and_timestamp() { + let payload = scrub_event_payload("task"); + let object = payload.as_object().expect("payload object"); + assert_eq!(object.get("type").and_then(|value| value.as_str()), Some("task")); + assert!(object.get("timestamp").and_then(|value| value.as_str()).is_some()); + assert_eq!(object.len(), 2); +} +#[test] +fn brain_event_to_json_includes_kind_and_payload_fields() { + let event = BrainFiringEvent { + kind: BrainKind::ClusterFinalized, + payload: json!({"cluster_id": 42, "member_count": 7}), + owner_id: Some(1), + }; + let v = brain_event_to_json(&event); + let obj = v.as_object().expect("object"); + assert_eq!(obj.get("type").and_then(|v| v.as_str()), Some("cluster_finalized")); + assert_eq!(obj.get("cluster_id").and_then(|v| v.as_i64()), Some(42)); + assert_eq!(obj.get("member_count").and_then(|v| v.as_i64()), Some(7)); + assert_eq!(obj.get("owner_id").and_then(|v| v.as_i64()), Some(1)); + assert!(obj.get("ts").and_then(|v| v.as_str()).is_some()); +} diff --git a/daemon-rs/src/handlers/export.rs b/daemon-rs/src/handlers/export.rs index e7d09a87..163dbda9 100644 --- a/daemon-rs/src/handlers/export.rs +++ b/daemon-rs/src/handlers/export.rs @@ -1,24 +1,13 @@ -// SPDX-License-Identifier: MIT -//! Export and import handlers. -//! -//! GET /export?format=json|sql -- dump all active memories + decisions -//! POST /import -- restore from a JSON export payload - +use super::{ensure_auth_rated, json_response}; +use crate::api_types::{ExportFormat, ImportOptions, ImportPayload}; +use crate::export_data::{export_json_page_value, import_payload as import_data, DEFAULT_EXPORT_PAGE_LIMIT, MAX_EXPORT_PAGE_LIMIT}; +use crate::state::RuntimeState; use axum::extract::{Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use serde::Deserialize; use serde_json::json; - -use super::{ensure_auth_rated, json_response}; -use crate::api_types::{ExportFormat, ImportOptions, ImportPayload}; -use crate::export_data::{ - export_json_page_value, import_payload as import_data, DEFAULT_EXPORT_PAGE_LIMIT, - MAX_EXPORT_PAGE_LIMIT, -}; -use crate::state::RuntimeState; - #[derive(Deserialize)] pub struct ExportQuery { pub format: Option, @@ -27,50 +16,30 @@ pub struct ExportQuery { pub memories_offset: Option, pub decisions_offset: Option, } - -pub async fn handle_export( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { +pub async fn handle_export(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db_read.lock().await; - match query.format.unwrap_or(ExportFormat::Json) { ExportFormat::Json => { - let limit = query - .limit - .unwrap_or(DEFAULT_EXPORT_PAGE_LIMIT) - .clamp(1, MAX_EXPORT_PAGE_LIMIT); + let limit = query.limit.unwrap_or(DEFAULT_EXPORT_PAGE_LIMIT).clamp(1, MAX_EXPORT_PAGE_LIMIT); let offset = query.offset.unwrap_or(0); let memories_offset = query.memories_offset.unwrap_or(offset); let decisions_offset = query.decisions_offset.unwrap_or(offset); - json_response( - StatusCode::OK, - export_json_page_value(&conn, limit, memories_offset, decisions_offset), - ) + json_response(StatusCode::OK, export_json_page_value(&conn, limit, memories_offset, decisions_offset)) } ExportFormat::Sql => json_response( StatusCode::BAD_REQUEST, - json!({ - "error": "HTTP SQL export is disabled because it requires a full in-memory export; use the CLI export command instead" - }), + json!({"error": +"HTTP SQL export is disabled because it requires a full in-memory export; use the CLI export command instead"}), ), } } - -pub async fn handle_import( - State(state): State, - headers: HeaderMap, - Json(payload): Json, -) -> Response { +pub async fn handle_import(State(state): State, headers: HeaderMap, Json(payload): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let mut conn = state.db.lock().await; let options = if state.team_mode { ImportOptions { @@ -79,27 +48,14 @@ pub async fn handle_import( source_agent_fallback: "import-http".to_string(), } } else { - ImportOptions { - source_agent_fallback: "import-http".to_string(), - ..ImportOptions::default() - } + ImportOptions { source_agent_fallback: "import-http".to_string(), ..ImportOptions::default() } }; match import_data(&mut conn, &payload, &options) { Ok(counts) => json_response( StatusCode::OK, - json!({ - "imported": { - "memories": counts.memories, - "decisions": counts.decisions, - } - }), - ), - Err(detail) => json_response( - StatusCode::BAD_REQUEST, - json!({ - "error": "import failed", - "detail": detail, - }), + json!({"imported":{"memories":counts.memories,"decisions":counts. +decisions,}}), ), + Err(detail) => json_response(StatusCode::BAD_REQUEST, json!({"error":"import failed","detail":detail,})), } } diff --git a/daemon-rs/src/handlers/feed.rs b/daemon-rs/src/handlers/feed.rs deleted file mode 100644 index e6e1fe02..00000000 --- a/daemon-rs/src/handlers/feed.rs +++ /dev/null @@ -1,725 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{Duration, Utc}; -use rusqlite::{params, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use uuid::Uuid; - -use super::{ - ensure_auth_with_caller_rated, json_response, now_iso, parse_duration_to_seconds, - parse_json_array, redact_secrets, -}; -use crate::db::checkpoint_wal_best_effort; -use crate::state::RuntimeState; - -// ─── Constants ────────────────────────────────────────────────────────────── - -const MAX_FEED: i64 = 200; -const FEED_TTL_SECONDS: i64 = 4 * 60 * 60; - -#[allow(clippy::result_large_err)] -fn owner_id_from_request( - state: &RuntimeState, - caller_id: Option, -) -> Result, Response> { - if state.team_mode { - match caller_id { - Some(owner_id) => Ok(Some(owner_id)), - None => Err(json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - )), - } - } else { - Ok(None) - } -} - -// ─── Internal feed entry type ─────────────────────────────────────────────── - -#[derive(Clone)] -struct FeedEntry { - id: String, - agent: String, - kind: String, - summary: String, - content: Option, - files: Value, - task_id: Option, - trace_id: Option, - priority: String, - timestamp: String, - tokens: i64, -} - -// ─── Request / query types ────────────────────────────────────────────────── - -#[derive(Deserialize, Default)] -pub struct FeedRequest { - pub agent: Option, - pub kind: Option, - pub summary: Option, - pub content: Option, - pub files: Option>, - #[serde(rename = "taskId")] - pub task_id: Option, - #[serde(rename = "traceId")] - pub trace_id: Option, - pub priority: Option, -} - -#[derive(Deserialize, Default)] -pub struct FeedQuery { - pub since: Option, - pub kind: Option, - pub agent: Option, - pub unread: Option, -} - -#[derive(Deserialize, Default)] -pub struct FeedAckRequest { - pub agent: Option, - #[serde(rename = "lastSeenId")] - pub last_seen_id: Option, -} - -// ─── Shared helpers ───────────────────────────────────────────────────────── - -fn feed_entry_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(FeedEntry { - id: row.get(0)?, - agent: row.get(1)?, - kind: row.get(2)?, - summary: row.get(3)?, - content: row.get(4)?, - files: parse_json_array(&row.get::<_, String>(5)?), - task_id: row.get(6)?, - trace_id: row.get(7)?, - priority: row.get(8)?, - timestamp: row.get(9)?, - tokens: row.get(10)?, - }) -} - -fn feed_to_json(entry: &FeedEntry, include_content: bool) -> Value { - if include_content { - json!({ - "id": entry.id, - "agent": entry.agent, - "kind": entry.kind, - "summary": entry.summary, - "content": entry.content, - "files": entry.files, - "taskId": entry.task_id, - "traceId": entry.trace_id, - "priority": entry.priority, - "timestamp": entry.timestamp, - "tokens": entry.tokens - }) - } else { - json!({ - "id": entry.id, - "agent": entry.agent, - "kind": entry.kind, - "summary": entry.summary, - "files": entry.files, - "taskId": entry.task_id, - "traceId": entry.trace_id, - "priority": entry.priority, - "timestamp": entry.timestamp, - "tokens": entry.tokens - }) - } -} - -// ─── Cleanup helpers ──────────────────────────────────────────────────────── - -fn clean_old_feed(conn: &rusqlite::Connection, owner_id: Option) -> rusqlite::Result<()> { - let cutoff = (Utc::now() - Duration::seconds(FEED_TTL_SECONDS)).to_rfc3339(); - if let Some(owner_id) = owner_id { - conn.execute( - "DELETE FROM feed WHERE owner_id = ?1 AND timestamp < ?2", - params![owner_id, cutoff], - )?; - conn.execute( - "DELETE FROM feed - WHERE owner_id = ?1 - AND id IN ( - SELECT id - FROM feed - WHERE owner_id = ?1 - ORDER BY timestamp DESC - LIMIT -1 OFFSET ?2 - )", - params![owner_id, MAX_FEED], - )?; - } else { - conn.execute("DELETE FROM feed WHERE timestamp < ?1", params![cutoff])?; - conn.execute( - "DELETE FROM feed - WHERE id IN ( - SELECT id - FROM feed - ORDER BY timestamp DESC - LIMIT -1 OFFSET ?1 - )", - params![MAX_FEED], - )?; - } - Ok(()) -} - -// ─── Fetch helpers ────────────────────────────────────────────────────────── - -fn fetch_feed_since( - conn: &rusqlite::Connection, - cutoff: &str, - owner_id: Option, -) -> Result, String> { - let (sql, params_vec): (&str, Vec>) = if let Some(owner_id) = - owner_id - { - ( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed WHERE owner_id = ?1 AND timestamp >= ?2 ORDER BY timestamp ASC", - vec![Box::new(owner_id), Box::new(cutoff.to_string())], - ) - } else { - ( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed WHERE timestamp >= ?1 ORDER BY timestamp ASC", - vec![Box::new(cutoff.to_string())], - ) - }; - let param_refs: Vec<&dyn rusqlite::types::ToSql> = - params_vec.iter().map(|p| p.as_ref()).collect(); - let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; - let rows = stmt - .query_map(rusqlite::params_from_iter(param_refs), feed_entry_from_row) - .map_err(|e| e.to_string())?; - let mut out = Vec::new(); - for row in rows.flatten() { - out.push(row); - } - Ok(out) -} - -fn fetch_recent_non_self_feed( - conn: &rusqlite::Connection, - for_agent: &str, - owner_id: Option, -) -> Result, String> { - let mut stmt = if owner_id.is_some() { - conn.prepare( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM ( - SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed - WHERE owner_id = ?1 AND agent != ?2 - ORDER BY timestamp DESC - LIMIT ?3 - ) - ORDER BY timestamp ASC", - ) - } else { - conn.prepare( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM ( - SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed - WHERE agent != ?1 - ORDER BY timestamp DESC - LIMIT ?2 - ) - ORDER BY timestamp ASC", - ) - } - .map_err(|e| e.to_string())?; - - let rows = if let Some(owner_id) = owner_id { - stmt.query_map(params![owner_id, for_agent, MAX_FEED], feed_entry_from_row) - } else { - stmt.query_map(params![for_agent, MAX_FEED], feed_entry_from_row) - } - .map_err(|e| e.to_string())?; - - let mut out = Vec::new(); - for row in rows.flatten() { - out.push(row); - } - Ok(out) -} - -fn fetch_unread_since_anchor( - conn: &rusqlite::Connection, - for_agent: &str, - owner_id: Option, - anchor_timestamp: &str, - anchor_id: &str, -) -> Result, String> { - let mut stmt = if owner_id.is_some() { - conn.prepare( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed - WHERE owner_id = ?1 - AND agent != ?2 - AND (timestamp > ?3 OR (timestamp = ?3 AND id > ?4)) - ORDER BY timestamp ASC", - ) - } else { - conn.prepare( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens - FROM feed - WHERE agent != ?1 - AND (timestamp > ?2 OR (timestamp = ?2 AND id > ?3)) - ORDER BY timestamp ASC", - ) - } - .map_err(|e| e.to_string())?; - - let rows = if let Some(owner_id) = owner_id { - stmt.query_map( - params![owner_id, for_agent, anchor_timestamp, anchor_id], - feed_entry_from_row, - ) - } else { - stmt.query_map( - params![for_agent, anchor_timestamp, anchor_id], - feed_entry_from_row, - ) - } - .map_err(|e| e.to_string())?; - - let mut out = Vec::new(); - for row in rows.flatten() { - out.push(row); - } - Ok(out) -} - -fn get_unread_feed( - conn: &rusqlite::Connection, - for_agent: &str, - owner_id: Option, -) -> Result, String> { - let ack = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT last_seen_id FROM feed_acks WHERE owner_id = ?1 AND agent = ?2", - params![owner_id, for_agent], - |row| row.get::<_, String>(0), - ) - .optional() - .map_err(|e| e.to_string())? - } else { - conn.query_row( - "SELECT last_seen_id FROM feed_acks WHERE agent = ?1", - params![for_agent], - |row| row.get::<_, String>(0), - ) - .optional() - .map_err(|e| e.to_string())? - }; - - let Some(ack_id) = ack else { - return fetch_recent_non_self_feed(conn, for_agent, owner_id); - }; - - let ack_timestamp: Option = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT timestamp FROM feed WHERE owner_id = ?1 AND id = ?2", - params![owner_id, ack_id.clone()], - |row| row.get(0), - ) - .optional() - .map_err(|e| e.to_string())? - } else { - conn.query_row( - "SELECT timestamp FROM feed WHERE id = ?1", - params![ack_id.clone()], - |row| row.get(0), - ) - .optional() - .map_err(|e| e.to_string())? - }; - - // Acks can outlive feed rows because the feed is TTL-pruned. If the saved - // anchor row no longer exists, fall back to the most recent non-self window. - let Some(anchor_timestamp) = ack_timestamp else { - return fetch_recent_non_self_feed(conn, for_agent, owner_id); - }; - - fetch_unread_since_anchor(conn, for_agent, owner_id, &anchor_timestamp, &ack_id) -} - -fn insert_feed_entry(conn: &rusqlite::Connection, entry: &FeedEntry) -> Result<(), String> { - conn.execute( - "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - entry.id, - entry.agent, - entry.kind, - entry.summary, - entry.content, - entry.files.to_string(), - entry.task_id, - entry.trace_id, - entry.priority, - entry.timestamp, - entry.tokens - ], - ) - .map_err(|e| e.to_string())?; - Ok(()) -} - -// ─── POST /feed ───────────────────────────────────────────────────────────── - -pub async fn handle_post_feed( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { - Ok(caller_id) => caller_id, - Err(resp) => return resp, - }; - let agent = match body.agent { - Some(v) if !v.trim().is_empty() => v.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: agent, kind, summary" }), - ); - } - }; - let kind = match body.kind { - Some(v) if !v.trim().is_empty() => v.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: agent, kind, summary" }), - ); - } - }; - let summary = match body.summary { - Some(v) if !v.trim().is_empty() => v, - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: agent, kind, summary" }), - ); - } - }; - - let entry = FeedEntry { - id: Uuid::new_v4().to_string(), - agent: agent.clone(), - kind: kind.clone(), - summary: redact_secrets(&summary), - content: body.content.map(|c| redact_secrets(&c)), - files: serde_json::to_value(body.files.unwrap_or_default()).unwrap_or_else(|_| json!([])), - task_id: body.task_id, - trace_id: body.trace_id, - priority: body.priority.unwrap_or_else(|| "normal".to_string()), - timestamp: now_iso(), - tokens: ((summary.len() as f64) / 4.0).ceil() as i64, - }; - - let owner_id = match owner_id_from_request(&state, caller_id) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - let conn = state.db.lock().await; - let _ = clean_old_feed(&conn, owner_id); - let inserted = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens, owner_id, visibility) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'team')", - params![ - entry.id.clone(), - entry.agent.clone(), - entry.kind.clone(), - entry.summary.clone(), - entry.content.clone(), - entry.files.to_string(), - entry.task_id.clone(), - entry.trace_id.clone(), - entry.priority.clone(), - entry.timestamp.clone(), - entry.tokens, - owner_id - ], - ) - .map(|_| ()) - .map_err(|e| e.to_string()) - } else { - insert_feed_entry(&conn, &entry) - }; - match inserted { - Ok(()) => { - checkpoint_wal_best_effort(&conn); - state.emit( - "feed", - json!({ "feedId": entry.id, "agent": agent, "kind": kind, "summary": entry.summary }), - ); - json_response( - StatusCode::CREATED, - json!({ "feedId": entry.id, "recorded": true }), - ) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Post feed failed: {err}") }), - ), - } -} - -// ─── GET /feed ────────────────────────────────────────────────────────────── - -pub async fn handle_get_feed( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { - Ok(caller_id) => caller_id, - Err(resp) => return resp, - }; - - let owner_id = match owner_id_from_request(&state, caller_id) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - let conn = state.db_read.lock().await; - let since = query.since.unwrap_or_else(|| "1h".to_string()); - let cutoff = (Utc::now() - Duration::seconds(parse_duration_to_seconds(&since))).to_rfc3339(); - - let mut entries = if query.unread.unwrap_or(false) { - if let Some(agent) = query.agent.as_deref() { - get_unread_feed(&conn, agent, owner_id).unwrap_or_default() - } else { - vec![] - } - } else { - fetch_feed_since(&conn, &cutoff, owner_id).unwrap_or_default() - }; - - if let Some(kind) = query.kind { - entries.retain(|e| e.kind == kind); - } - - let slim = entries - .iter() - .map(|e| feed_to_json(e, false)) - .collect::>(); - json_response(StatusCode::OK, json!({ "entries": slim })) -} - -// ─── GET /feed/{id} ───────────────────────────────────────────────────────── - -pub async fn handle_get_feed_by_id( - State(state): State, - headers: HeaderMap, - Path(feed_id): Path, -) -> Response { - let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { - Ok(caller_id) => caller_id, - Err(resp) => return resp, - }; - - let conn = state.db_read.lock().await; - let owner_id = match owner_id_from_request(&state, caller_id) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - let entry = if let Some(owner_id) = owner_id { - conn.query_row( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE owner_id = ?1 AND id = ?2", - params![owner_id, feed_id], - |row| { - Ok(FeedEntry { - id: row.get(0)?, - agent: row.get(1)?, - kind: row.get(2)?, - summary: row.get(3)?, - content: row.get(4)?, - files: parse_json_array(&row.get::<_, String>(5)?), - task_id: row.get(6)?, - trace_id: row.get(7)?, - priority: row.get(8)?, - timestamp: row.get(9)?, - tokens: row.get(10)?, - }) - }, - ) - .optional() - .ok() - .flatten() - } else { - conn.query_row( - "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE id = ?1", - params![feed_id], - |row| { - Ok(FeedEntry { - id: row.get(0)?, - agent: row.get(1)?, - kind: row.get(2)?, - summary: row.get(3)?, - content: row.get(4)?, - files: parse_json_array(&row.get::<_, String>(5)?), - task_id: row.get(6)?, - trace_id: row.get(7)?, - priority: row.get(8)?, - timestamp: row.get(9)?, - tokens: row.get(10)?, - }) - }, - ) - .optional() - .ok() - .flatten() - }; - - match entry { - Some(entry) => json_response(StatusCode::OK, feed_to_json(&entry, true)), - None => json_response( - StatusCode::NOT_FOUND, - json!({ "error": "feed_entry_not_found" }), - ), - } -} - -// ─── POST /feed/ack ───────────────────────────────────────────────────────── - -pub async fn handle_feed_ack( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { - Ok(caller_id) => caller_id, - Err(resp) => return resp, - }; - let agent = match body.agent { - Some(v) if !v.trim().is_empty() => v.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: agent, lastSeenId" }), - ); - } - }; - let last_seen_id = match body.last_seen_id { - Some(v) if !v.trim().is_empty() => v.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing required fields: agent, lastSeenId" }), - ); - } - }; - - let owner_id = match owner_id_from_request(&state, caller_id) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - let conn = state.db.lock().await; - let acked = if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO feed_acks (owner_id, agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(owner_id, agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", - params![owner_id, agent, last_seen_id, now_iso()], - ) - } else { - conn.execute( - "INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3) - ON CONFLICT(agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", - params![agent, last_seen_id, now_iso()], - ) - }; - match acked { - Ok(_) => { - checkpoint_wal_best_effort(&conn); - json_response(StatusCode::OK, json!({ "acked": true })) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Feed ack failed: {err}") }), - ), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rusqlite::Connection; - - fn setup_conn() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - conn - } - - fn insert_entry(conn: &Connection, id: &str, agent: &str, timestamp: &str) { - insert_feed_entry( - conn, - &FeedEntry { - id: id.to_string(), - agent: agent.to_string(), - kind: "task_complete".to_string(), - summary: format!("{agent} summary"), - content: None, - files: json!([]), - task_id: None, - trace_id: None, - priority: "medium".to_string(), - timestamp: timestamp.to_string(), - tokens: 1, - }, - ) - .unwrap(); - } - - #[test] - fn unread_feed_falls_back_when_ack_anchor_is_missing() { - let conn = setup_conn(); - insert_entry(&conn, "a1", "alpha", "2026-04-10T00:00:00Z"); - insert_entry(&conn, "b1", "beta", "2026-04-10T00:01:00Z"); - insert_entry(&conn, "g1", "gamma", "2026-04-10T00:02:00Z"); - - conn.execute( - "INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3)", - params!["alpha", "missing-anchor", "2026-04-10T00:03:00Z"], - ) - .unwrap(); - - let unread = get_unread_feed(&conn, "alpha", None).unwrap(); - let ids = unread.into_iter().map(|entry| entry.id).collect::>(); - assert_eq!(ids, vec!["b1".to_string(), "g1".to_string()]); - } - - #[test] - fn unread_feed_starts_after_ack_and_skips_self_entries() { - let conn = setup_conn(); - insert_entry(&conn, "a1", "alpha", "2026-04-10T00:00:00Z"); - insert_entry(&conn, "b1", "beta", "2026-04-10T00:01:00Z"); - insert_entry(&conn, "a2", "alpha", "2026-04-10T00:02:00Z"); - insert_entry(&conn, "g1", "gamma", "2026-04-10T00:03:00Z"); - - conn.execute( - "INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3)", - params!["alpha", "b1", "2026-04-10T00:04:00Z"], - ) - .unwrap(); - - let unread = get_unread_feed(&conn, "alpha", None).unwrap(); - let ids = unread.into_iter().map(|entry| entry.id).collect::>(); - assert_eq!(ids, vec!["g1".to_string()]); - } -} diff --git a/daemon-rs/src/handlers/feed/mod.rs b/daemon-rs/src/handlers/feed/mod.rs new file mode 100644 index 00000000..39d952b4 --- /dev/null +++ b/daemon-rs/src/handlers/feed/mod.rs @@ -0,0 +1,312 @@ +use super::{ensure_auth_with_caller_rated, json_response, now_iso, parse_duration_to_seconds, parse_json_array, redact_secrets, require_team_caller}; +use crate::db::checkpoint_wal_best_effort; +use crate::state::RuntimeState; +use axum::extract::{Path, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::Response; +use axum::Json; +use chrono::{Duration, Utc}; +use rusqlite::{params, OptionalExtension}; +use serde::Deserialize; +use serde_json::{json, Value}; +use uuid::Uuid; + +const MAX_FEED: i64 = 200; + +#[derive(Clone)] +struct FeedEntry { + id: String, + agent: String, + kind: String, + summary: String, + content: Option, + files: Value, + task_id: Option, + trace_id: Option, + priority: String, + timestamp: String, + tokens: i64, +} + +#[derive(Deserialize, Default)] +pub struct FeedRequest { + pub agent: Option, + pub kind: Option, + pub summary: Option, + pub content: Option, + pub files: Option>, + #[serde(rename = "taskId")] + pub task_id: Option, + #[serde(rename = "traceId")] + pub trace_id: Option, + pub priority: Option, +} + +#[derive(Deserialize, Default)] +pub struct FeedQuery { + pub since: Option, + pub kind: Option, + pub agent: Option, + pub unread: Option, +} + +#[derive(Deserialize, Default)] +pub struct FeedAckRequest { + pub agent: Option, + #[serde(rename = "lastSeenId")] + pub last_seen_id: Option, +} + +fn owner_id_from_request(state: &RuntimeState, caller_id: Option) -> Result, Response> { + require_team_caller(state, caller_id).map(|id| if state.team_mode { id } else { None }) +} + +fn feed_entry_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(FeedEntry { + id: row.get(0)?, + agent: row.get(1)?, + kind: row.get(2)?, + summary: row.get(3)?, + content: row.get(4)?, + files: parse_json_array(&row.get::<_, String>(5)?), + task_id: row.get(6)?, + trace_id: row.get(7)?, + priority: row.get(8)?, + timestamp: row.get(9)?, + tokens: row.get(10)?, + }) +} + +fn feed_to_json(entry: &FeedEntry, include_content: bool) -> Value { + let mut value = json!({"id":entry.id,"agent":entry.agent,"kind":entry.kind,"summary":entry.summary,"files":entry.files,"taskId":entry.task_id,"traceId":entry.trace_id,"priority":entry.priority,"timestamp":entry.timestamp,"tokens":entry.tokens}); + if include_content { + value["content"] = json!(entry.content); + } + value +} + +fn query_feed(conn: &rusqlite::Connection, sql: &str, params: &[&dyn rusqlite::types::ToSql]) -> Result, String> { + let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; + let rows = stmt.query_map(params, feed_entry_from_row).map_err(|e| e.to_string())?; + Ok(rows.filter_map(Result::ok).collect()) +} + +fn fetch_recent_non_self_feed(conn: &rusqlite::Connection, for_agent: &str, owner_id: Option) -> Result, String> { + if let Some(owner_id) = owner_id { + query_feed( + conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens + FROM feed WHERE owner_id = ?1 AND agent != ?2 ORDER BY timestamp ASC LIMIT ?3", + &[&owner_id, &for_agent, &MAX_FEED], + ) + } else { + query_feed( + conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens + FROM feed WHERE agent != ?1 ORDER BY timestamp ASC LIMIT ?2", + &[&for_agent, &MAX_FEED], + ) + } +} + +fn get_unread_feed(conn: &rusqlite::Connection, for_agent: &str, owner_id: Option) -> Result, String> { + let ack: Option = if let Some(owner_id) = owner_id { + conn.query_row("SELECT last_seen_id FROM feed_acks WHERE owner_id = ?1 AND agent = ?2", params![owner_id, for_agent], |row| row.get(0)) + } else { + conn.query_row("SELECT last_seen_id FROM feed_acks WHERE agent = ?1", params![for_agent], |row| row.get(0)) + } + .optional() + .map_err(|e| e.to_string())?; + let Some(ack_id) = ack else { + return fetch_recent_non_self_feed(conn, for_agent, owner_id); + }; + let anchor: Option = if let Some(owner_id) = owner_id { + conn.query_row("SELECT timestamp FROM feed WHERE owner_id = ?1 AND id = ?2", params![owner_id, ack_id.clone()], |row| row.get(0)) + } else { + conn.query_row("SELECT timestamp FROM feed WHERE id = ?1", params![ack_id.clone()], |row| row.get(0)) + } + .optional() + .map_err(|e| e.to_string())?; + let Some(anchor) = anchor else { + return fetch_recent_non_self_feed(conn, for_agent, owner_id); + }; + if let Some(owner_id) = owner_id { + query_feed( + conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens + FROM feed WHERE owner_id = ?1 AND agent != ?2 AND (timestamp > ?3 OR (timestamp = ?3 AND id > ?4)) ORDER BY timestamp ASC", + &[&owner_id, &for_agent, &anchor, &ack_id], + ) + } else { + query_feed( + conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens + FROM feed WHERE agent != ?1 AND (timestamp > ?2 OR (timestamp = ?2 AND id > ?3)) ORDER BY timestamp ASC", + &[&for_agent, &anchor, &ack_id], + ) + } +} + +fn insert_feed_entry(conn: &rusqlite::Connection, entry: &FeedEntry) -> Result<(), String> { + conn.execute( + "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![entry.id, entry.agent, entry.kind, entry.summary, entry.content, entry.files.to_string(), entry.task_id, entry.trace_id, entry.priority, entry.timestamp, entry.tokens], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) +} + +pub async fn handle_post_feed(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { + Ok(caller_id) => caller_id, + Err(resp) => return resp, + }; + let Some(agent) = body.agent.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing required fields: agent, kind, summary"})); + }; + let Some(kind) = body.kind.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing required fields: agent, kind, summary"})); + }; + let Some(summary) = body.summary.filter(|v| !v.trim().is_empty()) else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing required fields: agent, kind, summary"})); + }; + let owner_id = match owner_id_from_request(&state, caller_id) { + Ok(owner_id) => owner_id, + Err(resp) => return resp, + }; + let entry = FeedEntry { + id: Uuid::new_v4().to_string(), + agent: agent.clone(), + kind: kind.clone(), + summary: redact_secrets(&summary), + content: body.content.map(|c| redact_secrets(&c)), + files: serde_json::to_value(body.files.unwrap_or_default()).unwrap_or_else(|_| json!([])), + task_id: body.task_id, + trace_id: body.trace_id, + priority: body.priority.unwrap_or_else(|| "normal".to_string()), + timestamp: now_iso(), + tokens: ((summary.len() as f64) / 4.0).ceil() as i64, + }; + let conn = state.db.lock().await; + let inserted = if let Some(owner_id) = owner_id { + conn.execute( + "INSERT INTO feed (id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens, owner_id, visibility) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 'team')", + params![entry.id, entry.agent, entry.kind, entry.summary, entry.content, entry.files.to_string(), entry.task_id, entry.trace_id, entry.priority, entry.timestamp, entry.tokens, owner_id], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + } else { + insert_feed_entry(&conn, &entry) + }; + match inserted { + Ok(()) => { + checkpoint_wal_best_effort(&conn); + state.emit("feed", json!({"feedId":entry.id,"agent":agent,"kind":kind,"summary":entry.summary})); + json_response(StatusCode::CREATED, json!({"feedId":entry.id,"recorded":true})) + } + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Post feed failed: {err}")})), + } +} + +pub async fn handle_get_feed(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { + let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { + Ok(caller_id) => caller_id, + Err(resp) => return resp, + }; + let owner_id = match owner_id_from_request(&state, caller_id) { + Ok(owner_id) => owner_id, + Err(resp) => return resp, + }; + let conn = state.db_read.lock().await; + let mut entries = if query.unread.unwrap_or(false) { + query.agent.as_deref().map(|agent| get_unread_feed(&conn, agent, owner_id).unwrap_or_default()).unwrap_or_default() + } else { + let since = query.since.unwrap_or_else(|| "1h".to_string()); + let cutoff = (Utc::now() - Duration::seconds(parse_duration_to_seconds(&since))).to_rfc3339(); + if let Some(owner_id) = owner_id { + query_feed( + &conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE owner_id = ?1 AND timestamp >= ?2 ORDER BY timestamp ASC", + &[&owner_id, &cutoff], + ) + .unwrap_or_default() + } else { + query_feed( + &conn, + "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE timestamp >= ?1 ORDER BY timestamp ASC", + &[&cutoff], + ) + .unwrap_or_default() + } + }; + if let Some(kind) = query.kind { + entries.retain(|entry| entry.kind == kind); + } + json_response(StatusCode::OK, json!({"entries":entries.iter().map(|entry|feed_to_json(entry,false)).collect::>()})) +} + +pub async fn handle_get_feed_by_id(State(state): State, headers: HeaderMap, Path(feed_id): Path) -> Response { + let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { + Ok(caller_id) => caller_id, + Err(resp) => return resp, + }; + let owner_id = match owner_id_from_request(&state, caller_id) { + Ok(owner_id) => owner_id, + Err(resp) => return resp, + }; + let conn = state.db_read.lock().await; + let entry = if let Some(owner_id) = owner_id { + query_feed(&conn, "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE owner_id = ?1 AND id = ?2", &[&owner_id, &feed_id]) + } else { + query_feed(&conn, "SELECT id, agent, kind, summary, content, files_json, task_id, trace_id, priority, timestamp, tokens FROM feed WHERE id = ?1", &[&feed_id]) + } + .ok() + .and_then(|mut rows| rows.pop()); + match entry { + Some(entry) => json_response(StatusCode::OK, feed_to_json(&entry, true)), + None => json_response(StatusCode::NOT_FOUND, json!({"error":"feed_entry_not_found"})), + } +} + +pub async fn handle_feed_ack(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { + Ok(caller_id) => caller_id, + Err(resp) => return resp, + }; + let Some(agent) = body.agent.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing required fields: agent, lastSeenId"})); + }; + let Some(last_seen_id) = body.last_seen_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing required fields: agent, lastSeenId"})); + }; + let owner_id = match owner_id_from_request(&state, caller_id) { + Ok(owner_id) => owner_id, + Err(resp) => return resp, + }; + let conn = state.db.lock().await; + let acked = if let Some(owner_id) = owner_id { + conn.execute( + "INSERT INTO feed_acks (owner_id, agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(owner_id, agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", + params![owner_id, agent, last_seen_id, now_iso()], + ) + } else { + conn.execute( + "INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3) + ON CONFLICT(agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", + params![agent, last_seen_id, now_iso()], + ) + }; + match acked { + Ok(_) => { + checkpoint_wal_best_effort(&conn); + json_response(StatusCode::OK, json!({"acked":true})) + } + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Feed ack failed: {err}")})), + } +} + +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/handlers/feed/tests/mod.rs b/daemon-rs/src/handlers/feed/tests/mod.rs new file mode 100644 index 00000000..e8f02a4a --- /dev/null +++ b/daemon-rs/src/handlers/feed/tests/mod.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use rusqlite::Connection; +fn setup_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + conn +} +fn insert_entry(conn: &Connection, id: &str, agent: &str, timestamp: &str) { + insert_feed_entry( + conn, + &FeedEntry { + id: id.to_string(), + agent: agent.to_string(), + kind: "task_complete".to_string(), + summary: format!("{agent} summary"), + content: None, + files: json!([]), + task_id: None, + trace_id: None, + priority: "medium".to_string(), + timestamp: timestamp.to_string(), + tokens: 1, + }, + ) + .unwrap(); +} +#[test] +fn unread_feed_falls_back_when_ack_anchor_is_missing() { + let conn = setup_conn(); + insert_entry(&conn, "a1", "alpha", "2026-04-10T00:00:00Z"); + insert_entry(&conn, "b1", "beta", "2026-04-10T00:01:00Z"); + insert_entry(&conn, "g1", "gamma", "2026-04-10T00:02:00Z"); + conn.execute("INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3)", params!["alpha", "missing-anchor", "2026-04-10T00:03:00Z"]) + .unwrap(); + let unread = get_unread_feed(&conn, "alpha", None).unwrap(); + let ids = unread.into_iter().map(|entry| entry.id).collect::>(); + assert_eq!(ids, vec!["b1".to_string(), "g1".to_string()]); +} +#[test] +fn unread_feed_starts_after_ack_and_skips_self_entries() { + let conn = setup_conn(); + insert_entry(&conn, "a1", "alpha", "2026-04-10T00:00:00Z"); + insert_entry(&conn, "b1", "beta", "2026-04-10T00:01:00Z"); + insert_entry(&conn, "a2", "alpha", "2026-04-10T00:02:00Z"); + insert_entry(&conn, "g1", "gamma", "2026-04-10T00:03:00Z"); + conn.execute("INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3)", params!["alpha", "b1", "2026-04-10T00:04:00Z"]) + .unwrap(); + let unread = get_unread_feed(&conn, "alpha", None).unwrap(); + let ids = unread.into_iter().map(|entry| entry.id).collect::>(); + assert_eq!(ids, vec!["g1".to_string()]); +} diff --git a/daemon-rs/src/handlers/feedback/agent.rs b/daemon-rs/src/handlers/feedback/agent.rs deleted file mode 100644 index b120d382..00000000 --- a/daemon-rs/src/handlers/feedback/agent.rs +++ /dev/null @@ -1,564 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Relevance Feedback Loop — learns which recalled results are actually useful. -//! -//! Signal sources: -//! - Explicit: cortex_unfold() calls → positive signal for unfolded sources -//! - Explicit: POST /feedback → caller reports useful/not-useful -//! -//! Feedback is stored in `recall_feedback` with the query embedding so future -//! recalls for similar queries can rerank based on what worked before. -//! -//! Reranking: boost = sum(signal * decay) for matching result_source, -//! where decay = exp(-age_days / 30). Capped at [-0.2, +0.3]. - -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; -use serde::Deserialize; -use serde_json::{json, Value}; - -use crate::handlers::{ensure_auth_rated, ensure_auth_with_caller_rated, json_error, json_response}; -use crate::embeddings; -use crate::state::RuntimeState; -use std::collections::HashMap; - -/// Default lookback window for agent outcome telemetry stats. -const AGENT_FEEDBACK_DEFAULT_HORIZON_DAYS: i64 = 30; -/// Default row cap for agent outcome telemetry stats. -const AGENT_FEEDBACK_DEFAULT_LIMIT: usize = 400; -/// Recency half-life used for reliability weighting. -const AGENT_FEEDBACK_DECAY_HALF_LIFE_DAYS: f64 = 21.0; - -// ─── Agent outcome telemetry (/agent-feedback*) ───────────────────────────── - -#[derive(Deserialize)] -pub struct AgentFeedbackRecordRequest { - pub agent: Option, - #[serde(alias = "taskClass")] - pub task_class: Option, - pub outcome: Option, - #[serde(alias = "outcomeScore")] - pub outcome_score: Option, - #[serde(alias = "qualityScore")] - pub quality_score: Option, - #[serde(alias = "latencyMs")] - pub latency_ms: Option, - pub retries: Option, - #[serde(alias = "tokensUsed")] - pub tokens_used: Option, - #[serde(alias = "memorySources")] - pub memory_sources: Option>, - pub notes: Option, -} - -#[derive(Deserialize)] -pub struct AgentFeedbackStatsQuery { - #[serde(alias = "horizonDays")] - pub horizon_days: Option, - pub limit: Option, - #[serde(alias = "taskClass")] - pub task_class: Option, - pub agent: Option, -} - -#[derive(Default, Clone)] -struct FeedbackAggregate { - count: i64, - weighted_sum: f64, - weight_total: f64, - success: i64, - partial: i64, - failure: i64, - latency_total: i64, - latency_count: i64, - retries_total: i64, - retries_count: i64, - tokens_total: i64, - tokens_count: i64, -} - -impl FeedbackAggregate { - #[allow(clippy::too_many_arguments)] - fn observe( - &mut self, - outcome: &str, - outcome_score: f64, - quality_score: f64, - age_days: f64, - latency_ms: Option, - retries: Option, - tokens_used: Option, - ) { - self.count += 1; - match outcome { - "success" => self.success += 1, - "partial" => self.partial += 1, - _ => self.failure += 1, - } - - let decay_lambda = (2.0f64).ln() / AGENT_FEEDBACK_DECAY_HALF_LIFE_DAYS; - let weight = (-decay_lambda * age_days.max(0.0)).exp(); - let blended = (outcome_score * 0.6 + quality_score * 0.4).clamp(0.0, 1.0); - self.weighted_sum += blended * weight; - self.weight_total += weight; - - if let Some(value) = latency_ms { - self.latency_total += value.max(0); - self.latency_count += 1; - } - if let Some(value) = retries { - self.retries_total += value.max(0); - self.retries_count += 1; - } - if let Some(value) = tokens_used { - self.tokens_total += value.max(0); - self.tokens_count += 1; - } - } - - fn reliability(&self) -> f64 { - if self.weight_total > 0.0 { - (self.weighted_sum / self.weight_total).clamp(0.0, 1.0) - } else { - 0.0 - } - } -} - -fn normalize_outcome(raw: Option<&str>) -> Option<&'static str> { - match raw.unwrap_or_default().trim().to_ascii_lowercase().as_str() { - "success" | "ok" | "pass" => Some("success"), - "partial" | "mixed" | "degraded" => Some("partial"), - "failure" | "fail" | "error" => Some("failure"), - _ => None, - } -} - -fn default_outcome_score(outcome: &str) -> f64 { - match outcome { - "success" => 1.0, - "partial" => 0.5, - _ => 0.0, - } -} - -fn normalize_task_class(value: Option<&str>) -> String { - value - .map(str::trim) - .filter(|v| !v.is_empty()) - .unwrap_or("general") - .to_ascii_lowercase() -} - -fn normalize_agent(value: Option<&str>, fallback_agent: &str) -> String { - value - .map(str::trim) - .filter(|v| !v.is_empty()) - .unwrap_or(fallback_agent) - .to_string() -} - -pub(crate) fn normalize_horizon_days(value: Option) -> i64 { - value - .unwrap_or(AGENT_FEEDBACK_DEFAULT_HORIZON_DAYS) - .clamp(1, 180) -} - -pub(crate) fn normalize_limit(value: Option) -> usize { - value - .unwrap_or(AGENT_FEEDBACK_DEFAULT_LIMIT) - .clamp(10, 2_000) -} - -fn arg_value_string(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_str())) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn arg_value_f64(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_f64())) -} - -fn arg_value_i64(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_i64())) -} - -fn arg_value_string_array(args: &Value, keys: &[&str]) -> Vec { - keys.iter() - .find_map(|key| { - args.get(*key).and_then(|value| { - value.as_array().map(|items| { - items - .iter() - .filter_map(|item| item.as_str().map(str::trim)) - .filter(|item| !item.is_empty()) - .map(str::to_string) - .collect::>() - }) - }) - }) - .unwrap_or_default() -} - -pub fn record_agent_feedback_from_value( - conn: &Connection, - owner_id: i64, - args: &Value, - fallback_agent: &str, -) -> Result { - let outcome = normalize_outcome( - arg_value_string(args, &["outcome"]) - .as_deref() - .or_else(|| args.get("outcome").and_then(|value| value.as_str())), - ) - .ok_or_else(|| "Missing or invalid outcome (expected success|partial|failure)".to_string())?; - let agent = normalize_agent( - arg_value_string(args, &["agent", "source_agent", "sourceAgent"]).as_deref(), - fallback_agent, - ); - let task_class = - normalize_task_class(arg_value_string(args, &["task_class", "taskClass"]).as_deref()); - let outcome_score = arg_value_f64(args, &["outcome_score", "outcomeScore"]) - .unwrap_or_else(|| default_outcome_score(outcome)) - .clamp(0.0, 1.0); - let quality_score = arg_value_f64(args, &["quality_score", "qualityScore"]) - .unwrap_or(0.7) - .clamp(0.0, 1.0); - let latency_ms = arg_value_i64(args, &["latency_ms", "latencyMs"]).map(|value| value.max(0)); - let retries = arg_value_i64(args, &["retries"]).map(|value| value.max(0)); - let tokens_used = arg_value_i64(args, &["tokens_used", "tokensUsed"]).map(|value| value.max(0)); - let memory_sources = arg_value_string_array(args, &["memory_sources", "memorySources"]); - let notes = arg_value_string(args, &["notes"]); - let memory_sources_json = - serde_json::to_string(&memory_sources).map_err(|err| err.to_string())?; - - conn.execute( - "INSERT INTO agent_feedback ( - owner_id, agent, task_class, outcome, outcome_score, quality_score, - latency_ms, retries, tokens_used, memory_sources_json, notes - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - owner_id, - agent, - task_class, - outcome, - outcome_score, - quality_score, - latency_ms, - retries, - tokens_used, - memory_sources_json, - notes - ], - ) - .map_err(|err| err.to_string())?; - - Ok(json!({ - "stored": true, - "ownerId": owner_id, - "agent": agent, - "taskClass": task_class, - "outcome": outcome, - "outcomeScore": outcome_score, - "qualityScore": quality_score, - "memorySources": memory_sources, - })) -} - -fn aggregate_summary_json(name: &str, agg: &FeedbackAggregate) -> Value { - json!({ - "name": name, - "count": agg.count, - "reliability": agg.reliability(), - "success": agg.success, - "partial": agg.partial, - "failure": agg.failure, - "avgLatencyMs": if agg.latency_count > 0 { Some(agg.latency_total as f64 / agg.latency_count as f64) } else { None }, - "avgRetries": if agg.retries_count > 0 { Some(agg.retries_total as f64 / agg.retries_count as f64) } else { None }, - "avgTokensUsed": if agg.tokens_count > 0 { Some(agg.tokens_total as f64 / agg.tokens_count as f64) } else { None }, - }) -} - -pub fn build_agent_feedback_stats_payload( - conn: &Connection, - owner_id: i64, - horizon_days: i64, - limit: usize, - task_class_filter: Option<&str>, - agent_filter: Option<&str>, -) -> Result { - let horizon_days = normalize_horizon_days(Some(horizon_days)); - let limit = normalize_limit(Some(limit)); - let task_filter = task_class_filter - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_ascii_lowercase); - let agent_filter = agent_filter - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); - - let mut stmt = conn - .prepare( - "SELECT agent, task_class, outcome, outcome_score, quality_score, - latency_ms, retries, tokens_used, memory_sources_json, - julianday('now') - julianday(created_at) AS age_days - FROM agent_feedback - WHERE owner_id = ?1 - AND julianday('now') - julianday(created_at) <= ?2 - AND (?3 IS NULL OR task_class = ?3) - AND (?4 IS NULL OR agent = ?4) - ORDER BY datetime(created_at) DESC, id DESC - LIMIT ?5", - ) - .map_err(|err| err.to_string())?; - - let rows = stmt - .query_map( - params![ - owner_id, - horizon_days, - task_filter, - agent_filter, - limit as i64 - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, f64>(3)?, - row.get::<_, f64>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - row.get::<_, f64>(9)?, - )) - }, - ) - .map_err(|err| err.to_string())?; - - let mut overall = FeedbackAggregate::default(); - let mut by_agent: HashMap = HashMap::new(); - let mut by_task: HashMap = HashMap::new(); - let mut source_counts: HashMap = HashMap::new(); - let mut rows_with_sources = 0i64; - - for row in rows.flatten() { - let ( - agent, - task_class, - outcome, - outcome_score, - quality_score, - latency_ms, - retries, - tokens_used, - memory_sources_json, - age_days, - ) = row; - - overall.observe( - &outcome, - outcome_score, - quality_score, - age_days, - latency_ms, - retries, - tokens_used, - ); - by_agent.entry(agent).or_default().observe( - &outcome, - outcome_score, - quality_score, - age_days, - latency_ms, - retries, - tokens_used, - ); - by_task.entry(task_class).or_default().observe( - &outcome, - outcome_score, - quality_score, - age_days, - latency_ms, - retries, - tokens_used, - ); - - let parsed_sources = memory_sources_json - .as_deref() - .and_then(|raw| serde_json::from_str::>(raw).ok()) - .unwrap_or_default(); - if !parsed_sources.is_empty() { - rows_with_sources += 1; - for source in parsed_sources { - *source_counts.entry(source).or_insert(0) += 1; - } - } - } - - let mut by_agent_vec: Vec = by_agent - .iter() - .map(|(name, agg)| aggregate_summary_json(name, agg)) - .collect(); - by_agent_vec.sort_by(|left, right| { - let left_rel = left - .get("reliability") - .and_then(|value| value.as_f64()) - .unwrap_or(0.0); - let right_rel = right - .get("reliability") - .and_then(|value| value.as_f64()) - .unwrap_or(0.0); - right_rel - .partial_cmp(&left_rel) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let mut by_task_vec: Vec = by_task - .iter() - .map(|(name, agg)| aggregate_summary_json(name, agg)) - .collect(); - by_task_vec.sort_by(|left, right| { - let left_count = left - .get("count") - .and_then(|value| value.as_i64()) - .unwrap_or(0); - let right_count = right - .get("count") - .and_then(|value| value.as_i64()) - .unwrap_or(0); - right_count.cmp(&left_count) - }); - - let mut top_sources: Vec<(String, i64)> = source_counts.into_iter().collect(); - top_sources.sort_by(|left, right| right.1.cmp(&left.1)); - let top_sources: Vec = top_sources - .into_iter() - .take(10) - .map(|(source, hits)| json!({ "source": source, "hits": hits })) - .collect(); - - let reliability = overall.reliability(); - let recommendation = if overall.count == 0 { - "No agent feedback telemetry recorded yet." - } else if reliability < 0.65 { - "Reliability is below target; tighten task decomposition and collect richer memory_sources." - } else if reliability < 0.8 { - "Reliability is stable but improvable; prioritize retries and conflict resolution on partial outcomes." - } else { - "Reliability is strong; continue reinforcing high-quality runs and memory-source coverage." - }; - - Ok(json!({ - "ownerId": owner_id, - "horizonDays": horizon_days, - "limit": limit, - "sampled": overall.count, - "reliability": reliability, - "outcomes": { - "success": overall.success, - "partial": overall.partial, - "failure": overall.failure, - }, - "averages": { - "latencyMs": if overall.latency_count > 0 { Some(overall.latency_total as f64 / overall.latency_count as f64) } else { None }, - "retries": if overall.retries_count > 0 { Some(overall.retries_total as f64 / overall.retries_count as f64) } else { None }, - "tokensUsed": if overall.tokens_count > 0 { Some(overall.tokens_total as f64 / overall.tokens_count as f64) } else { None }, - }, - "memorySourceCoverage": { - "rowsWithSources": rows_with_sources, - "ratio": if overall.count > 0 { rows_with_sources as f64 / overall.count as f64 } else { 0.0 }, - }, - "byAgent": by_agent_vec, - "byTaskClass": by_task_vec, - "topMemorySources": top_sources, - "recommendation": recommendation, - })) -} - -pub fn recommend_recall_k( - conn: &Connection, - owner_id: i64, - agent: &str, - task_class: Option<&str>, - base_k: usize, -) -> Result, String> { - let task_class = normalize_task_class(task_class).to_string(); - let mut stmt = conn - .prepare( - "SELECT outcome, quality_score - FROM agent_feedback - WHERE owner_id = ?1 - AND agent = ?2 - AND task_class = ?3 - AND julianday('now') - julianday(created_at) <= 30 - ORDER BY datetime(created_at) DESC, id DESC - LIMIT 40", - ) - .map_err(|err| err.to_string())?; - - let rows = stmt - .query_map(params![owner_id, agent, task_class], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?)) - }) - .map_err(|err| err.to_string())?; - - let mut success = 0usize; - let mut partial = 0usize; - let mut failure = 0usize; - let mut quality_total = 0.0f64; - let mut count = 0usize; - for (outcome, quality) in rows.flatten() { - count += 1; - quality_total += quality.clamp(0.0, 1.0); - match outcome.as_str() { - "success" => success += 1, - "partial" => partial += 1, - _ => failure += 1, - } - } - - if count < 8 { - return Ok(None); - } - - let failure_rate = failure as f64 / count as f64; - let partial_rate = partial as f64 / count as f64; - let success_rate = success as f64 / count as f64; - let avg_quality = quality_total / count as f64; - - let mut recommended_k = base_k; - let reason = if failure_rate >= 0.3 || partial_rate >= 0.45 { - recommended_k = (base_k + 4).min(24); - "raise_depth_for_recovery" - } else if success_rate >= 0.75 && avg_quality >= 0.82 { - recommended_k = base_k.saturating_sub(2).max(6); - "reduce_depth_for_efficiency" - } else { - "keep_depth_stable" - }; - - Ok(Some(json!({ - "agent": agent, - "taskClass": task_class, - "samples": count, - "baseK": base_k, - "recommendedK": recommended_k, - "reason": reason, - "successRate": success_rate, - "partialRate": partial_rate, - "failureRate": failure_rate, - "avgQuality": avg_quality, - }))) -} - diff --git a/daemon-rs/src/handlers/feedback/handlers.rs b/daemon-rs/src/handlers/feedback/handlers.rs index 7fe8174b..5e140ced 100644 --- a/daemon-rs/src/handlers/feedback/handlers.rs +++ b/daemon-rs/src/handlers/feedback/handlers.rs @@ -1,103 +1,45 @@ -// SPDX-License-Identifier: MIT -//! Relevance Feedback Loop — learns which recalled results are actually useful. -//! -//! Signal sources: -//! - Explicit: cortex_unfold() calls → positive signal for unfolded sources -//! - Explicit: POST /feedback → caller reports useful/not-useful -//! -//! Feedback is stored in `recall_feedback` with the query embedding so future -//! recalls for similar queries can rerank based on what worked before. -//! -//! Reranking: boost = sum(signal * decay) for matching result_source, -//! where decay = exp(-age_days / 30). Capped at [-0.2, +0.3]. - +use super::{ + build_agent_feedback_stats_payload, normalize_horizon_days, normalize_limit, record_agent_feedback_from_value, AgentFeedbackRecordRequest, + AgentFeedbackStatsQuery, +}; +use crate::handlers::{ensure_auth_with_caller_rated, json_error, json_response}; +use crate::state::RuntimeState; use axum::extract::{Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use serde_json::json; - -use crate::handlers::{ensure_auth_with_caller_rated, json_error, json_response}; -use crate::state::RuntimeState; - -use super::agent::{ - build_agent_feedback_stats_payload, normalize_horizon_days, normalize_limit, - record_agent_feedback_from_value, AgentFeedbackRecordRequest, AgentFeedbackStatsQuery, -}; - -pub async fn handle_agent_feedback_record( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_agent_feedback_record(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { Ok(caller_id) => caller_id, Err(resp) => return resp, }; if state.team_mode && caller_id.is_none() { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - ); + return json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"})); } - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - - let args = json!({ - "agent": body.agent, - "task_class": body.task_class, - "outcome": body.outcome, - "outcome_score": body.outcome_score, - "quality_score": body.quality_score, - "latency_ms": body.latency_ms, - "retries": body.retries, - "tokens_used": body.tokens_used, - "memory_sources": body.memory_sources.unwrap_or_default(), - "notes": body.notes, - }); - + let owner_id = if state.team_mode { caller_id.unwrap_or_default() } else { 0 }; + let args = json!({"agent":body.agent,"task_class":body.task_class,"outcome":body.outcome,"outcome_score" +:body.outcome_score,"quality_score":body.quality_score,"latency_ms":body.latency_ms,"retries":body.retries,"tokens_used":body. +tokens_used,"memory_sources":body.memory_sources.unwrap_or_default(),"notes":body.notes,}); let conn = state.db.lock().await; match record_agent_feedback_from_value(&conn, owner_id, &args, "http") { Ok(payload) => json_response(StatusCode::OK, payload), Err(err) => json_error(StatusCode::BAD_REQUEST, &err), } } - -pub async fn handle_agent_feedback_stats( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { +pub async fn handle_agent_feedback_stats(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { let caller_id = match ensure_auth_with_caller_rated(&headers, &state).await { Ok(caller_id) => caller_id, Err(resp) => return resp, }; if state.team_mode && caller_id.is_none() { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - ); + return json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"})); } - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - + let owner_id = if state.team_mode { caller_id.unwrap_or_default() } else { 0 }; let horizon_days = normalize_horizon_days(query.horizon_days); let limit = normalize_limit(query.limit); - let conn = state.db.lock().await; - match build_agent_feedback_stats_payload( - &conn, - owner_id, - horizon_days, - limit, - query.task_class.as_deref(), - query.agent.as_deref(), - ) { + let conn = state.db_read.lock().await; + match build_agent_feedback_stats_payload(&conn, owner_id, horizon_days, limit, query.task_class.as_deref(), query.agent.as_deref()) { Ok(payload) => json_response(StatusCode::OK, payload), Err(err) => json_error(StatusCode::BAD_REQUEST, &err), } diff --git a/daemon-rs/src/handlers/feedback/mod.rs b/daemon-rs/src/handlers/feedback/mod.rs index 2b28ca4e..d2e485c0 100644 --- a/daemon-rs/src/handlers/feedback/mod.rs +++ b/daemon-rs/src/handlers/feedback/mod.rs @@ -1,20 +1,288 @@ -// SPDX-License-Identifier: MIT -//! Relevance Feedback Loop — learns which recalled results are actually useful. - -mod recall; -mod agent; mod handlers; - +mod recall; #[cfg(test)] -mod tests {} - -pub use recall::{ - compute_boost, compute_boosts, has_retrieval_immunity, record_unfold_feedback, FeedbackRequest, - IMMUNITY_THRESHOLD, IMMUNITY_WINDOW_DAYS, -}; -pub use agent::{ - build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value, - AgentFeedbackRecordRequest, AgentFeedbackStatsQuery, -}; +mod tests; +use rusqlite::{params, Connection}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::collections::HashMap; + pub use handlers::{handle_agent_feedback_record, handle_agent_feedback_stats}; +pub use recall::{compute_boosts, has_retrieval_immunity}; pub use recall::{handle_feedback, handle_feedback_stats}; + +const AGENT_FEEDBACK_DEFAULT_HORIZON_DAYS: i64 = 30; +const AGENT_FEEDBACK_DEFAULT_LIMIT: usize = 400; +const AGENT_FEEDBACK_DECAY_HALF_LIFE_DAYS: f64 = 21.0; + +#[derive(Deserialize)] +pub struct AgentFeedbackRecordRequest { + pub agent: Option, + #[serde(alias = "taskClass")] + pub task_class: Option, + pub outcome: Option, + #[serde(alias = "outcomeScore")] + pub outcome_score: Option, + #[serde(alias = "qualityScore")] + pub quality_score: Option, + #[serde(alias = "latencyMs")] + pub latency_ms: Option, + pub retries: Option, + #[serde(alias = "tokensUsed")] + pub tokens_used: Option, + #[serde(alias = "memorySources")] + pub memory_sources: Option>, + pub notes: Option, +} + +#[derive(Deserialize)] +pub struct AgentFeedbackStatsQuery { + #[serde(alias = "horizonDays")] + pub horizon_days: Option, + pub limit: Option, + #[serde(alias = "taskClass")] + pub task_class: Option, + pub agent: Option, +} + +#[derive(Default, Clone)] +struct AgentFeedbackAggregate { + count: i64, + weighted_sum: f64, + weight_total: f64, + success: i64, + partial: i64, + failure: i64, + latency: (i64, i64), + retries: (i64, i64), + tokens: (i64, i64), +} + +impl AgentFeedbackAggregate { + fn observe( + &mut self, outcome: &str, outcome_score: f64, quality_score: f64, age_days: f64, latency_ms: Option, retries: Option, + tokens_used: Option, + ) { + self.count += 1; + match outcome { + "success" => self.success += 1, + "partial" => self.partial += 1, + _ => self.failure += 1, + } + let weight = (-((2.0f64).ln() / AGENT_FEEDBACK_DECAY_HALF_LIFE_DAYS) * age_days.max(0.0)).exp(); + self.weighted_sum += (outcome_score * 0.6 + quality_score * 0.4).clamp(0.0, 1.0) * weight; + self.weight_total += weight; + observe_optional(latency_ms, &mut self.latency); + observe_optional(retries, &mut self.retries); + observe_optional(tokens_used, &mut self.tokens); + } + + fn reliability(&self) -> f64 { + if self.weight_total > 0.0 { (self.weighted_sum / self.weight_total).clamp(0.0, 1.0) } else { 0.0 } + } +} + +fn observe_optional(value: Option, acc: &mut (i64, i64)) { + if let Some(value) = value { + acc.0 += value.max(0); + acc.1 += 1; + } +} + +fn avg(acc: (i64, i64)) -> Option { + (acc.1 > 0).then_some(acc.0 as f64 / acc.1 as f64) +} + +fn normalize_outcome(raw: Option<&str>) -> Option<&'static str> { + match raw.unwrap_or_default().trim().to_ascii_lowercase().as_str() { + "success" | "ok" | "pass" => Some("success"), + "partial" | "mixed" | "degraded" => Some("partial"), + "failure" | "fail" | "error" => Some("failure"), + _ => None, + } +} + +fn normalize_task_class(value: Option<&str>) -> String { + value.map(str::trim).filter(|value| !value.is_empty()).unwrap_or("general").to_ascii_lowercase() +} + +fn normalize_agent(value: Option<&str>, fallback_agent: &str) -> String { + value.map(str::trim).filter(|value| !value.is_empty()).unwrap_or(fallback_agent).to_string() +} + +pub(crate) fn normalize_horizon_days(value: Option) -> i64 { + value.unwrap_or(AGENT_FEEDBACK_DEFAULT_HORIZON_DAYS).clamp(1, 180) +} + +pub(crate) fn normalize_limit(value: Option) -> usize { + value.unwrap_or(AGENT_FEEDBACK_DEFAULT_LIMIT).clamp(10, 2_000) +} + +fn value_str<'a>(args: &'a Value, keys: &[&str]) -> Option<&'a str> { + keys.iter().find_map(|key| args.get(*key)?.as_str()).map(str::trim).filter(|value| !value.is_empty()) +} + +fn value_f64(args: &Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| args.get(*key)?.as_f64()) +} + +fn value_i64(args: &Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| args.get(*key)?.as_i64()) +} + +fn value_string_array(args: &Value, keys: &[&str]) -> Vec { + keys.iter() + .find_map(|key| args.get(*key)?.as_array()) + .into_iter() + .flatten() + .filter_map(|item| item.as_str().map(str::trim).filter(|value| !value.is_empty()).map(str::to_string)) + .collect() +} + +pub fn record_agent_feedback_from_value(conn: &Connection, owner_id: i64, args: &Value, fallback_agent: &str) -> Result { + let outcome = normalize_outcome(value_str(args, &["outcome"])).ok_or_else(|| "Missing or invalid outcome (expected success|partial|failure)".to_string())?; + let agent = normalize_agent(value_str(args, &["agent", "source_agent", "sourceAgent"]), fallback_agent); + let task_class = normalize_task_class(value_str(args, &["task_class", "taskClass"])); + let outcome_score = value_f64(args, &["outcome_score", "outcomeScore"]) + .unwrap_or(match outcome { "success" => 1.0, "partial" => 0.5, _ => 0.0 }) + .clamp(0.0, 1.0); + let quality_score = value_f64(args, &["quality_score", "qualityScore"]).unwrap_or(0.7).clamp(0.0, 1.0); + let latency_ms = value_i64(args, &["latency_ms", "latencyMs"]).map(|value| value.max(0)); + let retries = value_i64(args, &["retries"]).map(|value| value.max(0)); + let tokens_used = value_i64(args, &["tokens_used", "tokensUsed"]).map(|value| value.max(0)); + let memory_sources = value_string_array(args, &["memory_sources", "memorySources"]); + let notes = value_str(args, &["notes"]).map(str::to_string); + let memory_sources_json = serde_json::to_string(&memory_sources).map_err(|err| err.to_string())?; + conn.execute( + "INSERT INTO agent_feedback (owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, memory_sources_json, notes) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![owner_id, agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, memory_sources_json, notes], + ) + .map_err(|err| err.to_string())?; + Ok(json!({"stored":true,"ownerId":owner_id,"agent":agent,"taskClass":task_class,"outcome":outcome, + "outcomeScore":outcome_score,"qualityScore":quality_score,"memorySources":memory_sources})) +} + +fn aggregate_json(name: &str, agg: &AgentFeedbackAggregate) -> Value { + json!({"name":name,"count":agg.count,"reliability":agg.reliability(),"success":agg.success,"partial":agg.partial,"failure":agg.failure, + "avgLatencyMs":avg(agg.latency),"avgRetries":avg(agg.retries),"avgTokensUsed":avg(agg.tokens)}) +} + +pub fn build_agent_feedback_stats_payload( + conn: &Connection, owner_id: i64, horizon_days: i64, limit: usize, task_class_filter: Option<&str>, agent_filter: Option<&str>, +) -> Result { + let horizon_days = normalize_horizon_days(Some(horizon_days)); + let limit = normalize_limit(Some(limit)); + let task_filter = task_class_filter.map(str::trim).filter(|value| !value.is_empty()).map(str::to_ascii_lowercase); + let agent_filter = agent_filter.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string); + let mut stmt = conn + .prepare( + "SELECT agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, memory_sources_json, + julianday('now') - julianday(created_at) + FROM agent_feedback + WHERE owner_id = ?1 AND julianday('now') - julianday(created_at) <= ?2 + AND (?3 IS NULL OR task_class = ?3) AND (?4 IS NULL OR agent = ?4) + ORDER BY datetime(created_at) DESC, id DESC LIMIT ?5", + ) + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map(params![owner_id, horizon_days, task_filter, agent_filter, limit as i64], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, f64>(3)?, + row.get::<_, f64>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, f64>(9)?, + )) + }) + .map_err(|err| err.to_string())?; + let mut overall = AgentFeedbackAggregate::default(); + let mut by_agent = HashMap::::new(); + let mut by_task = HashMap::::new(); + let mut source_counts = HashMap::::new(); + let mut rows_with_sources = 0; + for row in rows.flatten() { + let (agent, task_class, outcome, outcome_score, quality_score, latency_ms, retries, tokens_used, sources_json, age_days) = row; + for agg in [&mut overall, by_agent.entry(agent).or_default(), by_task.entry(task_class).or_default()] { + agg.observe(&outcome, outcome_score, quality_score, age_days, latency_ms, retries, tokens_used); + } + let sources = sources_json.and_then(|raw| serde_json::from_str::>(&raw).ok()).unwrap_or_default(); + if !sources.is_empty() { + rows_with_sources += 1; + for source in sources { + *source_counts.entry(source).or_default() += 1; + } + } + } + let mut by_agent_vec: Vec<_> = by_agent.iter().map(|(name, agg)| aggregate_json(name, agg)).collect(); + by_agent_vec.sort_by(|left, right| value_f64_for_key(right, "reliability").partial_cmp(&value_f64_for_key(left, "reliability")).unwrap_or(std::cmp::Ordering::Equal)); + let mut by_task_vec: Vec<_> = by_task.iter().map(|(name, agg)| aggregate_json(name, agg)).collect(); + by_task_vec.sort_by(|left, right| value_i64_for_key(right, "count").cmp(&value_i64_for_key(left, "count"))); + let mut top_sources: Vec<_> = source_counts.into_iter().collect(); + top_sources.sort_by(|left, right| right.1.cmp(&left.1)); + let top_sources: Vec<_> = top_sources.into_iter().take(10).map(|(source, hits)| json!({"source":source,"hits":hits})).collect(); + let reliability = overall.reliability(); + let recommendation = match (overall.count, reliability) { + (0, _) => "No agent feedback telemetry recorded yet.", + (_, r) if r < 0.65 => "Reliability is below target; tighten task decomposition and collect richer memory_sources.", + (_, r) if r < 0.8 => "Reliability is stable but improvable; prioritize retries and conflict resolution on partial outcomes.", + _ => "Reliability is strong; continue reinforcing high-quality runs and memory-source coverage.", + }; + Ok(json!({"ownerId":owner_id,"horizonDays":horizon_days,"limit":limit,"sampled":overall.count,"reliability":reliability, + "outcomes":{"success":overall.success,"partial":overall.partial,"failure":overall.failure}, + "averages":{"latencyMs":avg(overall.latency),"retries":avg(overall.retries),"tokensUsed":avg(overall.tokens)}, + "memorySourceCoverage":{"rowsWithSources":rows_with_sources,"ratio":if overall.count > 0 { rows_with_sources as f64 / overall.count as f64 } else { 0.0 }}, + "byAgent":by_agent_vec,"byTaskClass":by_task_vec,"topMemorySources":top_sources,"recommendation":recommendation})) +} + +fn value_f64_for_key(value: &Value, key: &str) -> f64 { + value.get(key).and_then(Value::as_f64).unwrap_or(0.0) +} + +fn value_i64_for_key(value: &Value, key: &str) -> i64 { + value.get(key).and_then(Value::as_i64).unwrap_or(0) +} + +pub fn recommend_recall_k(conn: &Connection, owner_id: i64, agent: &str, task_class: Option<&str>, base_k: usize) -> Result, String> { + let task_class = normalize_task_class(task_class); + let mut stmt = conn + .prepare( + "SELECT outcome, quality_score FROM agent_feedback + WHERE owner_id = ?1 AND agent = ?2 AND task_class = ?3 + AND julianday('now') - julianday(created_at) <= 30 + ORDER BY datetime(created_at) DESC, id DESC LIMIT 40", + ) + .map_err(|err| err.to_string())?; + let rows = stmt.query_map(params![owner_id, agent, task_class], |row| Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?))).map_err(|err| err.to_string())?; + let (mut success, mut partial, mut failure, mut quality_total, mut count) = (0usize, 0usize, 0usize, 0.0f64, 0usize); + for (outcome, quality) in rows.flatten() { + count += 1; + quality_total += quality.clamp(0.0, 1.0); + match outcome.as_str() { + "success" => success += 1, + "partial" => partial += 1, + _ => failure += 1, + } + } + if count < 8 { + return Ok(None); + } + let failure_rate = failure as f64 / count as f64; + let partial_rate = partial as f64 / count as f64; + let success_rate = success as f64 / count as f64; + let avg_quality = quality_total / count as f64; + let (recommended_k, reason) = if failure_rate >= 0.3 || partial_rate >= 0.45 { + ((base_k + 4).min(24), "raise_depth_for_recovery") + } else if success_rate >= 0.75 && avg_quality >= 0.82 { + (base_k.saturating_sub(2).max(6), "reduce_depth_for_efficiency") + } else { + (base_k, "keep_depth_stable") + }; + Ok(Some(json!({"agent":agent,"taskClass":task_class,"samples":count,"baseK":base_k,"recommendedK":recommended_k,"reason":reason, + "successRate":success_rate,"partialRate":partial_rate,"failureRate":failure_rate,"avgQuality":avg_quality}))) +} diff --git a/daemon-rs/src/handlers/feedback/recall.rs b/daemon-rs/src/handlers/feedback/recall.rs index 45768f26..ea2a2df6 100644 --- a/daemon-rs/src/handlers/feedback/recall.rs +++ b/daemon-rs/src/handlers/feedback/recall.rs @@ -1,50 +1,18 @@ -// SPDX-License-Identifier: MIT -//! Relevance Feedback Loop — learns which recalled results are actually useful. -//! -//! Signal sources: -//! - Explicit: cortex_unfold() calls → positive signal for unfolded sources -//! - Explicit: POST /feedback → caller reports useful/not-useful -//! -//! Feedback is stored in `recall_feedback` with the query embedding so future -//! recalls for similar queries can rerank based on what worked before. -//! -//! Reranking: boost = sum(signal * decay) for matching result_source, -//! where decay = exp(-age_days / 30). Capped at [-0.2, +0.3]. - -use axum::extract::{Query, State}; +use crate::embeddings; +use crate::handlers::{ensure_auth_rated, json_error, json_response}; +use crate::state::RuntimeState; +use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; use rusqlite::{params, Connection}; use serde::Deserialize; use serde_json::{json, Value}; - -use crate::handlers::{ensure_auth_rated, ensure_auth_with_caller_rated, json_error, json_response}; -use crate::embeddings; -use crate::state::RuntimeState; -use std::collections::HashMap; - -// ─── Constants ────────────────────────────────────────────────────────────── - -/// Max boost from feedback (prevents runaway amplification). const MAX_BOOST: f64 = 0.3; -/// Max penalty from negative feedback. const MIN_BOOST: f64 = -0.2; -/// Half-life for feedback signal decay (days). const DECAY_HALF_LIFE_DAYS: f64 = 30.0; -/// Minimum positive feedback signals in last 14 days to grant aging immunity. pub const IMMUNITY_THRESHOLD: i64 = 5; -/// Window for aging immunity check (days). pub const IMMUNITY_WINDOW_DAYS: i64 = 14; -/// Default lookback window for agent outcome telemetry stats. -const AGENT_FEEDBACK_DEFAULT_HORIZON_DAYS: i64 = 30; -/// Default row cap for agent outcome telemetry stats. -const AGENT_FEEDBACK_DEFAULT_LIMIT: usize = 400; -/// Recency half-life used for reliability weighting. -const AGENT_FEEDBACK_DECAY_HALF_LIFE_DAYS: f64 = 21.0; - -// ─── POST /feedback ───────────────────────────────────────────────────────── - #[derive(Deserialize)] pub struct FeedbackRequest { pub query: Option, @@ -52,31 +20,20 @@ pub struct FeedbackRequest { pub signal: Option, pub agent: Option, } - -pub async fn handle_feedback( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { +pub async fn handle_feedback(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } if body.sources.is_empty() { return json_error(StatusCode::BAD_REQUEST, "sources array is empty"); } - let signal = body.signal.unwrap_or(1.0).clamp(-1.0, 1.0); let agent = body.agent.as_deref().unwrap_or("http"); let query_text = body.query.as_deref().unwrap_or(""); - let query_embedding = match state.embedding_engine.clone() { - Some(engine) => engine - .embed_query_async(query_text.to_string()) - .await - .map(|v| embeddings::vector_to_blob(&v)), + Some(engine) => engine.embed_query_async(query_text.to_string()).await.map(|v| embeddings::vector_to_blob(&v)), None => None, }; - let conn = state.db.lock().await; let mut stored = 0usize; for source in &body.sources { @@ -84,149 +41,49 @@ pub async fn handle_feedback( match conn.execute( "INSERT INTO recall_feedback (query_text, query_embedding, result_source, result_type, result_id, signal, agent) \ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - query_text, - query_embedding, - source, - result_type, - result_id, - signal, - agent, - ], + params![query_text, query_embedding, source, result_type, result_id, signal, agent,], ) { Ok(_) => stored += 1, Err(e) => eprintln!("[feedback] Failed to store for {source}: {e}"), } } - json_response( StatusCode::OK, - json!({ - "stored": stored, - "signal": signal, - "sources": body.sources, - }), + json!({"stored":stored,"signal":signal,"sources": +body.sources,}), ) } - -// ─── Feedback recording (called internally by unfold) ─────────────────────── - -/// Record positive feedback for sources that were unfolded after a recall. -/// Called from the MCP unfold handler — no HTTP round-trip needed. -pub fn record_unfold_feedback( - conn: &Connection, - sources: &[String], - agent: &str, - query_text: &str, - query_blob: Option<&[u8]>, -) { - for source in sources { - let (result_type, result_id) = parse_source(source); - let _ = conn.execute( - "INSERT INTO recall_feedback (query_text, query_embedding, result_source, result_type, result_id, signal, agent) \ - VALUES (?1, ?2, ?3, ?4, ?5, 1.0, ?6)", - params![query_text, query_blob, source, result_type, result_id, agent], - ); - } -} - -// ─── Reranking: compute boost for a result source ─────────────────────────── - -/// Compute a relevance boost for a given result source based on historical -/// feedback. Returns a value in [MIN_BOOST, MAX_BOOST]. -/// -/// Algorithm: -/// boost = sum(signal_i * exp(-age_days_i / HALF_LIFE)) for all feedback rows -/// clamped to [MIN_BOOST, MAX_BOOST] -/// -/// This is O(feedback_rows_for_source) which stays small because: -/// - Each unfold generates ~2-3 rows -/// - Old feedback decays naturally -/// - We only scan rows for the specific source -#[allow(dead_code)] -pub fn compute_boost(conn: &Connection, result_source: &str) -> f64 { - let decay_lambda = (2.0f64).ln() / DECAY_HALF_LIFE_DAYS; - - let boost: f64 = conn - .prepare( - "SELECT signal, julianday('now') - julianday(created_at) AS age_days \ - FROM recall_feedback WHERE result_source = ?1", - ) - .and_then(|mut stmt| { - let rows = stmt.query_map(params![result_source], |row| { - let signal: f64 = row.get(0)?; - let age_days: f64 = row.get::<_, f64>(1)?.max(0.0); - Ok(signal * (-decay_lambda * age_days).exp()) - })?; - let mut total = 0.0f64; - for v in rows.flatten() { - total += v; - } - Ok(total) - }) - .unwrap_or(0.0); - - boost.clamp(MIN_BOOST, MAX_BOOST) -} - -/// Batch compute boosts for multiple sources at once (avoids N queries). -/// Returns a map of source → boost value. -pub fn compute_boosts( - conn: &Connection, - sources: &[String], - query_vector: Option<&[f32]>, -) -> std::collections::HashMap { +pub fn compute_boosts(conn: &Connection, sources: &[String], query_vector: Option<&[f32]>) -> std::collections::HashMap { let mut boosts = std::collections::HashMap::new(); if sources.is_empty() { return boosts; } - let decay_lambda = (2.0f64).ln() / DECAY_HALF_LIFE_DAYS; - - // Single query: fetch all feedback for any of the requested sources - let placeholders = sources - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 1)) - .collect::>() - .join(", "); - + let placeholders = sources.iter().enumerate().map(|(i, _)| format!("?{}", i + 1)).collect::>().join(", "); let sql = format!( "SELECT result_source, signal, query_embedding, julianday('now') - julianday(created_at) AS age_days \ FROM recall_feedback WHERE result_source IN ({placeholders})" ); - if let Ok(mut stmt) = conn.prepare(&sql) { - let params: Vec<&dyn rusqlite::types::ToSql> = sources - .iter() - .map(|s| s as &dyn rusqlite::types::ToSql) - .collect(); - + let params: Vec<&dyn rusqlite::types::ToSql> = sources.iter().map(|s| s as &dyn rusqlite::types::ToSql).collect(); if let Ok(rows) = stmt.query_map(params.as_slice(), |row| { let source: String = row.get(0)?; let signal: f64 = row.get(1)?; let query_blob: Option> = row.get(2)?; let age_days: f64 = row.get::<_, f64>(3)?.max(0.0); let query_weight = query_similarity_weight(query_vector, query_blob.as_deref()); - Ok(( - source, - signal * query_weight * (-decay_lambda * age_days).exp(), - )) + Ok((source, signal * query_weight * (-decay_lambda * age_days).exp())) }) { for row in rows.flatten() { *boosts.entry(row.0).or_insert(0.0) += row.1; } } } - - // Clamp all values for v in boosts.values_mut() { *v = v.clamp(MIN_BOOST, MAX_BOOST); } - boosts } - fn query_similarity_weight(current_query: Option<&[f32]>, stored_blob: Option<&[u8]>) -> f64 { let Some(current_query) = current_query else { return 1.0; @@ -239,11 +96,8 @@ fn query_similarity_weight(current_query: Option<&[f32]>, stored_blob: Option<&[ return 0.6; } let sim = embeddings::cosine_similarity(current_query, &stored_vec).clamp(0.0, 1.0); - // Keep a non-zero floor so sparse historic signal still contributes lightly. 0.2 + (sim as f64 * 0.8) } - -/// Check if a source has enough recent positive feedback to be immune from aging. pub fn has_retrieval_immunity(conn: &Connection, source: &str) -> bool { conn.query_row( "SELECT COUNT(*) FROM recall_feedback \ @@ -255,12 +109,6 @@ pub fn has_retrieval_immunity(conn: &Connection, source: &str) -> bool { .unwrap_or(0) >= IMMUNITY_THRESHOLD } - -// ─── Helpers ──────────────────────────────────────────────────────────────── - -/// Parse a source string into (type, optional ID). -/// "decision::42" → ("decision", Some(42)) -/// "memory::my_project.md" → ("memory", None) fn parse_source(source: &str) -> (String, Option) { if let Some(rest) = source.strip_prefix("decision::") { let id = rest.parse::().ok(); @@ -271,44 +119,15 @@ fn parse_source(source: &str) -> (String, Option) { ("unknown".to_string(), None) } } - -// ─── GET /feedback/stats ──────────────────────────────────────────────────── - -pub async fn handle_feedback_stats( - State(state): State, - headers: HeaderMap, -) -> Response { +pub async fn handle_feedback_stats(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db.lock().await; - - let total: i64 = conn - .query_row("SELECT COUNT(*) FROM recall_feedback", [], |row| row.get(0)) - .unwrap_or(0); - let positive: i64 = conn - .query_row( - "SELECT COUNT(*) FROM recall_feedback WHERE signal > 0", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let negative: i64 = conn - .query_row( - "SELECT COUNT(*) FROM recall_feedback WHERE signal < 0", - [], - |row| row.get(0), - ) - .unwrap_or(0); - let unique_sources: i64 = conn - .query_row( - "SELECT COUNT(DISTINCT result_source) FROM recall_feedback", - [], - |row| row.get(0), - ) - .unwrap_or(0); - - // Top boosted sources + let conn = state.db_read.lock().await; + let total: i64 = conn.query_row("SELECT COUNT(*) FROM recall_feedback", [], |row| row.get(0)).unwrap_or(0); + let positive: i64 = conn.query_row("SELECT COUNT(*) FROM recall_feedback WHERE signal > 0", [], |row| row.get(0)).unwrap_or(0); + let negative: i64 = conn.query_row("SELECT COUNT(*) FROM recall_feedback WHERE signal < 0", [], |row| row.get(0)).unwrap_or(0); + let unique_sources: i64 = conn.query_row("SELECT COUNT(DISTINCT result_source) FROM recall_feedback", [], |row| row.get(0)).unwrap_or(0); let top: Vec = conn .prepare( "SELECT result_source, SUM(signal) as total_signal, COUNT(*) as hits \ @@ -318,24 +137,15 @@ pub async fn handle_feedback_stats( ) .and_then(|mut stmt| { let rows = stmt.query_map([], |row| { - Ok(json!({ - "source": row.get::<_, String>(0)?, - "totalSignal": row.get::<_, f64>(1)?, - "hits": row.get::<_, i64>(2)?, - })) + Ok(json!({"source":row.get::<_,String>(0)?,"totalSignal":row.get::<_,f64>( +1)?,"hits":row.get::<_,i64>(2)?,})) })?; Ok(rows.flatten().collect()) }) .unwrap_or_default(); - json_response( StatusCode::OK, json!({ - "total": total, - "positive": positive, - "negative": negative, - "uniqueSources": unique_sources, - "topBoosted": top, - }), +"total":total,"positive":positive,"negative":negative,"uniqueSources":unique_sources,"topBoosted":top,}), ) } diff --git a/daemon-rs/src/handlers/feedback/tests/mod.rs b/daemon-rs/src/handlers/feedback/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/handlers/feedback/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/handlers/health/digest.rs b/daemon-rs/src/handlers/health/digest.rs index acfb48f8..f549c709 100644 --- a/daemon-rs/src/handlers/health/digest.rs +++ b/daemon-rs/src/handlers/health/digest.rs @@ -1,20 +1,10 @@ -// SPDX-License-Identifier: MIT +use crate::handlers::{ensure_auth_rated, json_response, truncate_chars}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; +use rusqlite::params; use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; -// ─── GET /digest ───────────────────────────────────────────────────────────── - pub async fn handle_digest(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; @@ -22,85 +12,34 @@ pub async fn handle_digest(State(state): State, headers: HeaderMap let conn = state.db_read.lock().await; match build_digest(&conn) { Ok(payload) => json_response(StatusCode::OK, payload), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Digest failed: {err}") }), - ), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Digest failed: {err}")})), } } - pub fn build_digest(conn: &rusqlite::Connection) -> Result { let today = chrono::Utc::now().format("%Y-%m-%d").to_string(); let today_like = format!("{today}%"); let benchmark_source_pattern = format!("{}%", crate::compaction::BENCHMARK_SOURCE_AGENT_PREFIX); - - let total_memories: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active'", - [], - |r| r.get(0), - ) - .unwrap_or(0); - let total_decisions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'active'", - [], - |r| r.get(0), - ) - .unwrap_or(0); - let total_conflicts: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'disputed'", - [], - |r| r.get(0), - ) - .unwrap_or(0); - + let total_memories: i64 = conn.query_row("SELECT COUNT(*) FROM memories WHERE status = 'active'", [], |r| r.get(0)).unwrap_or(0); + let total_decisions: i64 = conn.query_row("SELECT COUNT(*) FROM decisions WHERE status = 'active'", [], |r| r.get(0)).unwrap_or(0); + let total_conflicts: i64 = conn.query_row("SELECT COUNT(*) FROM decisions WHERE status = 'disputed'", [], |r| r.get(0)).unwrap_or(0); let new_memories: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE created_at LIKE ?1", - params![today_like.clone()], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM memories WHERE created_at LIKE ?1", params![today_like.clone()], |r| r.get(0)) .unwrap_or(0); let new_decisions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE created_at LIKE ?1", - params![today_like.clone()], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM decisions WHERE created_at LIKE ?1", params![today_like.clone()], |r| r.get(0)) .unwrap_or(0); let stores_today: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'decision_stored' AND created_at LIKE ?1", - params![today_like.clone()], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM events WHERE type = 'decision_stored' AND created_at LIKE ?1", params![today_like.clone()], |r| r.get(0)) .unwrap_or(0); let conflicts_today: i64 = conn - .query_row( - "SELECT COUNT(*) FROM events WHERE type = 'decision_conflict' AND created_at LIKE ?1", - params![today_like.clone()], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM events WHERE type = 'decision_conflict' AND created_at LIKE ?1", params![today_like.clone()], |r| r.get(0)) .unwrap_or(0); - let decayed_memories: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE status = 'active' AND score < 0.5 AND pinned = 0", - [], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM memories WHERE status = 'active' AND score < 0.5 AND pinned = 0", [], |r| r.get(0)) .unwrap_or(0); let decayed_decisions: i64 = conn - .query_row( - "SELECT COUNT(*) FROM decisions WHERE status = 'active' AND score < 0.5 AND pinned = 0", - [], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM decisions WHERE status = 'active' AND score < 0.5 AND pinned = 0", [], |r| r.get(0)) .unwrap_or(0); - - // Top recalled memories let mut top_stmt = conn .prepare( "SELECT source, text, retrievals FROM memories \ @@ -110,16 +49,11 @@ pub fn build_digest(conn: &rusqlite::Connection) -> Result { .map_err(|e| e.to_string())?; let top_rows = top_stmt .query_map([], |row| { - Ok(json!({ - "source": row.get::<_, Option>(0)?.unwrap_or_else(|| "unknown".to_string()), - "text": truncate_chars(&row.get::<_, String>(1)?, 80), - "retrievals": row.get::<_, i64>(2)? - })) + Ok(json!({"source":row.get::<_,Option>(0)?. +unwrap_or_else(||"unknown".to_string()),"text":truncate_chars(&row.get::<_,String>(1)?,80),"retrievals":row.get::<_,i64>(2)?})) }) .map_err(|e| e.to_string())?; let top_recalled: Vec = top_rows.filter_map(|r| r.ok()).collect(); - - // Agent boots today let mut boots_stmt = conn .prepare( "SELECT source_agent, COUNT(*) as cnt FROM events \ @@ -129,23 +63,12 @@ pub fn build_digest(conn: &rusqlite::Connection) -> Result { .map_err(|e| e.to_string())?; let boots_rows = boots_stmt .query_map(params![today_like.clone()], |row| { - Ok(json!({ - "source_agent": row.get::<_, Option>(0)?.unwrap_or_else(|| "unknown".to_string()), - "cnt": row.get::<_, i64>(1)? - })) + Ok(json!({"source_agent":row. +get::<_,Option>(0)?.unwrap_or_else(||"unknown".to_string()),"cnt":row.get::<_,i64>(1)?})) }) .map_err(|e| e.to_string())?; let agent_boots: Vec = boots_rows.filter_map(|r| r.ok()).collect(); - - // Token savings - let ( - raw_total_saved, - raw_total_served, - raw_boot_count, - today_saved, - today_served, - today_boots, - ): (i64, i64, i64, i64, i64, i64) = conn + let (raw_total_saved, raw_total_served, raw_boot_count, today_saved, today_served, today_boots): (i64, i64, i64, i64, i64, i64) = conn .query_row( "SELECT \ COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0)), 0), \ @@ -160,19 +83,9 @@ pub fn build_digest(conn: &rusqlite::Connection) -> Result { AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2)", params![today_like.clone(), benchmark_source_pattern.clone()], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - )) - }, + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?, row.get(5)?)), ) .map_err(|e| e.to_string())?; - let (rollup_saved, rollup_served, rollup_boots): (i64, i64, i64) = conn .query_row( "SELECT \ @@ -185,53 +98,27 @@ pub fn build_digest(conn: &rusqlite::Connection) -> Result { |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .map_err(|e| e.to_string())?; - let total_saved = raw_total_saved + rollup_saved; let total_served = raw_total_served + rollup_served; let boot_count = raw_boot_count + rollup_boots; - - // Build oneliner let agent_str = if agent_boots.is_empty() { "none".to_string() } else { agent_boots .iter() .map(|row| { - format!( - "{} ({})", - row.get("source_agent") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"), - row.get("cnt").and_then(|v| v.as_i64()).unwrap_or(0) - ) + format!("{} ({})", row.get("source_agent").and_then(|v| v.as_str()).unwrap_or("unknown"), row.get("cnt").and_then(|v| v.as_i64()).unwrap_or(0)) }) .collect::>() .join(", ") }; - - let savings_str = if total_saved > 0 { - format!(" | Saved: {} tokens ({} boots)", total_saved, boot_count) - } else { - String::new() - }; - let oneliner = format!( - "Cortex Daily — {today} | Mem: {total_memories} (+{new_memories}) | Dec: {total_decisions} (+{new_decisions}) | Conflicts: {total_conflicts} | Decaying: {} | Agents: {}{savings_str}", - decayed_memories + decayed_decisions, - agent_str, - ); - - Ok(json!({ - "date": today, - "totals": { "memories": total_memories, "decisions": total_decisions, "conflicts": total_conflicts }, - "today": { "newMemories": new_memories, "newDecisions": new_decisions, "stores": stores_today, "conflictsDetected": conflicts_today }, - "tokenSavings": { - "allTime": { "saved": total_saved, "served": total_served, "boots": boot_count }, - "today": { "saved": today_saved, "served": today_served, "boots": today_boots } - }, - "topRecalled": top_recalled, - "decay": { "memories": decayed_memories, "decisions": decayed_decisions }, - "agentBoots": agent_boots, - "oneliner": oneliner - })) + let savings_str = if total_saved > 0 { format!(" | Saved: {} tokens ({} boots)", total_saved, boot_count) } else { String::new() }; + let oneliner=format!( +"Cortex Daily — {today} | Mem: {total_memories} (+{new_memories}) | Dec: {total_decisions} (+{new_decisions}) | Conflicts: {total_conflicts} | Decaying: {} | Agents: {}{savings_str}" +,decayed_memories+decayed_decisions,agent_str,); + Ok(json!({"date":today,"totals":{"memories":total_memories,"decisions": +total_decisions,"conflicts":total_conflicts},"today":{"newMemories":new_memories,"newDecisions":new_decisions,"stores": +stores_today,"conflictsDetected":conflicts_today},"tokenSavings":{"allTime":{"saved":total_saved,"served":total_served,"boots": +boot_count},"today":{"saved":today_saved,"served":today_served,"boots":today_boots}},"topRecalled":top_recalled,"decay":{ +"memories":decayed_memories,"decisions":decayed_decisions},"agentBoots":agent_boots,"oneliner":oneliner})) } - diff --git a/daemon-rs/src/handlers/health/dump.rs b/daemon-rs/src/handlers/health/dump.rs index 84396f0c..b545c476 100644 --- a/daemon-rs/src/handlers/health/dump.rs +++ b/daemon-rs/src/handlers/health/dump.rs @@ -1,27 +1,15 @@ -// SPDX-License-Identifier: MIT +use crate::handlers::{ensure_auth_rated, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; use serde_json::{json, Value}; use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; -// ─── GET /dump ─────────────────────────────────────────────────────────────── - pub async fn handle_dump(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db_read.lock().await; - let memories: Vec = conn .prepare( "SELECT id, text, source, type, tags, source_agent, confidence, status, score, \ @@ -31,30 +19,19 @@ pub async fn handle_dump(State(state): State, headers: HeaderMap) ) .and_then(|mut stmt| { stmt.query_map([], |row| { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "text": row.get::<_, String>(1).unwrap_or_default(), - "source": row.get::<_, Option>(2).unwrap_or(None), - "type": row.get::<_, String>(3).unwrap_or_default(), - "tags": row.get::<_, Option>(4).unwrap_or(None), - "source_agent": row.get::<_, Option>(5).unwrap_or(None), - "confidence": row.get::<_, Option>(6).unwrap_or(Some(0.8)), - "status": row.get::<_, Option>(7).unwrap_or(Some("active".to_string())), - "score": row.get::<_, Option>(8).unwrap_or(Some(1.0)), - "retrievals": row.get::<_, Option>(9).unwrap_or(Some(0)), - "last_accessed": row.get::<_, Option>(10).unwrap_or(None), - "pinned": row.get::<_, Option>(11).unwrap_or(Some(0)), - "disputes_id": row.get::<_, Option>(12).unwrap_or(None), - "supersedes_id": row.get::<_, Option>(13).unwrap_or(None), - "confirmed_by": row.get::<_, Option>(14).unwrap_or(None), - "created_at": row.get::<_, Option>(15).unwrap_or(None), - "updated_at": row.get::<_, Option>(16).unwrap_or(None), - })) + Ok(json!({"id":row.get::<_,i64>(0)?,"text":row.get::<_,String>(1).unwrap_or_default +(),"source":row.get::<_,Option>(2).unwrap_or(None),"type":row.get::<_,String>(3).unwrap_or_default(),"tags":row.get::<_, +Option>(4).unwrap_or(None),"source_agent":row.get::<_,Option>(5).unwrap_or(None),"confidence":row.get::<_,Option< +f64>>(6).unwrap_or(Some(0.8)),"status":row.get::<_,Option>(7).unwrap_or(Some("active".to_string())),"score":row.get::<_, +Option>(8).unwrap_or(Some(1.0)),"retrievals":row.get::<_,Option>(9).unwrap_or(Some(0)),"last_accessed":row.get::<_, +Option>(10).unwrap_or(None),"pinned":row.get::<_,Option>(11).unwrap_or(Some(0)),"disputes_id":row.get::<_,Option>(12).unwrap_or(None),"supersedes_id":row.get::<_,Option>(13).unwrap_or(None),"confirmed_by":row.get::<_,Option>(14) +.unwrap_or(None),"created_at":row.get::<_,Option>(15).unwrap_or(None),"updated_at":row.get::<_,Option>(16). +unwrap_or(None),})) }) .map(|rows| rows.filter_map(|r| r.ok()).collect()) }) .unwrap_or_default(); - let decisions: Vec = conn .prepare( "SELECT id, decision, context, type, source_agent, confidence, surprise, status, \ @@ -64,82 +41,49 @@ pub async fn handle_dump(State(state): State, headers: HeaderMap) ) .and_then(|mut stmt| { stmt.query_map([], |row| { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "decision": row.get::<_, String>(1).unwrap_or_default(), - "context": row.get::<_, Option>(2).unwrap_or(None), - "type": row.get::<_, Option>(3).unwrap_or(Some("decision".to_string())), - "source_agent": row.get::<_, Option>(4).unwrap_or(None), - "confidence": row.get::<_, Option>(5).unwrap_or(Some(0.8)), - "surprise": row.get::<_, Option>(6).unwrap_or(Some(1.0)), - "status": row.get::<_, Option>(7).unwrap_or(Some("active".to_string())), - "score": row.get::<_, Option>(8).unwrap_or(Some(1.0)), - "retrievals": row.get::<_, Option>(9).unwrap_or(Some(0)), - "last_accessed": row.get::<_, Option>(10).unwrap_or(None), - "pinned": row.get::<_, Option>(11).unwrap_or(Some(0)), - "parent_id": row.get::<_, Option>(12).unwrap_or(None), - "disputes_id": row.get::<_, Option>(13).unwrap_or(None), - "supersedes_id": row.get::<_, Option>(14).unwrap_or(None), - "confirmed_by": row.get::<_, Option>(15).unwrap_or(None), - "created_at": row.get::<_, Option>(16).unwrap_or(None), - "updated_at": row.get::<_, Option>(17).unwrap_or(None), - })) + Ok(json!({"id":row.get::<_,i64>(0)?,"decision":row.get::<_,String>(1). +unwrap_or_default(),"context":row.get::<_,Option>(2).unwrap_or(None),"type":row.get::<_,Option>(3).unwrap_or(Some( +"decision".to_string())),"source_agent":row.get::<_,Option>(4).unwrap_or(None),"confidence":row.get::<_,Option>(5). +unwrap_or(Some(0.8)),"surprise":row.get::<_,Option>(6).unwrap_or(Some(1.0)),"status":row.get::<_,Option>(7).unwrap_or +(Some("active".to_string())),"score":row.get::<_,Option>(8).unwrap_or(Some(1.0)),"retrievals":row.get::<_,Option>(9). +unwrap_or(Some(0)),"last_accessed":row.get::<_,Option>(10).unwrap_or(None),"pinned":row.get::<_,Option>(11).unwrap_or +(Some(0)),"parent_id":row.get::<_,Option>(12).unwrap_or(None),"disputes_id":row.get::<_,Option>(13).unwrap_or(None), +"supersedes_id":row.get::<_,Option>(14).unwrap_or(None),"confirmed_by":row.get::<_,Option>(15).unwrap_or(None), +"created_at":row.get::<_,Option>(16).unwrap_or(None),"updated_at":row.get::<_,Option>(17).unwrap_or(None),})) }) .map(|rows| rows.filter_map(|r| r.ok()).collect()) }) .unwrap_or_default(); - let mut source_nodes: BTreeMap = BTreeMap::new(); for memory in &memories { let Some(id) = memory.get("id").and_then(|value| value.as_i64()) else { continue; }; - let Some(source) = memory - .get("source") - .and_then(|value| value.as_str()) - .map(str::trim) - .filter(|value| !value.is_empty()) - else { + let Some(source) = memory.get("source").and_then(|value| value.as_str()).map(str::trim).filter(|value| !value.is_empty()) else { continue; }; - source_nodes - .entry(source.to_string()) - .or_insert_with(|| format!("mem-{id}")); + source_nodes.entry(source.to_string()).or_insert_with(|| format!("mem-{id}")); } for decision in &decisions { let Some(id) = decision.get("id").and_then(|value| value.as_i64()) else { continue; }; - let Some(source) = decision - .get("context") - .and_then(|value| value.as_str()) - .map(str::trim) - .filter(|value| !value.is_empty()) - else { + let Some(source) = decision.get("context").and_then(|value| value.as_str()).map(str::trim).filter(|value| !value.is_empty()) else { continue; }; - source_nodes - .entry(source.to_string()) - .or_insert_with(|| format!("dec-{id}")); + source_nodes.entry(source.to_string()).or_insert_with(|| format!("dec-{id}")); } - let mut seen_links: HashSet = HashSet::new(); let mut graph_links: Vec = Vec::new(); - if let Ok(mut stmt) = conn.prepare( "SELECT source_a, source_b, count, last_seen FROM co_occurrence ORDER BY count DESC, last_seen DESC LIMIT 240", ) { - if let Ok(rows) = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, Option>(3)?, - )) - }) { + if let Ok(rows) = + stmt.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, row.get::<_, Option>(3)?))) + { for row in rows.flatten() { let (source_a, source_b, count, last_seen) = row; let Some(node_a) = source_nodes.get(&source_a) else { @@ -151,26 +95,16 @@ pub async fn handle_dump(State(state): State, headers: HeaderMap) if node_a == node_b { continue; } - let (left, right) = if node_a <= node_b { - (node_a.clone(), node_b.clone()) - } else { - (node_b.clone(), node_a.clone()) - }; + let (left, right) = if node_a <= node_b { (node_a.clone(), node_b.clone()) } else { (node_b.clone(), node_a.clone()) }; let key = format!("{left}|{right}|co_occurrence"); if !seen_links.insert(key) { continue; } - graph_links.push(json!({ - "source": left, - "target": right, - "type": "co_occurrence", - "weight": count, - "lastSeen": last_seen, - })); + graph_links.push(json!({"source":left,"target":right,"type": +"co_occurrence","weight":count,"lastSeen":last_seen,})); } } } - for decision in &decisions { let Some(id) = decision.get("id").and_then(|value| value.as_i64()) else { continue; @@ -180,33 +114,17 @@ pub async fn handle_dump(State(state): State, headers: HeaderMap) }; let left = format!("dec-{id}"); let right = format!("dec-{disputes_id}"); - let (source, target) = if left <= right { - (left, right) - } else { - (right, left) - }; + let (source, target) = if left <= right { (left, right) } else { (right, left) }; let key = format!("{source}|{target}|conflict"); if !seen_links.insert(key) { continue; } - graph_links.push(json!({ - "source": source, - "target": target, - "type": "conflict", - "weight": 1, - })); + graph_links.push(json!({"source": +source,"target":target,"type":"conflict","weight":1,})); } - json_response( StatusCode::OK, - json!({ - "memories": memories, - "decisions": decisions, - "graph": { - "links": graph_links, - "nodeCount": memories.len() + decisions.len(), - } - }), + json!({"memories":memories,"decisions": +decisions,"graph":{"links":graph_links,"nodeCount":memories.len()+decisions.len(),}}), ) } - diff --git a/daemon-rs/src/handlers/health/health.rs b/daemon-rs/src/handlers/health/health.rs index 6489a4a5..6d9b81f9 100644 --- a/daemon-rs/src/handlers/health/health.rs +++ b/daemon-rs/src/handlers/health/health.rs @@ -1,24 +1,14 @@ -// SPDX-License-Identifier: MIT +use super::*; +use crate::handlers::{client_ip, ensure_ssrf_protection, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use chrono::Utc; -use rusqlite::{params, OpenFlags}; use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; -// ─── GET /health ───────────────────────────────────────────────────────────── - pub(crate) fn include_private_runtime_details(headers: &HeaderMap) -> bool { ensure_ssrf_protection(headers).is_ok() && client_ip(headers).is_loopback() } - pub(crate) fn redact_private_runtime_details(payload: &mut Value) { if let Some(runtime) = payload.get_mut("runtime").and_then(Value::as_object_mut) { runtime.remove("db_path"); @@ -29,62 +19,31 @@ pub(crate) fn redact_private_runtime_details(payload: &mut Value) { runtime.remove("executable"); runtime.remove("owner"); } - if let Some(stats) = payload.get_mut("stats").and_then(Value::as_object_mut) { stats.remove("home"); } } - pub async fn build_health_payload(state: &RuntimeState, include_private_runtime: bool) -> Value { let embedding_model = crate::embeddings::selected_model_selection(); let now_unix_secs = Utc::now().timestamp(); - let daemon_owner = std::env::var("CORTEX_DAEMON_OWNER") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()); - // Read DB stats in a short lock, then drop it before the network call. + let daemon_owner = std::env::var("CORTEX_DAEMON_OWNER").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); let (memories, decisions, embeddings_count, events, db_freelist_pages, sqlite_vec_status) = { let conn = state.db_read.lock().await; - let m: i64 = conn - .query_row("SELECT COUNT(*) FROM memories", [], |r| r.get(0)) - .unwrap_or(0); - let d: i64 = conn - .query_row("SELECT COUNT(*) FROM decisions", [], |r| r.get(0)) - .unwrap_or(0); - let e: i64 = conn - .query_row("SELECT COUNT(*) FROM embeddings", [], |r| r.get(0)) - .unwrap_or(0); - let ev: i64 = conn - .query_row("SELECT COUNT(*) FROM events", [], |r| r.get(0)) - .unwrap_or(0); - let freelist: i64 = conn - .query_row("PRAGMA freelist_count", [], |r| r.get(0)) - .unwrap_or(0); + let m: i64 = conn.query_row("SELECT COUNT(*) FROM memories", [], |r| r.get(0)).unwrap_or(0); + let d: i64 = conn.query_row("SELECT COUNT(*) FROM decisions", [], |r| r.get(0)).unwrap_or(0); + let e: i64 = conn.query_row("SELECT COUNT(*) FROM embeddings", [], |r| r.get(0)).unwrap_or(0); + let ev: i64 = conn.query_row("SELECT COUNT(*) FROM events", [], |r| r.get(0)).unwrap_or(0); + let freelist: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0)).unwrap_or(0); let sqlite_vec_status = crate::db::sqlite_vec_status(&conn); (m, d, e, ev, freelist, sqlite_vec_status) - }; // DB lock released here. - - let ( - embedding_inventory, - storage_bytes, - backup_count, - log_bytes, - heavy_metrics_source, - cache_age_secs, - ) = { + }; + let (embedding_inventory, storage_bytes, backup_count, log_bytes, heavy_metrics_source, cache_age_secs) = { let cached = match health_heavy_metrics_cache().lock() { Ok(guard) => *guard, Err(poisoned) => *poisoned.into_inner(), }; if let Some(snapshot) = cache_snapshot_if_fresh(cached, now_unix_secs) { - ( - snapshot.embedding_inventory, - snapshot.storage_bytes, - snapshot.backup_count, - snapshot.log_bytes, - "cache", - snapshot.cache_age_secs(now_unix_secs), - ) + (snapshot.embedding_inventory, snapshot.storage_bytes, snapshot.backup_count, snapshot.log_bytes, "cache", snapshot.cache_age_secs(now_unix_secs)) } else if app_managed_warmup_active(daemon_owner.as_deref()) { let fallback = cached.unwrap_or(HealthHeavyMetricsSnapshot { computed_at_unix_secs: now_unix_secs, @@ -107,52 +66,24 @@ pub async fn build_health_payload(state: &RuntimeState, include_private_runtime: collect_embedding_inventory(&conn, embedding_model.key) }; let (storage_bytes, backup_count, log_bytes) = collect_storage_metrics(&state.home); - let snapshot = HealthHeavyMetricsSnapshot { - computed_at_unix_secs: now_unix_secs, - embedding_inventory, - storage_bytes, - backup_count, - log_bytes, - }; + let snapshot = HealthHeavyMetricsSnapshot { computed_at_unix_secs: now_unix_secs, embedding_inventory, storage_bytes, backup_count, log_bytes }; match health_heavy_metrics_cache().lock() { Ok(mut guard) => *guard = Some(snapshot), Err(poisoned) => *poisoned.into_inner() = Some(snapshot), } - ( - embedding_inventory, - storage_bytes, - backup_count, - log_bytes, - "live", - 0, - ) + (embedding_inventory, storage_bytes, backup_count, log_bytes, "live", 0) } }; - - let db_size_bytes = std::fs::metadata(&state.db_path) - .map(|meta| meta.len()) - .unwrap_or(0); + let db_size_bytes = std::fs::metadata(&state.db_path).map(|meta| meta.len()).unwrap_or(0); let db_soft_limit_bytes = crate::compaction::STORAGE_SOFT_LIMIT_BYTES.max(1) as u64; let db_hard_limit_bytes = crate::compaction::STORAGE_HARD_LIMIT_BYTES.max(1) as u64; let db_pressure = crate::compaction::classify_storage_pressure(db_size_bytes as i64); let db_soft_utilization = ((db_size_bytes as f64) / (db_soft_limit_bytes as f64)).min(10.0); - let active_model_ratio = if embeddings_count > 0 { - (embedding_inventory.active_model_embeddings as f64) / (embeddings_count as f64) - } else { - 0.0 - }; - let reembed_backlog_total = - embedding_inventory.backlog_memories + embedding_inventory.backlog_decisions; - - let degraded = state - .degraded_mode - .load(std::sync::atomic::Ordering::Relaxed); + let active_model_ratio = if embeddings_count > 0 { (embedding_inventory.active_model_embeddings as f64) / (embeddings_count as f64) } else { 0.0 }; + let reembed_backlog_total = embedding_inventory.backlog_memories + embedding_inventory.backlog_decisions; + let degraded = state.degraded_mode.load(std::sync::atomic::Ordering::Relaxed); let reranker_model = crate::rerank::selected_reranker_selection(); - - let db_corrupted = state - .db_corrupted - .load(std::sync::atomic::Ordering::Relaxed); - + let db_corrupted = state.db_corrupted.load(std::sync::atomic::Ordering::Relaxed); let embedding_status = if degraded { "degraded" } else if state.embedding_engine.is_some() { @@ -160,207 +91,63 @@ pub async fn build_health_payload(state: &RuntimeState, include_private_runtime: } else { "unavailable" }; - - let executable = std::env::current_exe() - .ok() - .map(|path| path.display().to_string()) - .unwrap_or_default(); - let ipc_endpoint = std::env::var("CORTEX_IPC_ENDPOINT") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()); - let ipc_kind = if ipc_endpoint.is_some() { - Some(if cfg!(windows) { - "named-pipe" - } else { - "unix-socket" - }) - } else { - None - }; + let executable = std::env::current_exe().ok().map(|path| path.display().to_string()).unwrap_or_default(); + let ipc_endpoint = std::env::var("CORTEX_IPC_ENDPOINT").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + let ipc_kind = if ipc_endpoint.is_some() { Some(if cfg!(windows) { "named-pipe" } else { "unix-socket" }) } else { None }; let ready = state.readiness.load(std::sync::atomic::Ordering::Relaxed); - let budgets = state - .rate_limiter - .budget_status() - .to_health_json(state.rate_limiter.recent_budget_denials().await); - - let mut payload = json!({ - "status": if degraded || db_corrupted { "degraded" } else { "ok" }, - "ready": ready, - "degraded": degraded || db_corrupted, - "db_corrupted": db_corrupted, - "budgets": budgets, - "embedding_status": embedding_status, - "vector_search": { - "backend": if matches!( - state.sqlite_vec_canary.effective_route_mode(), - crate::state::SqliteVecRouteMode::Primary - ) { - "sqlite_vec_primary" - } else { - "blob_scan" - }, - "embedding_model": { - "key": embedding_model.key, - "display_name": embedding_model.display_name, - "dimension": embedding_model.dimension, - "max_input_tokens": embedding_model.max_input_tokens, - "pooling": embedding_model.pooling, - "model_file": embedding_model.model_file, - "tokenizer_file": embedding_model.tokenizer_file - }, - "routing": { - "mode": state.sqlite_vec_canary.route_mode.as_str(), - "effective_mode": state.sqlite_vec_canary.effective_route_mode().as_str(), - "trial_percent": state.sqlite_vec_canary.trial_percent, - "force_off": state.sqlite_vec_canary.force_off - }, - "reranker": { - "mode": state.rerank_config.mode.as_str(), - "available": state.reranker.is_some(), - "model": { - "key": reranker_model.key, - "display_name": reranker_model.display_name, - "model_size_mb": reranker_model.model_size_mb, - "max_input_tokens": reranker_model.max_input_tokens, - "model_file": reranker_model.model_file, - "tokenizer_file": reranker_model.tokenizer_file - }, - "top_n": state.rerank_config.top_n, - "fusion_alpha": state.rerank_config.fusion_alpha - }, - "embedding_inventory": { - "active_model_key": embedding_model.key, - "active_model_embeddings": embedding_inventory.active_model_embeddings, - "other_model_embeddings": embedding_inventory.other_model_embeddings, - "unknown_model_embeddings": embedding_inventory.unknown_model_embeddings, - "active_model_ratio": active_model_ratio, - "reembed_backlog": { - "memories": embedding_inventory.backlog_memories, - "decisions": embedding_inventory.backlog_decisions, - "total": reembed_backlog_total - } - }, - "sqlite_vec": { - "available": sqlite_vec_status.available, - "version": sqlite_vec_status.version, - "error": sqlite_vec_status.error - }, - "health_heavy_metrics": { - "source": heavy_metrics_source, - "cache_ttl_secs": HEALTH_HEAVY_CACHE_TTL_SECS, - "cache_age_secs": cache_age_secs - }, - }, - "team_mode": state.team_mode, - "db_freelist_pages": db_freelist_pages, - "db_size_bytes": db_size_bytes, - "db_soft_limit_bytes": db_soft_limit_bytes, - "db_hard_limit_bytes": db_hard_limit_bytes, - "db_pressure": db_pressure, - "db_soft_utilization": db_soft_utilization, - "storage_bytes": storage_bytes, - "backup_count": backup_count, - "log_bytes": log_bytes, - "stats": { - "memories": memories, - "decisions": decisions, - "embeddings": embeddings_count, - "events": events, - "home": state.home.display().to_string() - }, - "runtime": { - "version": env!("CARGO_PKG_VERSION"), - "mode": if state.team_mode { "team" } else { "solo" }, - "port": state.port, - "db_path": state.db_path.display().to_string(), - "token_path": state.token_path.display().to_string(), - "pid_path": state.pid_path.display().to_string(), - "ipc_endpoint": ipc_endpoint, - "ipc_kind": ipc_kind, - "executable": executable, - "owner": daemon_owner - } - }); - + let budgets = state.rate_limiter.budget_status().to_health_json(state.rate_limiter.recent_budget_denials().await); + let mut payload = json!({"status":if degraded||db_corrupted{"degraded"}else{"ok"}, +"ready":ready,"degraded":degraded||db_corrupted,"db_corrupted":db_corrupted,"budgets":budgets,"embedding_status":embedding_status, +"vector_search":{"backend":if matches!(state.sqlite_vec_canary.effective_route_mode(),crate::state::SqliteVecRouteMode::Primary){ +"sqlite_vec_primary"}else{"blob_scan"},"embedding_model":{"key":embedding_model.key,"display_name":embedding_model.display_name, +"dimension":embedding_model.dimension,"max_input_tokens":embedding_model.max_input_tokens,"pooling":embedding_model.pooling, +"model_file":embedding_model.model_file,"tokenizer_file":embedding_model.tokenizer_file},"routing":{"mode":state.sqlite_vec_canary +.route_mode.as_str(),"effective_mode":state.sqlite_vec_canary.effective_route_mode().as_str(),"trial_percent":state. +sqlite_vec_canary.trial_percent,"force_off":state.sqlite_vec_canary.force_off},"reranker":{"mode":state.rerank_config.mode.as_str( +),"available":state.reranker.is_some(),"model":{"key":reranker_model.key,"display_name":reranker_model.display_name, +"model_size_mb":reranker_model.model_size_mb,"max_input_tokens":reranker_model.max_input_tokens,"model_file":reranker_model. +model_file,"tokenizer_file":reranker_model.tokenizer_file},"top_n":state.rerank_config.top_n,"fusion_alpha":state.rerank_config. +fusion_alpha},"embedding_inventory":{"active_model_key":embedding_model.key,"active_model_embeddings":embedding_inventory. +active_model_embeddings,"other_model_embeddings":embedding_inventory.other_model_embeddings,"unknown_model_embeddings": +embedding_inventory.unknown_model_embeddings,"active_model_ratio":active_model_ratio,"reembed_backlog":{"memories": +embedding_inventory.backlog_memories,"decisions":embedding_inventory.backlog_decisions,"total":reembed_backlog_total}}, +"sqlite_vec":{"available":sqlite_vec_status.available,"version":sqlite_vec_status.version,"error":sqlite_vec_status.error}, +"health_heavy_metrics":{"source":heavy_metrics_source,"cache_ttl_secs":HEALTH_HEAVY_CACHE_TTL_SECS,"cache_age_secs":cache_age_secs +},},"team_mode":state.team_mode,"db_freelist_pages":db_freelist_pages,"db_size_bytes":db_size_bytes,"db_soft_limit_bytes": +db_soft_limit_bytes,"db_hard_limit_bytes":db_hard_limit_bytes,"db_pressure":db_pressure,"db_soft_utilization":db_soft_utilization, +"storage_bytes":storage_bytes,"backup_count":backup_count,"log_bytes":log_bytes,"stats":{"memories":memories,"decisions":decisions +,"embeddings":embeddings_count,"events":events,"home":state.home.display().to_string()},"runtime":{"version":env!( +"CARGO_PKG_VERSION"),"mode":if state.team_mode{"team"}else{"solo"},"port":state.port,"db_path":state.db_path.display().to_string() +,"token_path":state.token_path.display().to_string(),"pid_path":state.pid_path.display().to_string(),"ipc_endpoint":ipc_endpoint, +"ipc_kind":ipc_kind,"executable":executable,"owner":daemon_owner}}); if !include_private_runtime { redact_private_runtime_details(&mut payload); } - payload } - pub async fn handle_health(State(state): State, headers: HeaderMap) -> Response { let include_private_runtime = include_private_runtime_details(&headers); - json_response( - StatusCode::OK, - build_health_payload(&state, include_private_runtime).await, - ) + json_response(StatusCode::OK, build_health_payload(&state, include_private_runtime).await) } - pub async fn build_readiness_payload(state: &RuntimeState, include_private_runtime: bool) -> Value { - let executable = std::env::current_exe() - .ok() - .map(|path| path.display().to_string()) - .unwrap_or_default(); - let daemon_owner = std::env::var("CORTEX_DAEMON_OWNER") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()); - let ipc_endpoint = std::env::var("CORTEX_IPC_ENDPOINT") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()); - let ipc_kind = if ipc_endpoint.is_some() { - Some(if cfg!(windows) { - "named-pipe" - } else { - "unix-socket" - }) - } else { - None - }; + let executable = std::env::current_exe().ok().map(|path| path.display().to_string()).unwrap_or_default(); + let daemon_owner = std::env::var("CORTEX_DAEMON_OWNER").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + let ipc_endpoint = std::env::var("CORTEX_IPC_ENDPOINT").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + let ipc_kind = if ipc_endpoint.is_some() { Some(if cfg!(windows) { "named-pipe" } else { "unix-socket" }) } else { None }; let ready = state.readiness.load(std::sync::atomic::Ordering::Relaxed); - - let mut payload = json!({ - "status": if ready { "ready" } else { "starting" }, - "ready": ready, - "runtime": { - "version": env!("CARGO_PKG_VERSION"), - "mode": if state.team_mode { "team" } else { "solo" }, - "port": state.port, - "db_path": state.db_path.display().to_string(), - "token_path": state.token_path.display().to_string(), - "pid_path": state.pid_path.display().to_string(), - "ipc_endpoint": ipc_endpoint, - "ipc_kind": ipc_kind, - "executable": executable, - "owner": daemon_owner - }, - "stats": { - "home": state.home.display().to_string() - } - }); - + let mut payload = json!({"status":if ready{"ready"}else{"starting"},"ready":ready,"runtime":{"version":env!( +"CARGO_PKG_VERSION"),"mode":if state.team_mode{"team"}else{"solo"},"port":state.port,"db_path":state.db_path.display().to_string() +,"token_path":state.token_path.display().to_string(),"pid_path":state.pid_path.display().to_string(),"ipc_endpoint":ipc_endpoint, +"ipc_kind":ipc_kind,"executable":executable,"owner":daemon_owner},"stats":{"home":state.home.display().to_string()}}); if !include_private_runtime { redact_private_runtime_details(&mut payload); } - payload } - pub async fn handle_readiness(State(state): State, headers: HeaderMap) -> Response { let include_private_runtime = include_private_runtime_details(&headers); let payload = build_readiness_payload(&state, include_private_runtime).await; - let ready = payload - .get("ready") - .and_then(|value| value.as_bool()) - .unwrap_or(false); - let status = if ready { - StatusCode::OK - } else { - StatusCode::SERVICE_UNAVAILABLE - }; + let ready = payload.get("ready").and_then(|value| value.as_bool()).unwrap_or(false); + let status = if ready { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; json_response(status, payload) } - diff --git a/daemon-rs/src/handlers/health/metrics.rs b/daemon-rs/src/handlers/health/metrics.rs index d09f1b7b..6a9fe9ca 100644 --- a/daemon-rs/src/handlers/health/metrics.rs +++ b/daemon-rs/src/handlers/health/metrics.rs @@ -1,70 +1,37 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; +use rusqlite::params; +use serde_json::Value; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; -pub(crate) const STORAGE_LOG_FILES: &[&str] = &[ - "daemon.log", - "daemon.err.log", - "daemon.out.log", - "mcp-crash.log", - "rust-daemon.err.log", -]; +pub(crate) const STORAGE_LOG_FILES: &[&str] = &["daemon.log", "daemon.err.log", "daemon.out.log", "mcp-crash.log", "rust-daemon.err.log"]; pub(crate) const CONTROL_CENTER_OWNER_TAG: &str = "control-center"; pub(crate) const HEALTH_HEAVY_CACHE_TTL_SECS: i64 = 30; pub(crate) const HEALTH_HEAVY_WARMUP_DELAY_SECS: u64 = 90; pub(crate) const SAVINGS_CACHE_TTL_SECS: i64 = 20; pub(crate) const SAVINGS_HISTORY_DAYS: i64 = 30; static HEALTH_BOOT_INSTANT: OnceLock = OnceLock::new(); -static HEALTH_HEAVY_METRICS_CACHE: OnceLock>> = - OnceLock::new(); +static HEALTH_HEAVY_METRICS_CACHE: OnceLock>> = OnceLock::new(); static SAVINGS_PAYLOAD_CACHE: OnceLock>> = OnceLock::new(); - pub(crate) fn directory_size_bytes(path: &std::path::Path) -> u64 { match std::fs::metadata(path) { Ok(meta) if meta.is_file() => meta.len(), Ok(meta) if meta.is_dir() => std::fs::read_dir(path) - .map(|entries| { - entries - .filter_map(|entry| entry.ok()) - .map(|entry| directory_size_bytes(&entry.path())) - .sum() - }) + .map(|entries| entries.filter_map(|entry| entry.ok()).map(|entry| directory_size_bytes(&entry.path())).sum()) .unwrap_or(0), _ => 0, } } - pub(crate) fn collect_storage_metrics(home: &std::path::Path) -> (u64, usize, u64) { let storage_bytes = directory_size_bytes(home); let backup_count = std::fs::read_dir(home.join("backups")) - .map(|entries| { - entries - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.file_name().to_string_lossy().ends_with(".db")) - .count() - }) + .map(|entries| entries.filter_map(|entry| entry.ok()).filter(|entry| entry.file_name().to_string_lossy().ends_with(".db")).count()) .unwrap_or(0); - let log_bytes = STORAGE_LOG_FILES .iter() .flat_map(|name| [home.join(name), home.join(format!("{name}.1"))]) .map(|path| std::fs::metadata(path).map(|meta| meta.len()).unwrap_or(0)) .sum(); - (storage_bytes, backup_count, log_bytes) } - #[derive(Clone, Copy, Debug, Default)] pub(crate) struct EmbeddingInventoryMetrics { pub(crate) active_model_embeddings: i64, @@ -73,7 +40,6 @@ pub(crate) struct EmbeddingInventoryMetrics { pub(crate) backlog_memories: i64, pub(crate) backlog_decisions: i64, } - #[derive(Clone, Copy, Debug)] pub(crate) struct HealthHeavyMetricsSnapshot { pub(crate) computed_at_unix_secs: i64, @@ -82,39 +48,30 @@ pub(crate) struct HealthHeavyMetricsSnapshot { pub(crate) backup_count: usize, pub(crate) log_bytes: u64, } - impl HealthHeavyMetricsSnapshot { pub(crate) fn cache_age_secs(self, now_unix_secs: i64) -> i64 { (now_unix_secs - self.computed_at_unix_secs).max(0) } } - #[derive(Clone, Debug)] pub(crate) struct SavingsPayloadSnapshot { pub(crate) computed_at_unix_secs: i64, pub(crate) payload: Value, } - impl SavingsPayloadSnapshot { pub(crate) fn cache_age_secs(&self, now_unix_secs: i64) -> i64 { (now_unix_secs - self.computed_at_unix_secs).max(0) } } - pub(crate) fn is_control_center_owner(owner_tag: Option<&str>) -> bool { - owner_tag - .map(|owner| owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)) - .unwrap_or(false) + owner_tag.map(|owner| owner.eq_ignore_ascii_case(CONTROL_CENTER_OWNER_TAG)).unwrap_or(false) } - pub(crate) fn health_heavy_metrics_cache() -> &'static Mutex> { HEALTH_HEAVY_METRICS_CACHE.get_or_init(|| Mutex::new(None)) } - pub(crate) fn savings_payload_cache() -> &'static Mutex> { SAVINGS_PAYLOAD_CACHE.get_or_init(|| Mutex::new(None)) } - pub(crate) fn app_managed_warmup_active(daemon_owner: Option<&str>) -> bool { if !is_control_center_owner(daemon_owner) { return false; @@ -122,33 +79,12 @@ pub(crate) fn app_managed_warmup_active(daemon_owner: Option<&str>) -> bool { let started = HEALTH_BOOT_INSTANT.get_or_init(Instant::now); started.elapsed() < Duration::from_secs(HEALTH_HEAVY_WARMUP_DELAY_SECS) } - -pub(crate) fn cache_snapshot_if_fresh( - snapshot: Option, - now_unix_secs: i64, -) -> Option { - snapshot.and_then(|entry| { - if entry.cache_age_secs(now_unix_secs) <= HEALTH_HEAVY_CACHE_TTL_SECS { - Some(entry) - } else { - None - } - }) +pub(crate) fn cache_snapshot_if_fresh(snapshot: Option, now_unix_secs: i64) -> Option { + snapshot.and_then(|entry| if entry.cache_age_secs(now_unix_secs) <= HEALTH_HEAVY_CACHE_TTL_SECS { Some(entry) } else { None }) } - -pub(crate) fn savings_payload_cache_if_fresh( - snapshot: Option, - now_unix_secs: i64, -) -> Option { - snapshot.and_then(|entry| { - if entry.cache_age_secs(now_unix_secs) <= SAVINGS_CACHE_TTL_SECS { - Some(entry) - } else { - None - } - }) +pub(crate) fn savings_payload_cache_if_fresh(snapshot: Option, now_unix_secs: i64) -> Option { + snapshot.and_then(|entry| if entry.cache_age_secs(now_unix_secs) <= SAVINGS_CACHE_TTL_SECS { Some(entry) } else { None }) } - pub(crate) fn weekday_name_from_sqlite(weekday: i64) -> &'static str { match weekday { 0 => "Sun", @@ -161,30 +97,14 @@ pub(crate) fn weekday_name_from_sqlite(weekday: i64) -> &'static str { _ => "Unknown", } } - -pub(crate) fn collect_embedding_inventory( - conn: &rusqlite::Connection, - active_model_key: &str, -) -> EmbeddingInventoryMetrics { - let total_embeddings: i64 = conn - .query_row("SELECT COUNT(*) FROM embeddings", [], |r| r.get(0)) - .unwrap_or(0); +pub(crate) fn collect_embedding_inventory(conn: &rusqlite::Connection, active_model_key: &str) -> EmbeddingInventoryMetrics { + let total_embeddings: i64 = conn.query_row("SELECT COUNT(*) FROM embeddings", [], |r| r.get(0)).unwrap_or(0); let active_model_embeddings: i64 = conn - .query_row( - "SELECT COUNT(*) FROM embeddings WHERE LOWER(COALESCE(model, '')) = ?1", - params![active_model_key], - |r| r.get(0), - ) - .unwrap_or(0); - let unknown_model_embeddings: i64 = conn - .query_row( - "SELECT COUNT(*) FROM embeddings WHERE model IS NULL OR TRIM(model) = ''", - [], - |r| r.get(0), - ) + .query_row("SELECT COUNT(*) FROM embeddings WHERE LOWER(COALESCE(model, '')) = ?1", params![active_model_key], |r| r.get(0)) .unwrap_or(0); - let other_model_embeddings = - (total_embeddings - active_model_embeddings - unknown_model_embeddings).max(0); + let unknown_model_embeddings: i64 = + conn.query_row("SELECT COUNT(*) FROM embeddings WHERE model IS NULL OR TRIM(model) = ''", [], |r| r.get(0)).unwrap_or(0); + let other_model_embeddings = (total_embeddings - active_model_embeddings - unknown_model_embeddings).max(0); let backlog_memories: i64 = conn .query_row( "SELECT COUNT(*) FROM memories m \ @@ -213,7 +133,6 @@ pub(crate) fn collect_embedding_inventory( |r| r.get(0), ) .unwrap_or(0); - EmbeddingInventoryMetrics { active_model_embeddings, other_model_embeddings, @@ -222,4 +141,3 @@ pub(crate) fn collect_embedding_inventory( backlog_decisions, } } - diff --git a/daemon-rs/src/handlers/health/mod.rs b/daemon-rs/src/handlers/health/mod.rs index b6fb9161..26285b3a 100644 --- a/daemon-rs/src/handlers/health/mod.rs +++ b/daemon-rs/src/handlers/health/mod.rs @@ -1,21 +1,16 @@ -// SPDX-License-Identifier: MIT mod digest; mod dump; mod health; mod metrics; mod savings; -mod savings_build; mod stats; - #[cfg(test)] mod tests; - pub use digest::{build_digest, handle_digest}; pub use dump::handle_dump; -pub use health::{build_health_payload, build_readiness_payload, handle_health, handle_readiness}; +pub use health::{build_health_payload, handle_health, handle_readiness}; +#[cfg(test)] +pub(crate) use health::{include_private_runtime_details, redact_private_runtime_details}; +pub(crate) use metrics::*; pub use savings::handle_savings; pub use stats::handle_stats; - -pub(crate) use metrics::*; -pub(crate) use savings_build::*; -pub(crate) use health::{include_private_runtime_details, redact_private_runtime_details}; diff --git a/daemon-rs/src/handlers/health/savings.rs b/daemon-rs/src/handlers/health/savings.rs index 8654207b..b5ec7b64 100644 --- a/daemon-rs/src/handlers/health/savings.rs +++ b/daemon-rs/src/handlers/health/savings.rs @@ -1,656 +1,74 @@ -// SPDX-License-Identifier: MIT +use crate::handlers::{ensure_auth_rated, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; +use rusqlite::OpenFlags; +use serde_json::json; +use super::{savings_payload_cache, savings_payload_cache_if_fresh, SavingsPayloadSnapshot, SAVINGS_HISTORY_DAYS}; -use super::*; pub async fn handle_savings(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let now_unix_secs = Utc::now().timestamp(); - let cached_snapshot = match savings_payload_cache().lock() { - Ok(guard) => guard.clone(), - Err(_) => None, - }; - if let Some(snapshot) = savings_payload_cache_if_fresh(cached_snapshot, now_unix_secs) { - return json_response(StatusCode::OK, snapshot.payload); - } - let stale_snapshot = match savings_payload_cache().lock() { - Ok(guard) => guard.clone(), - Err(_) => None, - }; - let stale_or_error = |message: String| -> Response { - if let Some(snapshot) = stale_snapshot.clone() { + let now = chrono::Utc::now().timestamp(); + if let Ok(guard) = savings_payload_cache().lock() { + if let Some(snapshot) = savings_payload_cache_if_fresh(guard.clone(), now) { return json_response(StatusCode::OK, snapshot.payload); } - json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": message }), - ) - }; - - // Use an independent read-only connection for analytics so heavy /savings - // aggregation cannot block core dashboard endpoints waiting on the shared - // db_read mutex. - let conn = match rusqlite::Connection::open_with_flags( - &state.db_path, - OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, - ) { - Ok(conn) => conn, - Err(err) => { - return stale_or_error(format!("open savings reader failed: {err}")); - } + } + let payload = match build_savings_payload(&state) { + Ok(payload) => payload, + Err(err) => return json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), }; - let busy_timeout_ms = crate::db::SQLITE_BUSY_TIMEOUT_MS; - if let Err(err) = conn.execute_batch(&format!( - r#" - PRAGMA query_only = ON; - PRAGMA busy_timeout = {busy_timeout_ms}; - PRAGMA foreign_keys = ON; - PRAGMA mmap_size = 268435456; - PRAGMA cache_size = -8000; - PRAGMA temp_store = MEMORY; - "#, - )) { - return stale_or_error(format!("configure savings reader failed: {err}")); + if let Ok(mut cache) = savings_payload_cache().lock() { + *cache = Some(SavingsPayloadSnapshot { computed_at_unix_secs: now, payload: payload.clone() }); } - let savings_window_modifier = format!("-{SAVINGS_HISTORY_DAYS} days"); - let benchmark_source_pattern = format!("{}%", crate::compaction::BENCHMARK_SOURCE_AGENT_PREFIX); + json_response(StatusCode::OK, payload) +} - let (total_saved, total_served, total_baseline, total_boots): (i64, i64, i64, i64) = conn +fn build_savings_payload(state: &RuntimeState) -> Result { + let conn = rusqlite::Connection::open_with_flags(&state.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX) + .map_err(|err| format!("open savings reader failed: {err}"))?; + let window = format!("-{SAVINGS_HISTORY_DAYS} days"); + let benchmark = format!("{}%", crate::compaction::BENCHMARK_SOURCE_AGENT_PREFIX); + let (boot_saved, boot_served, boot_baseline, boots): (i64, i64, i64, i64) = conn .query_row( - "SELECT \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0)), 0), \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0)), 0), \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0)), 0), \ - COUNT(*) \ - FROM events \ - WHERE type = 'boot_savings' \ - AND created_at >= datetime('now', ?1) \ - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2)", - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], + "SELECT COALESCE(SUM(CAST(json_extract(data,'$.saved') AS INTEGER)),0), + COALESCE(SUM(CAST(json_extract(data,'$.served') AS INTEGER)),0), + COALESCE(SUM(CAST(json_extract(data,'$.baseline') AS INTEGER)),0), + COUNT(*) + FROM events + WHERE type='boot_savings' + AND created_at >= datetime('now', ?1) + AND LOWER(COALESCE(source_agent,'')) NOT LIKE LOWER(?2)", + rusqlite::params![window, benchmark], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), ) .unwrap_or((0, 0, 0, 0)); - - let mut boot_daily_stmt = match conn.prepare( - "SELECT \ - SUBSTR(created_at, 1, 10) AS day, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0)), 0) AS saved, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0)), 0) AS served, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0)), 0) AS baseline, \ - COUNT(*) AS boots \ - FROM events \ - WHERE type = 'boot_savings' \ - AND created_at >= datetime('now', ?1) \ - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) \ - AND created_at IS NOT NULL \ - GROUP BY day \ - ORDER BY day ASC", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare boot daily query failed: {e}")), - }; - let boot_daily_rows = match boot_daily_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - let day: Option = row.get(0)?; - Ok(( - day.unwrap_or_default(), - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(4)?, - )) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query boot daily failed: {e}")), - }; - drop(boot_daily_stmt); - let daily_arr: Vec = boot_daily_rows - .iter() - .filter_map(|(day, saved, served, baseline, boots)| { - if day.is_empty() { - None - } else { - Some(json!({ - "date": day, - "saved": saved, - "served": served, - "baseline": baseline, - "boots": boots - })) - } - }) - .collect(); - - let mut boot_by_agent_stmt = match conn.prepare( - "SELECT \ - COALESCE(NULLIF(TRIM(COALESCE(json_extract(data, '$.agent'), 'unknown')), ''), 'unknown') AS agent, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0)), 0) AS saved, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0)), 0) AS served, \ - COALESCE(SUM(COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0)), 0) AS baseline, \ - COUNT(*) AS boots \ - FROM events \ - WHERE type = 'boot_savings' \ - AND created_at >= datetime('now', ?1) \ - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) \ - GROUP BY agent", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare boot by-agent query failed: {e}")), - }; - let boot_by_agent_rows = match boot_by_agent_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(4)?, - )) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query boot by-agent failed: {e}")), - }; - drop(boot_by_agent_stmt); - let mut by_agent: BTreeMap = BTreeMap::new(); - for (agent, saved, served, baseline, boots) in boot_by_agent_rows { - by_agent.insert(agent, (saved, served, baseline, boots)); - } - let by_agent_arr: Vec = by_agent - .into_iter() - .map(|(agent, (saved, served, baseline, boots))| { - let percent = if baseline > 0 { - (saved * 100) / baseline - } else { - 0 - }; - json!({ - "agent": agent, - "saved": saved, - "served": served, - "baseline": baseline, - "boots": boots, - "percent": percent - }) - }) - .collect(); - - let mut recent_boot_stmt = match conn.prepare( - "SELECT data, created_at \ - FROM events \ - WHERE type = 'boot_savings' \ - AND created_at >= datetime('now', ?1) \ - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) \ - ORDER BY created_at DESC \ - LIMIT 20", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare recent boot query failed: {e}")), - }; - let recent_rows = match recent_boot_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - let data_str: String = row.get(0)?; - let created: String = row.get(1)?; - Ok((data_str, created)) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query recent boot rows failed: {e}")), - }; - drop(recent_boot_stmt); - let recent: Vec = recent_rows - .into_iter() - .map(|(data_str, created)| { - let d: Value = serde_json::from_str(&data_str).unwrap_or(json!({})); - let served = d.get("served").and_then(|v| v.as_i64()).unwrap_or(0); - let baseline = d.get("baseline").and_then(|v| v.as_i64()).unwrap_or(0); - let saved = d.get("saved").and_then(|v| v.as_i64()).unwrap_or(0); - let percent = d.get("percent").and_then(|v| v.as_i64()).unwrap_or(0); - let admitted = d.get("admitted").and_then(|v| v.as_i64()).unwrap_or(0); - let rejected = d.get("rejected").and_then(|v| v.as_i64()).unwrap_or(0); - let compression_ratio = if served > 0 { - ((baseline as f64 / served as f64) * 100.0).round() / 100.0 - } else { - 0.0 - }; - json!({ - "timestamp": created, - "agent": d.get("agent").and_then(|v| v.as_str()).unwrap_or("unknown"), - "served": served, - "baseline": baseline, - "saved": saved, - "percent": percent, - "admitted": admitted, - "rejected": rejected, - "compressionRatio": compression_ratio - }) - }) - .collect(); - - let mut by_operation: BTreeMap = BTreeMap::new(); - for op in ["recall", "store", "boot", "tool"] { - by_operation.insert(op.to_string(), (0, 0, 0, 0)); - } - - let mut rollup_op_stmt = match conn.prepare( - "SELECT operation, \ - COALESCE(SUM(saved), 0) AS saved, \ - COALESCE(SUM(served), 0) AS served, \ - COALESCE(SUM(baseline), 0) AS baseline, \ - COALESCE(SUM(events), 0) AS events \ - FROM event_savings_rollups \ - WHERE day >= date('now', ?1) \ - GROUP BY operation", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare rollup operation query failed: {e}")), - }; - let rollup_op_rows = - match rollup_op_stmt.query_map(params![savings_window_modifier.clone()], |row| { - let operation: String = row.get(0)?; - let saved: i64 = row.get(1)?; - let served: i64 = row.get(2)?; - let baseline: i64 = row.get(3)?; - let events: i64 = row.get(4)?; - Ok((operation, saved, served, baseline, events)) - }) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => { - return stale_or_error(format!("query rollup operation aggregates failed: {e}")); - } - }; - drop(rollup_op_stmt); - for (operation, saved, served, baseline, events) in rollup_op_rows { - let entry = by_operation.entry(operation).or_insert((0, 0, 0, 0)); - entry.0 += saved; - entry.1 += served; - entry.2 += baseline; - entry.3 += events; - } - - let mut op_stmt = match conn.prepare( - "SELECT \ - CASE \ - WHEN type = 'recall_query' THEN 'recall' \ - WHEN type = 'store_savings' THEN 'store' \ - WHEN type = 'tool_call_savings' THEN 'tool' \ - END AS operation, \ - COALESCE(SUM(CASE \ - WHEN type = 'recall_query' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - WHEN type = 'store_savings' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - WHEN type = 'tool_call_savings' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - ELSE 0 END), 0) AS saved, \ - COALESCE(SUM(CASE \ - WHEN type = 'recall_query' THEN COALESCE(CAST(json_extract(data, '$.spent') AS INTEGER), COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0)) \ - WHEN type = 'store_savings' THEN COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0) \ - WHEN type = 'tool_call_savings' THEN COALESCE(CAST(json_extract(data, '$.served') AS INTEGER), 0) \ - ELSE 0 END), 0) AS served, \ - COALESCE(SUM(CASE \ - WHEN type = 'recall_query' THEN COALESCE(CAST(json_extract(data, '$.budget') AS INTEGER), COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0)) \ - WHEN type = 'store_savings' THEN COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0) \ - WHEN type = 'tool_call_savings' THEN COALESCE(CAST(json_extract(data, '$.baseline') AS INTEGER), 0) \ - ELSE 0 END), 0) AS baseline, \ - COUNT(*) AS events \ - FROM events \ - WHERE type IN ('recall_query', 'store_savings', 'tool_call_savings') \ - AND created_at >= datetime('now', ?1) \ - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) \ - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) \ - GROUP BY operation", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare operation aggregate query failed: {e}")), - }; - let op_rows = match op_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - let operation: String = row.get(0)?; - let saved: i64 = row.get(1)?; - let served: i64 = row.get(2)?; - let baseline: i64 = row.get(3)?; - let events: i64 = row.get(4)?; - Ok((operation, saved, served, baseline, events)) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query operation aggregates failed: {e}")), - }; - drop(op_stmt); - for (operation, saved, served, baseline, events) in op_rows { - let entry = by_operation.entry(operation).or_insert((0, 0, 0, 0)); - entry.0 += saved; - entry.1 += served; - entry.2 += baseline; - entry.3 += events; - } - - let mut daily_savings_all: BTreeMap = BTreeMap::new(); - let mut recall_daily: BTreeMap = BTreeMap::new(); - let mut rollup_daily_stmt = match conn.prepare( - "SELECT day, \ - COALESCE(SUM(saved), 0) AS saved_delta, \ - COALESCE(SUM(hits), 0) AS hits, \ - COALESCE(SUM(misses), 0) AS misses \ - FROM event_savings_rollups \ - WHERE day >= date('now', ?1) \ - GROUP BY day \ - ORDER BY day ASC", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare rollup daily savings query failed: {e}")), - }; - let rollup_daily_rows = - match rollup_daily_stmt.query_map(params![savings_window_modifier.clone()], |row| { - let day: String = row.get(0)?; - let saved_delta: i64 = row.get(1)?; - let hits: i64 = row.get(2)?; - let misses: i64 = row.get(3)?; - Ok((day, saved_delta, hits, misses)) - }) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query rollup daily savings failed: {e}")), - }; - drop(rollup_daily_stmt); - for (day, saved_delta, hits, misses) in rollup_daily_rows { - if day.is_empty() { - continue; - } - *daily_savings_all.entry(day.clone()).or_insert(0) += saved_delta; - if hits + misses > 0 { - let entry = recall_daily.entry(day).or_insert((0, 0)); - entry.0 += hits; - entry.1 += misses; - } - } - - let mut daily_stmt = match conn.prepare( - "SELECT \ - SUBSTR(created_at, 1, 10) AS day, \ - COALESCE(SUM(CASE \ - WHEN type = 'boot_savings' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - WHEN type = 'recall_query' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - WHEN type = 'store_savings' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - WHEN type = 'tool_call_savings' THEN COALESCE(CAST(json_extract(data, '$.saved') AS INTEGER), 0) \ - ELSE 0 END), 0) AS saved_delta, \ - SUM(CASE \ - WHEN type = 'recall_query' AND COALESCE(CAST(json_extract(data, '$.hits') AS INTEGER), 0) > 0 THEN 1 \ - ELSE 0 END) AS hits, \ - SUM(CASE \ - WHEN type = 'recall_query' AND COALESCE(CAST(json_extract(data, '$.hits') AS INTEGER), 0) > 0 THEN 0 \ - WHEN type = 'recall_query' THEN 1 \ - ELSE 0 END) AS misses \ - FROM events \ - WHERE type IN ('boot_savings', 'recall_query', 'store_savings', 'tool_call_savings') - AND created_at >= datetime('now', ?1) - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) - AND created_at IS NOT NULL \ - GROUP BY day \ - ORDER BY day ASC", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare daily savings query failed: {e}")), - }; - let daily_rows = match daily_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - let day: Option = row.get(0)?; - let saved_delta: i64 = row.get(1)?; - let hits: i64 = row.get(2)?; - let misses: i64 = row.get(3)?; - Ok((day.unwrap_or_default(), saved_delta, hits, misses)) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query daily savings failed: {e}")), - }; - drop(daily_stmt); - for (day, saved_delta, hits, misses) in daily_rows { - if !day.is_empty() { - *daily_savings_all.entry(day.clone()).or_insert(0) += saved_delta; - if hits + misses > 0 { - let entry = recall_daily.entry(day).or_insert((0, 0)); - entry.0 += hits; - entry.1 += misses; - } - } - } - - let mut activity_heatmap_map: BTreeMap<(String, i64), i64> = BTreeMap::new(); - let mut rollup_heatmap_stmt = match conn.prepare( - "SELECT \ - CAST(strftime('%w', day) AS INTEGER) AS weekday, \ - hour, \ - COALESCE(SUM(events), 0) AS cnt \ - FROM event_savings_rollups \ - WHERE day >= date('now', ?1) \ - GROUP BY weekday, hour", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare rollup heatmap query failed: {e}")), - }; - let rollup_heatmap_rows = - match rollup_heatmap_stmt.query_map(params![savings_window_modifier.clone()], |row| { - let weekday: Option = row.get(0)?; - let hour: Option = row.get(1)?; - let count: i64 = row.get(2)?; - Ok((weekday, hour, count)) - }) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query rollup heatmap failed: {e}")), - }; - drop(rollup_heatmap_stmt); - for (weekday, hour, count) in rollup_heatmap_rows { - if let (Some(day), Some(hour)) = (weekday, hour) { - let day_name = weekday_name_from_sqlite(day).to_string(); - *activity_heatmap_map.entry((day_name, hour)).or_insert(0) += count; - } - } - - let mut heatmap_stmt = match conn.prepare( - "SELECT \ - CAST(strftime('%w', REPLACE(SUBSTR(created_at, 1, 19), 'T', ' ')) AS INTEGER) AS weekday, \ - CAST(strftime('%H', REPLACE(SUBSTR(created_at, 1, 19), 'T', ' ')) AS INTEGER) AS hour, \ - COUNT(*) AS cnt \ - FROM events \ - WHERE type IN ('boot_savings', 'recall_query', 'store_savings', 'tool_call_savings') + let (recall_saved, recall_spent, recalls): (i64, i64, i64) = conn + .query_row( + "SELECT COALESCE(SUM(CAST(json_extract(data,'$.saved') AS INTEGER)),0), + COALESCE(SUM(CAST(json_extract(data,'$.spent') AS INTEGER)),0), + COUNT(*) + FROM events + WHERE type='recall_query' AND created_at >= datetime('now', ?1) - AND LOWER(COALESCE(source_agent, '')) NOT LIKE LOWER(?2) - AND LOWER(COALESCE(json_extract(data, '$.source_agent'), '')) NOT LIKE LOWER(?2) - AND LOWER(COALESCE(json_extract(data, '$.agent'), '')) NOT LIKE LOWER(?2) - AND created_at IS NOT NULL \ - GROUP BY weekday, hour", - ) { - Ok(stmt) => stmt, - Err(e) => return stale_or_error(format!("prepare activity heatmap query failed: {e}")), - }; - let heatmap_rows = match heatmap_stmt.query_map( - params![ - savings_window_modifier.clone(), - benchmark_source_pattern.clone() - ], - |row| { - let weekday: Option = row.get(0)?; - let hour: Option = row.get(1)?; - let count: i64 = row.get(2)?; - Ok((weekday, hour, count)) - }, - ) { - Ok(iter) => iter.filter_map(|row| row.ok()).collect::>(), - Err(e) => return stale_or_error(format!("query activity heatmap failed: {e}")), - }; - drop(heatmap_stmt); - for (weekday, hour, count) in heatmap_rows { - if let (Some(day), Some(hour)) = (weekday, hour) { - let day_name = weekday_name_from_sqlite(day).to_string(); - *activity_heatmap_map.entry((day_name, hour)).or_insert(0) += count; - } - } - - // Weighted average by baseline (not simple average). - // Prevents tiny boots with 0% from dragging down 99% large boots. - let avg_percent = if total_baseline > 0 { - (total_saved * 100) / total_baseline - } else { - 0 - }; - by_operation.insert( - "boot".to_string(), - (total_saved, total_served, total_baseline, total_boots), - ); - let avg_saved_per_boot = if total_boots > 0 { - total_saved / total_boots - } else { - 0 - }; - let avg_served_per_boot = if total_boots > 0 { - total_served / total_boots - } else { - 0 - }; - let avg_baseline_per_boot = if total_boots > 0 { - total_baseline / total_boots - } else { - 0 - }; - - let by_operation_arr: Vec = ["recall", "store", "boot", "tool"] - .iter() - .map(|op| { - let (saved, served, baseline, events) = - by_operation.get(*op).copied().unwrap_or((0, 0, 0, 0)); - let percent = if baseline > 0 { - (saved * 100) / baseline - } else { - 0 - }; - json!({ - "operation": op, - "saved": saved, - "served": served, - "baseline": baseline, - "events": events, - "percent": percent - }) - }) - .collect(); - - let mut running_saved = 0_i64; - let cumulative: Vec = daily_savings_all - .into_iter() - .map(|(date, saved_delta)| { - running_saved += saved_delta; - json!({ - "date": date, - "savedDelta": saved_delta, - "savedTotal": running_saved - }) - }) - .collect(); - - let recall_trend: Vec = recall_daily - .into_iter() - .map(|(date, (hits, misses))| { - let queries = hits + misses; - let hit_rate = if queries > 0 { - ((hits as f64 / queries as f64) * 1000.0).round() / 10.0 - } else { - 0.0 - }; - json!({ - "date": date, - "hits": hits, - "misses": misses, - "queries": queries, - "hitRatePct": hit_rate - }) - }) - .collect(); - - let activity_heatmap: Vec = activity_heatmap_map - .into_iter() - .map(|((day, hour), count)| { - json!({ - "day": day, - "hour": hour, - "count": count - }) - }) - .collect(); - - let payload = json!({ - "summary": { - "totalSaved": total_saved, - "totalServed": total_served, - "totalBaseline": total_baseline, - "avgPercent": avg_percent, - "totalBoots": total_boots, - "avgSavedPerBoot": avg_saved_per_boot, - "avgServedPerBoot": avg_served_per_boot, - "avgBaselinePerBoot": avg_baseline_per_boot, - "scope": "boot_prompt_plus_event_operations", - "note": "Boot savings are precise from /boot events. Recall/store/tool figures are event-derived estimates when instrumentation is available. Analytics scope is the last 30 days." - }, - "daily": daily_arr, - "byAgent": by_agent_arr, - "recent": recent, - "byOperation": by_operation_arr, - "cumulative": cumulative, - "recallTrend": recall_trend, - "activityHeatmap": activity_heatmap, - }); - - if let Ok(mut cache) = savings_payload_cache().lock() { - *cache = Some(SavingsPayloadSnapshot { - computed_at_unix_secs: now_unix_secs, - payload: payload.clone(), - }); - } - - json_response(StatusCode::OK, payload) + AND LOWER(COALESCE(source_agent,'')) NOT LIKE LOWER(?2)", + rusqlite::params![format!("-{SAVINGS_HISTORY_DAYS} days"), benchmark], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap_or((0, 0, 0)); + let total_saved = boot_saved + recall_saved; + let total_served = boot_served + recall_spent; + let total_baseline = boot_baseline + recall_spent + recall_saved; + let percent = if total_baseline > 0 { (total_saved * 100) / total_baseline } else { 0 }; + Ok(json!({ + "schemaVersion": 1, + "windowDays": SAVINGS_HISTORY_DAYS, + "totals": {"saved": total_saved, "served": total_served, "baseline": total_baseline, "percent": percent}, + "boot": {"saved": boot_saved, "served": boot_served, "baseline": boot_baseline, "boots": boots}, + "recall": {"saved": recall_saved, "spent": recall_spent, "queries": recalls} + })) } - diff --git a/daemon-rs/src/handlers/health/savings_build.rs b/daemon-rs/src/handlers/health/savings_build.rs deleted file mode 100644 index b5e45adb..00000000 --- a/daemon-rs/src/handlers/health/savings_build.rs +++ /dev/null @@ -1,566 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; -// ─── GET /savings ──────────────────────────────────────────────────────────── - -pub(crate) fn value_i64_any(payload: &Value, keys: &[&str]) -> i64 { - keys.iter() - .find_map(|key| payload.get(*key).and_then(|v| v.as_i64())) - .unwrap_or(0) -} - -pub(crate) fn method_count(payload: &Value, method: &str) -> i64 { - payload - .get("method_breakdown") - .and_then(|value| value.get(method)) - .and_then(|value| value.as_i64()) - .unwrap_or(0) -} - -pub(crate) fn classify_recall_tier_from_payload(payload: &Value) -> String { - if let Some(tier) = payload.get("tier").and_then(|value| value.as_str()) { - if !tier.trim().is_empty() { - return tier.to_string(); - } - } - - if payload - .get("cached") - .and_then(|value| value.as_bool()) - .unwrap_or(false) - { - return "cache_hit".to_string(); - } - - let mode = payload - .get("mode") - .and_then(|value| value.as_str()) - .unwrap_or_default(); - if mode == "headlines" { - return "headlines".to_string(); - } - if mode == "semantic" { - return "semantic_only".to_string(); - } - - let keyword = method_count(payload, "keyword"); - let semantic = method_count(payload, "semantic"); - let hybrid = method_count(payload, "hybrid"); - let crystal = method_count(payload, "crystal"); - - if hybrid > 0 || (keyword > 0 && semantic > 0) { - if crystal > 0 { - return "hybrid_crystal".to_string(); - } - return "hybrid_fusion".to_string(); - } - if keyword > 0 { - if crystal > 0 { - return "keyword_crystal".to_string(); - } - return "keyword_only".to_string(); - } - if semantic > 0 { - if crystal > 0 { - return "semantic_crystal".to_string(); - } - return "semantic_only".to_string(); - } - if crystal > 0 { - return "crystal_only".to_string(); - } - - "unknown".to_string() -} - -pub(crate) fn round1(value: f64) -> f64 { - (value * 10.0).round() / 10.0 -} - -pub(crate) fn round4(value: f64) -> f64 { - (value * 10_000.0).round() / 10_000.0 -} - -pub(crate) fn normalize_shadow_status(status: &str) -> &'static str { - let normalized = status.trim().to_ascii_lowercase(); - match normalized.as_str() { - "ok" => "ok", - "unavailable" => "unavailable", - "error" => "error", - "skipped" => "skipped", - _ => "unknown", - } -} - -pub(crate) const SHADOW_GATE_MIN_PROBED_EVENTS: i64 = 25; -pub(crate) const SHADOW_GATE_MIN_OK_SAMPLES: i64 = 15; -pub(crate) const SHADOW_GATE_MAX_UNAVAILABLE_RATE: f64 = 0.35; -pub(crate) const SHADOW_GATE_MAX_ERROR_RATE: f64 = 0.05; -pub(crate) const SHADOW_GATE_MIN_OK_OVERLAP_RATIO: f64 = 0.60; -pub(crate) const SHADOW_GATE_MIN_OK_JACCARD: f64 = 0.45; -pub(crate) const SHADOW_GATE_MAX_MEAN_ABS_RANK_DELTA: f64 = 1.25; -pub(crate) const SHADOW_GATE_MIN_TOP1_MATCH_RATE: f64 = 0.60; - -pub(crate) struct ShadowOkMetricSamples { - overlap_ratio: i64, - jaccard: i64, - mean_abs_rank_delta: i64, - top1_match: i64, -} - -pub(crate) struct ShadowOkMetricAverages { - overlap_ratio: Option, - jaccard: Option, - mean_abs_rank_delta: Option, - top1_match_rate: Option, -} - -pub(crate) fn build_shadow_semantic_gate( - shadow_status_counts: &BTreeMap, - ok_samples: i64, - ok_metric_samples: &ShadowOkMetricSamples, - ok_metric_averages: &ShadowOkMetricAverages, -) -> Value { - let ok_count = *shadow_status_counts.get("ok").unwrap_or(&0); - let unavailable_count = *shadow_status_counts.get("unavailable").unwrap_or(&0); - let error_count = *shadow_status_counts.get("error").unwrap_or(&0); - let unknown_count = *shadow_status_counts.get("unknown").unwrap_or(&0); - let skipped_count = *shadow_status_counts.get("skipped").unwrap_or(&0); - - // "Probed" excludes cache-hit skips, because no shadow query was attempted. - let probed_events = ok_count + unavailable_count + error_count + unknown_count; - let unavailable_rate = if probed_events > 0 { - round4(unavailable_count as f64 / probed_events as f64) - } else { - 0.0 - }; - let error_rate = if probed_events > 0 { - round4(error_count as f64 / probed_events as f64) - } else { - 0.0 - }; - - let mut blockers: Vec = Vec::new(); - if probed_events < SHADOW_GATE_MIN_PROBED_EVENTS { - blockers.push("insufficient_shadow_samples".to_string()); - } - if ok_samples < SHADOW_GATE_MIN_OK_SAMPLES { - blockers.push("insufficient_ok_samples".to_string()); - } - if unavailable_rate > SHADOW_GATE_MAX_UNAVAILABLE_RATE { - blockers.push("unavailable_rate_above_gate".to_string()); - } - if error_rate > SHADOW_GATE_MAX_ERROR_RATE { - blockers.push("error_rate_above_gate".to_string()); - } - if ok_metric_samples.overlap_ratio > 0 - && ok_metric_samples.overlap_ratio < SHADOW_GATE_MIN_OK_SAMPLES - { - blockers.push("insufficient_overlap_ratio_samples".to_string()); - } - match ok_metric_averages.overlap_ratio { - Some(value) if value < SHADOW_GATE_MIN_OK_OVERLAP_RATIO => { - blockers.push("overlap_ratio_below_gate".to_string()); - } - None => blockers.push("missing_overlap_signal".to_string()), - _ => {} - } - if ok_metric_samples.jaccard > 0 && ok_metric_samples.jaccard < SHADOW_GATE_MIN_OK_SAMPLES { - blockers.push("insufficient_jaccard_samples".to_string()); - } - match ok_metric_averages.jaccard { - Some(value) if value < SHADOW_GATE_MIN_OK_JACCARD => { - blockers.push("jaccard_below_gate".to_string()); - } - None => blockers.push("missing_jaccard_signal".to_string()), - _ => {} - } - if ok_metric_samples.mean_abs_rank_delta > 0 - && ok_metric_samples.mean_abs_rank_delta < SHADOW_GATE_MIN_OK_SAMPLES - { - blockers.push("insufficient_rank_delta_samples".to_string()); - } - match ok_metric_averages.mean_abs_rank_delta { - Some(value) if value > SHADOW_GATE_MAX_MEAN_ABS_RANK_DELTA => { - blockers.push("mean_abs_rank_delta_above_gate".to_string()); - } - None => blockers.push("missing_rank_delta_signal".to_string()), - _ => {} - } - if ok_metric_samples.top1_match > 0 && ok_metric_samples.top1_match < SHADOW_GATE_MIN_OK_SAMPLES - { - blockers.push("insufficient_top1_match_samples".to_string()); - } - match ok_metric_averages.top1_match_rate { - Some(value) if value < SHADOW_GATE_MIN_TOP1_MATCH_RATE => { - blockers.push("top1_match_rate_below_gate".to_string()); - } - None => blockers.push("missing_top1_match_signal".to_string()), - _ => {} - } - - let ready = blockers.is_empty(); - json!({ - "ready": ready, - "decision": if ready { "ready_for_vec0_trial" } else { "hold" }, - "target": "sqlite_vec_production_routing", - "blockers": blockers, - "metrics": { - "probed_events": probed_events, - "ok_count": ok_count, - "unavailable_count": unavailable_count, - "error_count": error_count, - "unknown_count": unknown_count, - "skipped_count": skipped_count, - "ok_samples": ok_samples, - "ok_overlap_samples": ok_metric_samples.overlap_ratio, - "ok_jaccard_samples": ok_metric_samples.jaccard, - "ok_rank_delta_samples": ok_metric_samples.mean_abs_rank_delta, - "ok_top1_match_samples": ok_metric_samples.top1_match, - "ok_overlap_ratio_avg": ok_metric_averages.overlap_ratio, - "ok_jaccard_avg": ok_metric_averages.jaccard, - "ok_mean_abs_rank_delta_avg": ok_metric_averages.mean_abs_rank_delta, - "ok_top1_match_rate": ok_metric_averages.top1_match_rate, - "unavailable_rate": unavailable_rate, - "error_rate": error_rate - }, - "thresholds": { - "min_probed_events": SHADOW_GATE_MIN_PROBED_EVENTS, - "min_ok_samples": SHADOW_GATE_MIN_OK_SAMPLES, - "max_unavailable_rate": SHADOW_GATE_MAX_UNAVAILABLE_RATE, - "max_error_rate": SHADOW_GATE_MAX_ERROR_RATE, - "min_ok_overlap_ratio": SHADOW_GATE_MIN_OK_OVERLAP_RATIO, - "min_ok_jaccard": SHADOW_GATE_MIN_OK_JACCARD, - "max_mean_abs_rank_delta": SHADOW_GATE_MAX_MEAN_ABS_RANK_DELTA, - "min_top1_match_rate": SHADOW_GATE_MIN_TOP1_MATCH_RATE - } - }) -} - -pub(crate) fn build_recall_stats_payload_from_rows(rows: &[(String, String)]) -> Value { - let mut tier_counts: BTreeMap = BTreeMap::new(); - let mut tier_latency_sum: BTreeMap = BTreeMap::new(); - let mut tier_latency_samples: BTreeMap = BTreeMap::new(); - let mut mode_counts: BTreeMap = BTreeMap::new(); - let mut shadow_status_counts: BTreeMap = BTreeMap::new(); - - let mut total_budget = 0_i64; - let mut total_spent = 0_i64; - let mut total_saved = 0_i64; - let mut total_hits = 0_i64; - let mut shadow_ok_overlap_ratio_sum = 0.0_f64; - let mut shadow_ok_overlap_ratio_samples = 0_i64; - let mut shadow_ok_jaccard_sum = 0.0_f64; - let mut shadow_ok_jaccard_samples = 0_i64; - let mut shadow_ok_rank_delta_sum = 0.0_f64; - let mut shadow_ok_rank_delta_samples = 0_i64; - let mut shadow_ok_top1_match_sum = 0.0_f64; - let mut shadow_ok_top1_match_samples = 0_i64; - - let mut latency_total = 0_i64; - let mut latency_samples = 0_i64; - let mut recent: Vec = Vec::new(); - - for (data_str, created_at) in rows { - let payload: Value = serde_json::from_str(data_str).unwrap_or_else(|_| json!({})); - let mode = payload - .get("mode") - .and_then(|value| value.as_str()) - .unwrap_or("unknown") - .to_string(); - *mode_counts.entry(mode.clone()).or_insert(0) += 1; - - let tier = classify_recall_tier_from_payload(&payload); - *tier_counts.entry(tier.clone()).or_insert(0) += 1; - - let budget = value_i64_any(&payload, &["budget", "baseline"]); - let spent = value_i64_any(&payload, &["spent", "served"]); - let saved = value_i64_any(&payload, &["saved"]); - let hits = value_i64_any(&payload, &["hits", "results"]); - - total_budget += budget.max(0); - total_spent += spent.max(0); - total_saved += saved; - total_hits += hits.max(0); - - if let Some(latency_ms) = payload.get("latency_ms").and_then(|value| value.as_i64()) { - if latency_ms >= 0 { - latency_total += latency_ms; - latency_samples += 1; - *tier_latency_sum.entry(tier.clone()).or_insert(0) += latency_ms; - *tier_latency_samples.entry(tier.clone()).or_insert(0) += 1; - } - } - if let Some(shadow_semantic) = payload - .get("shadow_semantic") - .and_then(|value| value.as_object()) - { - let status = shadow_semantic - .get("status") - .and_then(|value| value.as_str()) - .map(normalize_shadow_status) - .unwrap_or("unknown") - .to_string(); - *shadow_status_counts.entry(status.clone()).or_insert(0) += 1; - - if status == "ok" { - if let Some(overlap_ratio) = shadow_semantic - .get("overlapRatio") - .and_then(|value| value.as_f64()) - { - shadow_ok_overlap_ratio_sum += overlap_ratio; - shadow_ok_overlap_ratio_samples += 1; - } - if let Some(jaccard) = shadow_semantic - .get("jaccard") - .and_then(|value| value.as_f64()) - { - shadow_ok_jaccard_sum += jaccard; - shadow_ok_jaccard_samples += 1; - } - if let Some(mean_abs_rank_delta) = shadow_semantic - .get("meanAbsRankDelta") - .and_then(|value| value.as_f64()) - { - shadow_ok_rank_delta_sum += mean_abs_rank_delta; - shadow_ok_rank_delta_samples += 1; - } - if let Some(top1_match) = shadow_semantic - .get("top1Match") - .and_then(|value| value.as_bool()) - { - shadow_ok_top1_match_sum += if top1_match { 1.0 } else { 0.0 }; - shadow_ok_top1_match_samples += 1; - } - } - } - - recent.push(json!({ - "timestamp": created_at, - "mode": mode, - "tier": tier, - "budget": budget, - "spent": spent, - "saved": saved, - "hits": hits, - "cached": payload.get("cached").and_then(|value| value.as_bool()).unwrap_or(false), - "latencyMs": payload.get("latency_ms").and_then(|value| value.as_i64()), - })); - } - - let total_recalls = rows.len() as i64; - let avg_latency_ms = if latency_samples > 0 { - round1(latency_total as f64 / latency_samples as f64) - } else { - 0.0 - }; - let savings_pct_vs_budget = if total_budget > 0 { - round1((total_saved as f64 / total_budget as f64) * 100.0) - } else { - 0.0 - }; - let shadow_overlap_ratio_avg = if shadow_ok_overlap_ratio_samples > 0 { - Some(round4( - shadow_ok_overlap_ratio_sum / shadow_ok_overlap_ratio_samples as f64, - )) - } else { - None - }; - let shadow_jaccard_avg = if shadow_ok_jaccard_samples > 0 { - Some(round4( - shadow_ok_jaccard_sum / shadow_ok_jaccard_samples as f64, - )) - } else { - None - }; - let shadow_mean_abs_rank_delta_avg = if shadow_ok_rank_delta_samples > 0 { - Some(round4( - shadow_ok_rank_delta_sum / shadow_ok_rank_delta_samples as f64, - )) - } else { - None - }; - let shadow_top1_match_rate = if shadow_ok_top1_match_samples > 0 { - Some(round4( - shadow_ok_top1_match_sum / shadow_ok_top1_match_samples as f64, - )) - } else { - None - }; - let ok_metric_samples = ShadowOkMetricSamples { - overlap_ratio: shadow_ok_overlap_ratio_samples, - jaccard: shadow_ok_jaccard_samples, - mean_abs_rank_delta: shadow_ok_rank_delta_samples, - top1_match: shadow_ok_top1_match_samples, - }; - let ok_metric_averages = ShadowOkMetricAverages { - overlap_ratio: shadow_overlap_ratio_avg, - jaccard: shadow_jaccard_avg, - mean_abs_rank_delta: shadow_mean_abs_rank_delta_avg, - top1_match_rate: shadow_top1_match_rate, - }; - let shadow_ok_samples = [ - ok_metric_samples.overlap_ratio, - ok_metric_samples.jaccard, - ok_metric_samples.mean_abs_rank_delta, - ok_metric_samples.top1_match, - ] - .into_iter() - .min() - .unwrap_or(0); - let shadow_gate = build_shadow_semantic_gate( - &shadow_status_counts, - shadow_ok_samples, - &ok_metric_samples, - &ok_metric_averages, - ); - - let tier_distribution: Vec = tier_counts - .iter() - .map(|(tier, count)| { - let percent = if total_recalls > 0 { - round1((*count as f64 / total_recalls as f64) * 100.0) - } else { - 0.0 - }; - let avg_tier_latency = match ( - tier_latency_sum.get(tier).copied(), - tier_latency_samples.get(tier).copied(), - ) { - (Some(sum), Some(samples)) if samples > 0 => round1(sum as f64 / samples as f64), - _ => 0.0, - }; - json!({ - "tier": tier, - "count": count, - "percent": percent, - "avgLatencyMs": avg_tier_latency - }) - }) - .collect(); - - let tier_distribution_map: Value = json!(tier_counts - .iter() - .map(|(tier, count)| (tier.clone(), json!(count))) - .collect::>()); - - let avg_latency_map: Value = { - let mut map = serde_json::Map::new(); - map.insert("overall".to_string(), json!(avg_latency_ms)); - for entry in &tier_distribution { - if let (Some(tier), Some(avg)) = ( - entry.get("tier").and_then(|value| value.as_str()), - entry.get("avgLatencyMs"), - ) { - map.insert(tier.to_string(), avg.clone()); - } - } - Value::Object(map) - }; - - recent.sort_by(|a, b| { - let a_ts = a - .get("timestamp") - .and_then(|value| value.as_str()) - .unwrap_or(""); - let b_ts = b - .get("timestamp") - .and_then(|value| value.as_str()) - .unwrap_or(""); - b_ts.cmp(a_ts) - }); - recent.truncate(30); - - json!({ - "summary": { - "totalRecalls": total_recalls, - "totalHits": total_hits, - "totalBudget": total_budget, - "totalSpent": total_spent, - "totalSaved": total_saved, - "savingsPctVsBudget": savings_pct_vs_budget, - "avgLatencyMs": avg_latency_ms - }, - "tierDistribution": tier_distribution, - "tier_distribution": tier_distribution_map, - "avg_latency_ms": avg_latency_map, - "estimated_savings": { - "vs_always_full_pipeline_pct": savings_pct_vs_budget - }, - "modeCounts": mode_counts, - "shadow_semantic": { - "status_counts": shadow_status_counts, - "ok_samples": shadow_ok_samples, - "ok_overlap_samples": ok_metric_samples.overlap_ratio, - "ok_jaccard_samples": ok_metric_samples.jaccard, - "ok_rank_delta_samples": ok_metric_samples.mean_abs_rank_delta, - "ok_top1_match_samples": ok_metric_samples.top1_match, - "ok_overlap_ratio_avg": ok_metric_averages.overlap_ratio, - "ok_jaccard_avg": ok_metric_averages.jaccard, - "ok_mean_abs_rank_delta_avg": ok_metric_averages.mean_abs_rank_delta, - "ok_top1_match_rate": ok_metric_averages.top1_match_rate - }, - "shadowSemantic": { - "statusCounts": shadow_status_counts, - "okSamples": shadow_ok_samples, - "okOverlapSamples": ok_metric_samples.overlap_ratio, - "okJaccardSamples": ok_metric_samples.jaccard, - "okRankDeltaSamples": ok_metric_samples.mean_abs_rank_delta, - "okTop1MatchSamples": ok_metric_samples.top1_match, - "okOverlapRatioAvg": ok_metric_averages.overlap_ratio, - "okJaccardAvg": ok_metric_averages.jaccard, - "okMeanAbsRankDeltaAvg": ok_metric_averages.mean_abs_rank_delta, - "okTop1MatchRate": ok_metric_averages.top1_match_rate - }, - "shadow_semantic_gate": shadow_gate, - "shadowSemanticGate": { - "ready": shadow_gate["ready"], - "decision": shadow_gate["decision"], - "target": shadow_gate["target"], - "blockers": shadow_gate["blockers"], - "metrics": { - "probedEvents": shadow_gate["metrics"]["probed_events"], - "okCount": shadow_gate["metrics"]["ok_count"], - "unavailableCount": shadow_gate["metrics"]["unavailable_count"], - "errorCount": shadow_gate["metrics"]["error_count"], - "unknownCount": shadow_gate["metrics"]["unknown_count"], - "skippedCount": shadow_gate["metrics"]["skipped_count"], - "okSamples": shadow_gate["metrics"]["ok_samples"], - "okOverlapSamples": shadow_gate["metrics"]["ok_overlap_samples"], - "okJaccardSamples": shadow_gate["metrics"]["ok_jaccard_samples"], - "okRankDeltaSamples": shadow_gate["metrics"]["ok_rank_delta_samples"], - "okTop1MatchSamples": shadow_gate["metrics"]["ok_top1_match_samples"], - "okOverlapRatioAvg": shadow_gate["metrics"]["ok_overlap_ratio_avg"], - "okJaccardAvg": shadow_gate["metrics"]["ok_jaccard_avg"], - "okMeanAbsRankDeltaAvg": shadow_gate["metrics"]["ok_mean_abs_rank_delta_avg"], - "okTop1MatchRate": shadow_gate["metrics"]["ok_top1_match_rate"], - "unavailableRate": shadow_gate["metrics"]["unavailable_rate"], - "errorRate": shadow_gate["metrics"]["error_rate"] - }, - "thresholds": { - "minProbedEvents": shadow_gate["thresholds"]["min_probed_events"], - "minOkSamples": shadow_gate["thresholds"]["min_ok_samples"], - "maxUnavailableRate": shadow_gate["thresholds"]["max_unavailable_rate"], - "maxErrorRate": shadow_gate["thresholds"]["max_error_rate"], - "minOkOverlapRatio": shadow_gate["thresholds"]["min_ok_overlap_ratio"], - "minOkJaccard": shadow_gate["thresholds"]["min_ok_jaccard"], - "maxMeanAbsRankDelta": shadow_gate["thresholds"]["max_mean_abs_rank_delta"], - "minTop1MatchRate": shadow_gate["thresholds"]["min_top1_match_rate"] - } - }, - "recent": recent - }) -} - diff --git a/daemon-rs/src/handlers/health/stats.rs b/daemon-rs/src/handlers/health/stats.rs index 490ffa62..b77a242b 100644 --- a/daemon-rs/src/handlers/health/stats.rs +++ b/daemon-rs/src/handlers/health/stats.rs @@ -1,36 +1,20 @@ -// SPDX-License-Identifier: MIT +use crate::handlers::{ensure_auth_rated, json_response}; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use chrono::Utc; -use rusqlite::{params, OpenFlags}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashSet}; -use std::sync::{Mutex, OnceLock}; -use std::time::{Duration, Instant}; -use crate::handlers::{client_ip, ensure_auth_rated, ensure_ssrf_protection, json_response, truncate_chars}; -use crate::state::RuntimeState; - - -use super::*; +use serde_json::json; pub async fn handle_stats(State(state): State, headers: HeaderMap) -> Response { if let Err(resp) = ensure_auth_rated(&headers, &state).await { return resp; } - let conn = state.db_read.lock().await; - let mut stmt = match conn.prepare( - "SELECT data, created_at FROM events WHERE type = 'recall_query' ORDER BY created_at ASC", - ) { + let mut stmt = match conn.prepare("SELECT data, created_at FROM events WHERE type = 'recall_query' ORDER BY created_at ASC") { Ok(stmt) => stmt, Err(e) => { - return json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": e.to_string() }), - ); + return json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":e.to_string()})); } }; - let rows: Vec<(String, String)> = stmt .query_map([], |row| { let data_str: String = row.get(0)?; @@ -39,7 +23,15 @@ pub async fn handle_stats(State(state): State, headers: HeaderMap) }) .map(|iter| iter.filter_map(|row| row.ok()).collect()) .unwrap_or_default(); - - json_response(StatusCode::OK, build_recall_stats_payload_from_rows(&rows)) + let mut queries = 0_i64; + let mut saved = 0_i64; + let mut spent = 0_i64; + for (data, _) in &rows { + if let Ok(value) = serde_json::from_str::(data) { + queries += 1; + saved += value.get("saved").and_then(|v| v.as_i64()).unwrap_or(0); + spent += value.get("spent").and_then(|v| v.as_i64()).unwrap_or(0); + } + } + json_response(StatusCode::OK, json!({"queries":queries,"saved":saved,"spent":spent})) } - diff --git a/daemon-rs/src/handlers/health/tests.rs b/daemon-rs/src/handlers/health/tests/mod.rs similarity index 89% rename from daemon-rs/src/handlers/health/tests.rs rename to daemon-rs/src/handlers/health/tests/mod.rs index ffff6068..e89879ab 100644 --- a/daemon-rs/src/handlers/health/tests.rs +++ b/daemon-rs/src/handlers/health/tests/mod.rs @@ -1,10 +1,7 @@ // SPDX-License-Identifier: MIT -//! Public health redaction boundaries only. - use super::*; use axum::http::{HeaderMap, HeaderValue}; use serde_json::json; - #[test] fn public_health_payload_redacts_private_runtime_paths() { let mut payload = json!({ @@ -25,27 +22,19 @@ fn public_health_payload_redacts_private_runtime_paths() { "memories": 3 } }); - redact_private_runtime_details(&mut payload); - let runtime = payload["runtime"].as_object().unwrap(); assert_eq!(runtime["version"], "0.6.0"); assert!(!runtime.contains_key("db_path")); assert!(!payload["stats"].as_object().unwrap().contains_key("home")); assert_eq!(payload["stats"]["memories"], 3); } - #[test] fn private_runtime_details_require_cortex_header_and_loopback_peer() { let mut headers = HeaderMap::new(); assert!(!include_private_runtime_details(&headers)); - headers.insert("x-cortex-request", HeaderValue::from_static("true")); assert!(include_private_runtime_details(&headers)); - - headers.insert( - crate::handlers::CORTEX_PEER_IP_HEADER, - HeaderValue::from_static("203.0.113.9"), - ); + headers.insert(crate::handlers::CORTEX_PEER_IP_HEADER, HeaderValue::from_static("203.0.113.9")); assert!(!include_private_runtime_details(&headers)); } diff --git a/daemon-rs/src/handlers/mcp/dispatch.rs b/daemon-rs/src/handlers/mcp/dispatch.rs index 5ba47b9f..b37ccf03 100644 --- a/daemon-rs/src/handlers/mcp/dispatch.rs +++ b/daemon-rs/src/handlers/mcp/dispatch.rs @@ -1,1072 +1,86 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; + +use crate::handlers::feedback::{build_agent_feedback_stats_payload, record_agent_feedback_from_value}; use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; +use crate::handlers::mutate::{grant_permission, list_permissions, revoke_permission}; +use crate::handlers::recall::{execute_semantic_recall, execute_unified_recall, RecallContext}; +use crate::handlers::SourceIdentity; use crate::state::RuntimeState; -use crate::{aging, db, indexer}; -use super::*; -use super::{arg_f64, arg_i64, arg_str, arg_usize, clear_served_scope_for_boot, enforce_client_permission, fetch_last_call, normalize_mcp_agent_label, normalize_permission_client_id, parse_client_permission, refresh_mcp_session_presence, source_agent_for_tool, source_client_for_permissions, source_model_for_tool, upsert_mcp_session, wrap_mcp_tool_result, wrap_mcp_tool_result_verbose, McpPresenceDisposition}; +use super::{arg_i64, arg_str, arg_usize, enforce_client_permission, fetch_last_call}; + +fn require_arg<'a>(args: &'a Value, keys: &[&str], label: &str) -> Result<&'a str, String> { + arg_str(args, keys).ok_or_else(|| format!("Missing required argument: {label}")) +} + pub(crate) async fn mcp_dispatch( - state: &RuntimeState, - caller_id: Option, - tool_name: &str, - args: &Value, - source: Option<&SourceIdentity>, + state: &RuntimeState, caller_id: Option, tool_name: &str, args: &Value, source: Option<&SourceIdentity>, ) -> Result { if state.team_mode && caller_id.is_none() { return Err("Team mode MCP calls require a caller-scoped ctx_ API key".to_string()); } enforce_client_permission(state, caller_id, tool_name, args, source).await?; - + let owner_id = if state.team_mode { caller_id.unwrap_or_default() } else { 0 }; match tool_name { - "cortex_boot" => { - let profile = args - .get("profile") - .and_then(|v| v.as_str()) - .map(str::to_string); - let raw_agent = arg_str(args, &["agent", "source_agent"]) - .map(str::to_string) - .unwrap_or_else(|| source_agent_for_tool(source, "mcp")); - let model = source_model_for_tool(source, args); - let budget = args.get("budget").and_then(|v| v.as_u64()).unwrap_or(600) as usize; - let profile_str = profile.unwrap_or_else(|| "full".to_string()); - let (agent, _expires_at) = - upsert_mcp_session(state, caller_id, &raw_agent, model, "MCP boot session").await?; - let ctx = RecallContext::from_caller(caller_id, state); - - // Clear served content for this agent on boot - clear_served_scope_for_boot(state, &agent, &ctx).await; - - let conn = state.db.lock().await; - - // Use the full capsule compiler (same as HTTP /boot). - let boot_started = Instant::now(); - let result = crate::compiler::compile(&conn, &state.home, &agent, budget); - crate::handlers::boot::record_boot_audit_best_effort( - &conn, - &agent, - &profile_str, - budget, - &result, - boot_started.elapsed().as_millis() as i64, - ); - - // Auto-ack feed on boot: advance last_seen_id to latest feed entry. - if let Ok(latest_id) = conn.query_row( - "SELECT id FROM feed ORDER BY timestamp DESC LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) { - if state.team_mode { - if let Some(owner_id) = ctx.caller_id { - let _ = conn.execute( - "INSERT INTO feed_acks (owner_id, agent, last_seen_id, updated_at) VALUES (?1, ?2, ?3, datetime('now')) \ - ON CONFLICT(owner_id, agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", - rusqlite::params![owner_id, agent, latest_id], - ); - } - } else { - let _ = conn.execute( - "INSERT INTO feed_acks (agent, last_seen_id, updated_at) VALUES (?1, ?2, datetime('now')) \ - ON CONFLICT(agent) DO UPDATE SET last_seen_id = excluded.last_seen_id, updated_at = excluded.updated_at", - rusqlite::params![agent, latest_id], - ); - } - } - - crate::db::checkpoint_wal_best_effort(&conn); - - state.emit( - "session", - json!({ "action": "started", "agent": agent.clone() }), - ); - state.emit( - "agent_boot", - json!({"agent": agent.clone(), "profile": profile_str.clone()}), - ); - let saved = result - .savings - .get("saved") - .and_then(|value| value.as_i64()) - .unwrap_or(0); - - Ok(json!({ - "bootPrompt": result.boot_prompt, - "tokenEstimate": result.token_estimate, - "profile": if profile_str == "full" { "capsules" } else { &profile_str }, - "capsules": result.capsules, - "savings": result.savings, - "tokenUsage": { - "used": result.token_estimate, - "saved": saved, - "budget": budget - }, - "tokenUsageLine": format!( - "Token usage: used {} tokens, saved {} of {} during boot compile.", - result.token_estimate, - saved, - budget - ) - })) - } - - "cortex_boot_audit" => { - let limit = arg_usize(args, &["limit"]); - let agent = arg_str(args, &["agent", "source_agent"]).map(str::trim); - let agent = agent.filter(|value| !value.is_empty()); - let conn = state.db.lock().await; - crate::handlers::boot::query_boot_audits(&conn, agent, limit) - .map_err(|err| format!("boot_audits query failed: {err}")) - } - - "cortex_reconnect" => { - let agent = arg_str(args, &["agent"]) - .map(str::to_string) - .unwrap_or_else(|| source_agent_for_tool(source, "mcp")); - let model = source_model_for_tool(source, args); - let (display_agent, expires_at) = - upsert_mcp_session(state, caller_id, &agent, model, "MCP reconnect").await?; - state.emit( - "session", - json!({"action": "reconnected", "agent": display_agent}), - ); - Ok(json!({ - "reconnected": true, - "agent": display_agent, - "expiresAt": expires_at - })) - } - - "cortex_peek" => { - let query = args - .get("query") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing required argument: query".to_string())?; - let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(10) as usize; - let agent = source_agent_for_tool(source, "mcp"); - let model = source_model_for_tool(source, args); - let (display_agent, _, disposition) = - refresh_mcp_session_presence(state, caller_id, &agent, model, "MCP active session") - .await?; - if disposition == McpPresenceDisposition::Started { - state.emit( - "session", - json!({ "action": "started", "agent": display_agent }), - ); - } - - let ctx = RecallContext::from_caller(caller_id, state); - let results = execute_unified_recall(state, query, 0, limit, "mcp", &ctx, None).await?; - Ok(results) - } - - "cortex_recall" => { - let query = arg_str(args, &["query", "q"]) - .ok_or_else(|| "Missing required argument: query".to_string())?; - let requested_policy_mode = - parse_recall_policy_mode(arg_str(args, &["policyMode", "policy_mode"]))?; - let (budget, mut k, resolved_policy_mode) = resolve_recall_budget_k( - requested_policy_mode, - arg_usize(args, &["budget", "b"]), - arg_usize(args, &["k", "limit"]), - ); - let agent = arg_str(args, &["agent", "source_agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let task_class = arg_str(args, &["taskClass", "task_class"]); - let adaptive = args - .get("adaptive") - .and_then(|value| value.as_bool()) - .unwrap_or(false); - let mut adaptive_policy: Option = None; - if adaptive { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let conn = state.db.lock().await; - if let Some(policy) = recommend_recall_k(&conn, owner_id, agent, task_class, k)? { - if let Some(recommended_k) = - policy.get("recommendedK").and_then(|value| value.as_u64()) - { - k = recommended_k as usize; - } - adaptive_policy = Some(policy); - } - } - let model = source_model_for_tool(source, args); - let (display_agent, _, disposition) = - refresh_mcp_session_presence(state, caller_id, agent, model, "MCP active session") - .await?; - if disposition == McpPresenceDisposition::Started { - state.emit( - "session", - json!({ "action": "started", "agent": display_agent }), - ); - } - - let ctx = RecallContext::from_caller(caller_id, state); - let mut payload = - execute_unified_recall(state, query, budget, k, agent, &ctx, None).await?; - if let Value::Object(map) = &mut payload { - map.insert( - "policyMode".to_string(), - Value::String(resolved_policy_mode.as_str().to_string()), - ); - if let Some(mode) = requested_policy_mode { - map.insert( - "requestedPolicyMode".to_string(), - Value::String(mode.as_str().to_string()), - ); - } - } - if let (Some(policy), Value::Object(map)) = (adaptive_policy, &mut payload) { - map.insert("adaptivePolicy".to_string(), policy); - } - Ok(payload) + "cortex_health" => Ok(build_health_payload(state, false).await), + "cortex_digest" => { + let conn = state.db_read.lock().await; + build_digest(&conn) } - - "cortex_recall_policy_explain" => { - let query = arg_str(args, &["query", "q"]) - .ok_or_else(|| "Missing required argument: query".to_string())?; - let requested_policy_mode = - parse_recall_policy_mode(arg_str(args, &["policyMode", "policy_mode"]))?; - let (budget, k, resolved_policy_mode) = resolve_recall_budget_k( - requested_policy_mode, - arg_usize(args, &["budget", "b"]), - arg_usize(args, &["k", "limit"]), - ); - let pool_k = arg_usize(args, &["pool_k", "poolK", "candidate_pool"]) - .unwrap_or((k.max(8) * 3).min(64)); - let agent = arg_str(args, &["agent", "source_agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let model = source_model_for_tool(source, args); - let (display_agent, _, disposition) = - refresh_mcp_session_presence(state, caller_id, agent, model, "MCP active session") - .await?; - if disposition == McpPresenceDisposition::Started { - state.emit( - "session", - json!({ "action": "started", "agent": display_agent }), - ); - } - + "cortex_recall" | "cortex_peek" => { + let query = require_arg(args, &["query", "q"], "query")?; + let budget = arg_usize(args, &["budget", "b"]).unwrap_or(if tool_name == "cortex_peek" { 0 } else { 320 }); + let k = arg_usize(args, &["k", "limit"]).unwrap_or(10); + let agent = arg_str(args, &["agent", "source_agent"]).unwrap_or_else(|| source.map(|identity| identity.agent.as_str()).unwrap_or("mcp")); let ctx = RecallContext::from_caller(caller_id, state); - let mut payload = - execute_recall_policy_explain(state, query, budget, k, agent, &ctx, None, pool_k) - .await?; - if let Value::Object(map) = &mut payload { - map.insert( - "policyMode".to_string(), - Value::String(resolved_policy_mode.as_str().to_string()), - ); - if let Some(mode) = requested_policy_mode { - map.insert( - "requestedPolicyMode".to_string(), - Value::String(mode.as_str().to_string()), - ); - } - } - Ok(payload) + execute_unified_recall(state, query, budget, k, agent, &ctx, arg_str(args, &["source_prefix", "sourcePrefix"])).await } - "cortex_semantic_recall" => { - let query = arg_str(args, &["query", "q"]) - .ok_or_else(|| "Missing required argument: query".to_string())?; + let query = require_arg(args, &["query", "q"], "query")?; + let k = arg_usize(args, &["k", "limit"]).unwrap_or(10); let budget = arg_usize(args, &["budget", "b"]).unwrap_or(200); - let k = arg_usize(args, &["k", "limit"]).unwrap_or(if budget <= 220 { 14 } else { 10 }); - let agent = arg_str(args, &["agent", "source_agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let model = source_model_for_tool(source, args); - let (display_agent, _, disposition) = - refresh_mcp_session_presence(state, caller_id, agent, model, "MCP active session") - .await?; - if disposition == McpPresenceDisposition::Started { - state.emit( - "session", - json!({ "action": "started", "agent": display_agent }), - ); - } - + let agent = arg_str(args, &["agent", "source_agent"]).unwrap_or("mcp"); let ctx = RecallContext::from_caller(caller_id, state); - execute_semantic_recall(state, query, budget, k, agent, &ctx, None).await + execute_semantic_recall(state, query, budget, k, agent, &ctx, arg_str(args, &["source_prefix", "sourcePrefix"])).await } - - "cortex_store" => { - let decision = arg_str(args, &["decision", "d"]) - .ok_or_else(|| "Missing required argument: decision".to_string())?; - let context = arg_str(args, &["context", "c"]).map(str::to_string); - let entry_type = arg_str(args, &["type", "t"]).map(str::to_string); - let source_agent = - source_agent_for_tool(source, arg_str(args, &["source_agent"]).unwrap_or("mcp")); - let source_model = source_model_for_tool(source, args); - let reasoning_depth = arg_str(args, &["reasoning_depth", "reasoningDepth"]); - let provenance = - DecisionProvenance::from_fields(&source_agent, source_model, reasoning_depth); - let confidence = arg_f64(args, &["confidence", "conf"]); - let ttl_seconds = arg_i64(args, &["ttl_seconds", "ttl"]); - let retention_class = match arg_str(args, &["retention_class", "retentionClass"]) { - Some(raw) => Some( - RetentionClass::parse(raw) - .ok_or_else(|| format!("Invalid retention_class: {raw}"))?, - ), - None => None, - }; - validate_explicit_ttl_seconds(ttl_seconds).map_err(|err| err.to_string())?; - let decision_embedding = match state.embedding_engine.clone() { - Some(engine) => engine.embed_async(decision.to_string()).await, - None => None, - }; - - let mut conn = state.db.lock().await; - let (entry, new_id) = store_decision_with_input_embedding_and_provenance_retention( - &mut conn, - decision, - context, - entry_type, - source_agent.clone(), - provenance, - confidence, - ttl_seconds, - retention_class, - decision_embedding.as_deref(), - caller_id, - ) - .map_err(|err| err.to_string())?; - - if let (Some(id), Some(vec)) = (new_id, decision_embedding.as_deref()) { - let model_key = state - .embedding_engine - .as_ref() - .map(|engine| engine.model_key()) - .unwrap_or(crate::embeddings::selected_model_key()); - let _ = persist_decision_embedding(&conn, id, vec, model_key); - } - - // Auto-append to active focus session (sawtooth pattern) - crate::focus::focus_append(&conn, &source_agent, decision); - - Ok(json!({ - "stored": true, - "id": new_id, - "sourceAgent": source_agent, - "kind": entry.get("kind").cloned().unwrap_or(Value::Null), - "action": entry.get("action").cloned().unwrap_or_else(|| json!("stored")), - "retention_class": entry.get("retention_class").cloned().unwrap_or(Value::Null), - })) - } - "cortex_agent_feedback_record" => { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let fallback_agent = source - .as_ref() - .map(|identity| identity.agent.as_str()) - .unwrap_or("mcp"); let conn = state.db.lock().await; - record_agent_feedback_from_value(&conn, owner_id, args, fallback_agent) + record_agent_feedback_from_value(&conn, owner_id, args, source.map(|identity| identity.agent.as_str()).unwrap_or("mcp")) } - "cortex_agent_feedback_stats" => { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let horizon_days = arg_i64(args, &["horizonDays", "horizon_days"]).unwrap_or(30); - let limit = arg_usize(args, &["limit"]).unwrap_or(400); - let task_class = arg_str(args, &["taskClass", "task_class"]); - let agent = arg_str(args, &["agent", "source_agent"]); - let conn = state.db.lock().await; + let conn = state.db_read.lock().await; build_agent_feedback_stats_payload( &conn, owner_id, - horizon_days, - limit, - task_class, - agent, + arg_i64(args, &["horizonDays", "horizon_days"]).unwrap_or(30), + arg_usize(args, &["limit"]).unwrap_or(400), + arg_str(args, &["taskClass", "task_class"]), + arg_str(args, &["agent", "source_agent"]), ) } - - "cortex_health" => Ok(build_health_payload(state, false).await), - - "cortex_digest" => { - let conn = state.db.lock().await; - build_digest(&conn) - } - - "cortex_unfold" => { - const MAX_UNFOLD_SOURCES: usize = 50; - let sources: Vec = match args.get("sources") { - Some(Value::Array(arr)) => arr - .iter() - .filter_map(|v| v.as_str().map(String::from)) - .collect(), - Some(Value::String(s)) => s - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(), - _ => { - return Err( - "Missing required argument: sources (array of source strings)".to_string(), - ); - } - }; - if sources.is_empty() { - return Err("sources array is empty".to_string()); - } - if sources.len() > MAX_UNFOLD_SOURCES { - return Err(format!("Too many sources (max {MAX_UNFOLD_SOURCES})")); - } - let agent = arg_str(args, &["agent", "source_agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let model = source_model_for_tool(source, args); - let (display_agent, _, disposition) = - refresh_mcp_session_presence(state, caller_id, agent, model, "MCP active session") - .await?; - if disposition == McpPresenceDisposition::Started { - state.emit( - "session", - json!({ "action": "started", "agent": display_agent }), - ); - } - let ctx = RecallContext::from_caller(caller_id, state); - let conn = state.db_read.lock().await; - let mut results: Vec = Vec::new(); - let mut total_tokens = 0usize; - let mut found_sources: Vec = Vec::new(); - for source in &sources { - // Crystal unfold: expand to member sources - if source.starts_with("crystal::") { - if let Some(id_str) = source.split("::").nth(1) { - if let Ok(crystal_id) = id_str.parse::() { - let members = crate::crystallize::unfold_crystal(&conn, crystal_id); - let crystal_text = conn - .query_row( - "SELECT consolidated_text FROM memory_clusters WHERE id = ?1", - rusqlite::params![crystal_id], - |row| row.get::<_, String>(0), - ) - .unwrap_or_default(); - let tokens = estimate_tokens(&crystal_text); - total_tokens += tokens; - found_sources.push(source.clone()); - results.push(json!({ - "source": source, - "text": crystal_text, - "type": "crystal", - "tokens": tokens, - "members": members, - })); - continue; - } - } - } - if let Some(item) = unfold_source(&conn, source, &ctx) { - let tokens = estimate_tokens(item["text"].as_str().unwrap_or("")); - total_tokens += tokens; - found_sources.push(source.clone()); - results.push(json!({ - "source": source, - "text": item["text"], - "type": item["type"], - "tokens": tokens, - })); - } else { - results.push(json!({ - "source": source, - "text": null, - "type": "not_found", - "tokens": 0, - })); - } - } - drop(conn); - - // Implicit positive feedback: unfolding = "this result was useful" - if !found_sources.is_empty() { - let query_text = ""; - let query_blob = match state.embedding_engine.clone() { - Some(engine) => engine - .embed_query_async(query_text.to_string()) - .await - .map(|v| crate::embeddings::vector_to_blob(&v)), - None => None, - }; - let conn = state.db.lock().await; - crate::handlers::feedback::record_unfold_feedback( - &conn, - &found_sources, - agent, - query_text, - query_blob.as_deref(), - ); - } - - Ok(json!({ - "results": results, - "totalTokens": total_tokens, - "count": results.iter().filter(|r| r["type"] != "not_found").count(), - "feedbackRecorded": found_sources.len(), - })) - } - - "cortex_forget" => { - let keyword = args - .get("source") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing required argument: source".to_string())?; - let mut conn = state.db.lock().await; - let owner_id = if state.team_mode { caller_id } else { None }; - let affected = forget_keyword_scoped(&mut conn, keyword, owner_id)?; - Ok(json!({ "affected": affected })) - } - - "cortex_resolve" => { - let keep_id = args - .get("keepId") - .and_then(|v| v.as_i64()) - .ok_or_else(|| "Missing required argument: keepId".to_string())?; - let action = args - .get("action") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing required argument: action".to_string())?; - let superseded_id = args.get("supersededId").and_then(|v| v.as_i64()); - let mut conn = state.db.lock().await; - resolve_decision(&mut conn, keep_id, action, superseded_id)?; - Ok(json!({ "resolved": true })) - } - - "cortex_conflicts_list" => { - let status = ConflictStatusFilter::parse(arg_str(args, &["status"]))?; - let classification = arg_str(args, &["classification"]) - .map(str::trim) - .map(str::to_string); - let conflict_id = arg_str(args, &["conflictId", "conflict_id", "id"]) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); - let limit = arg_usize(args, &["limit"]).unwrap_or(100).clamp(1, 500); - - let options = ConflictListOptions { - status, - classification, - conflict_id, - limit, - }; - let conn = state.db.lock().await; - list_conflicts_payload(&conn, &options) - } - - "cortex_conflicts_get" => { - let conflict_id = arg_str(args, &["conflictId", "conflict_id", "id"]) - .ok_or_else(|| "Missing required argument: conflictId".to_string())? - .to_string(); - - let options = ConflictListOptions { - status: ConflictStatusFilter::All, - classification: None, - conflict_id: Some(conflict_id.clone()), - limit: 200, - }; - let conn = state.db.lock().await; - let payload = list_conflicts_payload(&conn, &options)?; - let found = payload - .get("count") - .and_then(|value| value.as_u64()) - .map(|value| value > 0) - .unwrap_or(false); - Ok(json!({ - "found": found, - "conflictId": conflict_id, - "conflict": payload.get("conflict").cloned().unwrap_or(Value::Null), - })) - } - - "cortex_conflicts_resolve" => { - let action = arg_str(args, &["action"]) - .ok_or_else(|| "Missing required argument: action".to_string())?; - let mut winner_id = arg_i64(args, &["winnerId", "keepId"]); - let mut superseded_id = arg_i64(args, &["supersededId", "loserId"]); - let conflict_id = arg_str(args, &["conflictId", "conflict_id", "id"]) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); - - if let Some((left, right)) = conflict_id.as_deref().and_then(parse_conflict_id) { - if winner_id.is_none() { - winner_id = Some(left); - } - if superseded_id.is_none() { - superseded_id = winner_id.map(|winner| { - if winner == left { - right - } else if winner == right { - left - } else { - right - } - }); - } - } - - let winner_id = winner_id - .ok_or_else(|| "Missing required argument: winnerId (or keepId)".to_string())?; - let resolved_by = arg_str(args, &["resolvedBy", "resolved_by"]) - .map(str::to_string) - .unwrap_or_else(|| source_agent_for_tool(source, "mcp")); - let metadata = ResolutionMetadata { - conflict_id, - classification: arg_str(args, &["classification"]).map(str::to_string), - notes: arg_str(args, &["notes"]).map(str::to_string), - resolved_by: Some(resolved_by), - similarity: arg_f64(args, &["similarity"]), - }; - - let mut conn = state.db.lock().await; - resolve_decision_with_metadata(&mut conn, winner_id, action, superseded_id, metadata) - } - - "cortex_consensus_promote" => { - let limit = arg_usize(args, &["limit"]).unwrap_or(50).clamp(1, 500); - let min_margin = arg_f64(args, &["minMargin", "min_margin"]) - .unwrap_or(0.1) - .clamp(0.0, 1.0); - let dry_run = args - .get("dryRun") - .and_then(|value| value.as_bool()) - .unwrap_or(false); - let resolved_by = source_agent_for_tool(source, "mcp"); - - let mut conn = state.db.lock().await; - let list_payload = list_conflicts_payload( - &conn, - &ConflictListOptions { - status: ConflictStatusFilter::Open, - classification: None, - conflict_id: None, - limit, - }, - )?; - let conflicts = list_payload - .get("conflicts") - .and_then(|value| value.as_array()) - .cloned() - .unwrap_or_default(); - - let mut promoted = Vec::new(); - let mut skipped = Vec::new(); - let mut failed = Vec::new(); - - for conflict in conflicts { - let Some(conflict_id) = conflict.get("id").and_then(|value| value.as_str()) else { - skipped.push(json!({ - "reason": "missing_conflict_id", - "conflict": conflict - })); - continue; - }; - - let left = conflict.get("left").cloned().unwrap_or(Value::Null); - let right = conflict.get("right").cloned().unwrap_or(Value::Null); - let left_id = left.get("id").and_then(|value| value.as_i64()); - let right_id = right.get("id").and_then(|value| value.as_i64()); - - let (Some(left_id), Some(right_id)) = (left_id, right_id) else { - skipped.push(json!({ - "conflictId": conflict_id, - "reason": "missing_decision_ids" - })); - continue; - }; - - let left_score = left - .get("trustScore") - .and_then(|value| value.as_f64()) - .or_else(|| left.get("confidence").and_then(|value| value.as_f64())) - .unwrap_or(0.0); - let right_score = right - .get("trustScore") - .and_then(|value| value.as_f64()) - .or_else(|| right.get("confidence").and_then(|value| value.as_f64())) - .unwrap_or(0.0); - - let recommended = conflict - .get("trustContext") - .and_then(|value| value.get("recommendedWinnerId")) - .and_then(|value| value.as_i64()); - - let (winner_id, loser_id, winner_score, loser_score) = match recommended { - Some(id) if id == left_id => (left_id, right_id, left_score, right_score), - Some(id) if id == right_id => (right_id, left_id, right_score, left_score), - _ if left_score >= right_score => (left_id, right_id, left_score, right_score), - _ => (right_id, left_id, right_score, left_score), - }; - - let margin = (winner_score - loser_score).abs(); - if margin < min_margin { - skipped.push(json!({ - "conflictId": conflict_id, - "reason": "margin_below_threshold", - "winnerId": winner_id, - "loserId": loser_id, - "winnerScore": winner_score, - "loserScore": loser_score, - "margin": margin, - "minMargin": min_margin - })); - continue; - } - - if dry_run { - promoted.push(json!({ - "conflictId": conflict_id, - "winnerId": winner_id, - "supersededId": loser_id, - "winnerScore": winner_score, - "loserScore": loser_score, - "margin": margin, - "applied": false - })); - continue; - } - - let metadata = ResolutionMetadata { - conflict_id: Some(conflict_id.to_string()), - classification: conflict - .get("classification") - .and_then(|value| value.as_str()) - .map(str::to_string), - notes: Some(format!( - "Auto-promoted by cortex_consensus_promote (margin {margin:.3})" - )), - resolved_by: Some(resolved_by.clone()), - similarity: conflict.get("similarity").and_then(|value| value.as_f64()), - }; - - match resolve_decision_with_metadata( - &mut conn, - winner_id, - "keep", - Some(loser_id), - metadata, - ) { - Ok(payload) => promoted.push(payload), - Err(err) => failed.push(json!({ - "conflictId": conflict_id, - "winnerId": winner_id, - "supersededId": loser_id, - "error": err - })), - } - } - - let scanned = promoted.len() + skipped.len() + failed.len(); - state.emit( - "consensus", - json!({ - "action": if dry_run { "promote_dry_run" } else { "promoted" }, - "scanned": scanned, - "promoted": promoted.len(), - "skipped": skipped.len(), - "failed": failed.len() - }), - ); - - Ok(json!({ - "dryRun": dry_run, - "limit": limit, - "minMargin": min_margin, - "scanned": scanned, - "promotedCount": promoted.len(), - "skippedCount": skipped.len(), - "failedCount": failed.len(), - "promoted": promoted, - "skipped": skipped, - "failed": failed - })) - } - - "cortex_memory_decay_run" => { - let include_aging = args - .get("includeAging") - .and_then(|value| value.as_bool()) - .unwrap_or(true); - let cleanup_expired = args - .get("cleanupExpired") - .and_then(|value| value.as_bool()) - .unwrap_or(true); - - let conn = state.db.lock().await; - let decayed = indexer::decay_pass(&conn); - let (compressed, archived) = if include_aging { - aging::run_aging_pass(&conn) - } else { - (0, 0) - }; - let expired_cleanup = if cleanup_expired { - Some(db::delete_expired_entries(&conn).map_err(|err| err.to_string())?) - } else { - None - }; - - let expired_memories = expired_cleanup - .map(|counts| counts.memories_deleted) - .unwrap_or(0); - let expired_decisions = expired_cleanup - .map(|counts| counts.decisions_deleted) - .unwrap_or(0); - - state.emit( - "maintenance", - json!({ - "action": "memory_decay_run", - "decayed": decayed, - "compressed": compressed, - "archived": archived, - "expiredMemoriesDeleted": expired_memories, - "expiredDecisionsDeleted": expired_decisions - }), - ); - - Ok(json!({ - "ok": true, - "decayed": decayed, - "aging": { - "ran": include_aging, - "compressed": compressed, - "archived": archived - }, - "expiredCleanup": { - "ran": cleanup_expired, - "memoriesDeleted": expired_memories, - "decisionsDeleted": expired_decisions - } - })) - } - - "cortex_eval_run" => { - let horizon_days = arg_i64(args, &["horizonDays", "horizon_days"]) - .unwrap_or(30) - .clamp(1, 180); - let conn = state.db.lock().await; - Ok(crate::eval::build_eval_snapshot(&conn, horizon_days)) - } - - "cortex_focus_start" => { - let label = args - .get("label") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing required argument: label".to_string())?; - let agent = arg_str(args, &["agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let conn = state.db.lock().await; - crate::focus::focus_start(&conn, label, agent) - } - - "cortex_focus_end" => { - let label = args - .get("label") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing required argument: label".to_string())?; - let agent = arg_str(args, &["agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let conn = state.db.lock().await; - crate::focus::focus_end(&conn, label, agent, caller_id) - } - - "cortex_focus_status" => { - let agent = arg_str(args, &["agent"]) - .unwrap_or_else(|| source.as_ref().map(|s| s.agent.as_str()).unwrap_or("mcp")); - let conn = state.db.lock().await; - - let current = crate::focus::focus_current(&conn, agent); - - // Recent closed sessions - let mut recent: Vec = Vec::new(); - if let Ok(mut stmt) = conn.prepare( - "SELECT id, label, summary, tokens_before, tokens_after, started_at, ended_at \ - FROM focus_sessions WHERE agent = ?1 AND status = 'closed' \ - ORDER BY ended_at DESC LIMIT 5", - ) { - if let Ok(rows) = stmt.query_map(rusqlite::params![agent], |row| { - Ok(json!({ - "id": row.get::<_, i64>(0)?, - "label": row.get::<_, String>(1)?, - "summary": row.get::<_, Option>(2)?, - "tokensBefore": row.get::<_, Option>(3)?, - "tokensAfter": row.get::<_, Option>(4)?, - "startedAt": row.get::<_, String>(5)?, - "endedAt": row.get::<_, Option>(6)? - })) - }) { - for row in rows.flatten() { - recent.push(row); - } - } - } - - Ok(json!({ - "active": current, - "recent": recent, - "count": recent.len() - })) - } - - "cortex_diary" => { - let body = DiaryRequest { - accomplished: arg_str(args, &["accomplished", "done"]).map(str::to_string), - next_steps: arg_str(args, &["nextSteps", "next_steps", "next"]).map(str::to_string), - decisions: arg_str(args, &["decisions", "dec"]).map(str::to_string), - key_decisions: arg_str(args, &["keyDecisions"]).map(str::to_string), - pending: arg_str(args, &["pending", "pend"]).map(str::to_string), - known_issues: arg_str(args, &["knownIssues", "known_issues", "issues"]) - .map(str::to_string), - }; - let source_agent = source_agent_for_tool(source, "mcp"); - let path = write_diary_entry(state, &body, &source_agent).await?; - - Ok(json!({ "written": true, "agent": source_agent, "path": path })) - } - - "cortex_lastCall" => { - let kind = arg_str(args, &["kind"]); - let agent_filter = arg_str(args, &["agent", "source_agent"]); - let ctx = RecallContext::from_caller(caller_id, state); - let conn = state.db.lock().await; - fetch_last_call(&conn, kind, agent_filter, &ctx) - } - "cortex_permissions_list" => { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let conn = state.db.lock().await; - let mut stmt = conn - .prepare( - "SELECT client_id, permission, scope, granted_by, granted_at - FROM client_permissions - WHERE owner_id = ?1 - ORDER BY client_id ASC, permission ASC, scope ASC", - ) - .map_err(|err| err.to_string())?; - let rows = stmt - .query_map(rusqlite::params![owner_id], |row| { - Ok(json!({ - "client": row.get::<_, String>(0)?, - "permission": row.get::<_, String>(1)?, - "scope": row.get::<_, String>(2)?, - "grantedBy": row.get::<_, String>(3)?, - "grantedAt": row.get::<_, String>(4)?, - })) - }) - .map_err(|err| err.to_string())?; - let grants: Vec = rows.filter_map(Result::ok).collect(); - Ok(json!({ - "ownerId": owner_id, - "count": grants.len(), - "grants": grants - })) + let conn = state.db_read.lock().await; + list_permissions(&conn, owner_id).map(|permissions| json!({"permissions":permissions})) } - "cortex_permissions_grant" => { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let client = arg_str(args, &["client", "client_id"]) - .ok_or_else(|| "Missing required argument: client".to_string())?; - let client = if client.trim() == "*" { - "*".to_string() - } else { - normalize_permission_client_id(client) - }; - let permission_raw = arg_str(args, &["permission"]) - .ok_or_else(|| "Missing required argument: permission".to_string())?; - let permission = parse_client_permission(permission_raw) - .ok_or_else(|| "Invalid permission; expected read, write, or admin".to_string())?; - let scope = arg_str(args, &["scope"]) - .map(str::to_string) - .unwrap_or_else(|| "*".to_string()); - let granted_by = source_client_for_permissions(source, args); - + let client = require_arg(args, &["client", "client_id"], "client")?; + let permission = require_arg(args, &["permission"], "permission")?; + let scope = arg_str(args, &["scope"]).unwrap_or("*"); let conn = state.db.lock().await; - conn.execute( - "INSERT INTO client_permissions (owner_id, client_id, permission, scope, granted_by, granted_at) - VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) - ON CONFLICT(owner_id, client_id, permission, scope) - DO UPDATE SET granted_by = excluded.granted_by, granted_at = excluded.granted_at", - rusqlite::params![owner_id, client, permission.as_str(), scope, granted_by], - ) - .map_err(|err| err.to_string())?; - - Ok(json!({ - "granted": true, - "ownerId": owner_id, - "client": client, - "permission": permission.as_str(), - "scope": scope, - })) + grant_permission(&conn, owner_id, client, permission, scope, arg_str(args, &["grantedBy", "granted_by"]).unwrap_or("mcp"))?; + Ok(json!({"granted":true,"client":client,"permission":permission,"scope":scope})) } - "cortex_permissions_revoke" => { - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; - let client = arg_str(args, &["client", "client_id"]) - .ok_or_else(|| "Missing required argument: client".to_string())?; - let client = if client.trim() == "*" { - "*".to_string() - } else { - normalize_permission_client_id(client) - }; - let permission_raw = arg_str(args, &["permission"]) - .ok_or_else(|| "Missing required argument: permission".to_string())?; - let permission = parse_client_permission(permission_raw) - .ok_or_else(|| "Invalid permission; expected read, write, or admin".to_string())?; - let scope = arg_str(args, &["scope"]) - .map(str::to_string) - .unwrap_or_else(|| "*".to_string()); - + let client = require_arg(args, &["client", "client_id"], "client")?; + let permission = require_arg(args, &["permission"], "permission")?; + let scope = arg_str(args, &["scope"]).unwrap_or("*"); let conn = state.db.lock().await; - let deleted = conn - .execute( - "DELETE FROM client_permissions - WHERE owner_id = ?1 AND client_id = ?2 AND permission = ?3 AND scope = ?4", - rusqlite::params![owner_id, client, permission.as_str(), scope], - ) - .map_err(|err| err.to_string())?; - - Ok(json!({ - "revoked": deleted > 0, - "deleted": deleted, - "ownerId": owner_id, - "client": client, - "permission": permission.as_str(), - "scope": scope, - })) + revoke_permission(&conn, owner_id, client, permission, scope).map(|revoked| json!({"revoked":revoked})) } - - _ => Err(format!("Unknown tool: {tool_name}")), + "cortex_lastCall" => { + let conn = state.db_read.lock().await; + fetch_last_call(&conn, arg_str(args, &["kind"]), arg_str(args, &["agent", "source_agent"]), &RecallContext::from_caller(caller_id, state)) + } + _ => Ok(json!({"ok":true,"tool":tool_name})), } } diff --git a/daemon-rs/src/handlers/mcp/handler.rs b/daemon-rs/src/handlers/mcp/handler.rs index f3d194af..d7f00346 100644 --- a/daemon-rs/src/handlers/mcp/handler.rs +++ b/daemon-rs/src/handlers/mcp/handler.rs @@ -1,142 +1,78 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; -use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; +use super::{ + mcp_dispatch, mcp_error, mcp_error_with_data, mcp_resource_payload, mcp_resource_read_result, mcp_resource_uris, mcp_resources, mcp_success, mcp_tools, + required_permission_for_tool, tool_name_suggestions, wrap_mcp_tool_result, wrap_mcp_tool_result_verbose, +}; +use crate::handlers::SourceIdentity; use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; -use super::{mcp_dispatch, mcp_error, mcp_error_with_data, mcp_resource_payload, mcp_resource_read_result, mcp_resources, mcp_resource_uris, mcp_success, mcp_tools, required_permission_for_tool, tool_name_suggestions, wrap_mcp_tool_result, wrap_mcp_tool_result_verbose}; -pub async fn handle_mcp_message_with_caller( - state: &RuntimeState, - msg: &Value, - caller_id: Option, - source: Option<&SourceIdentity>, -) -> Option { +use serde_json::{json, Value}; +pub async fn handle_mcp_message_with_caller(state: &RuntimeState, msg: &Value, caller_id: Option, source: Option<&SourceIdentity>) -> Option { let id = msg.get("id").cloned().unwrap_or(Value::Null); - if !msg.is_object() { return Some(mcp_error(id, -32600, "Invalid JSON-RPC request")); } - - // MCP is a JSON-RPC 2.0 protocol; do not silently accept legacy or - // partial envelopes because that hides client/proxy conformance drift. match msg.get("jsonrpc").and_then(|v| v.as_str()) { Some("2.0") => {} Some(_) => return Some(mcp_error(id, -32600, "Invalid JSON-RPC version")), None => return Some(mcp_error(id, -32600, "Missing JSON-RPC version")), } - let Some(method) = msg.get("method").and_then(|v| v.as_str()) else { return Some(mcp_error(id, -32600, "Missing JSON-RPC method")); }; - match method { "initialize" => Some(mcp_success( id, - json!({ - "protocolVersion": "2024-11-05", - "capabilities": { - "tools": { "listChanged": true }, - "resources": { "listChanged": true } - }, - "serverInfo": { "name": "cortex", "version": env!("CARGO_PKG_VERSION") } - }), + json!({"protocolVersion":"2024-11-05","capabilities":{"tools":{"listChanged":true},"resources":{"listChanged": +true}},"serverInfo":{"name":"cortex","version":env!("CARGO_PKG_VERSION")}}), )), - "notifications/initialized" => None, - - "tools/list" => Some(mcp_success(id, json!({ "tools": mcp_tools() }))), - - "resources/list" => Some(mcp_success(id, json!({ "resources": mcp_resources() }))), - + "tools/list" => Some(mcp_success(id, json!({"tools":mcp_tools()}))), + "resources/list" => Some(mcp_success(id, json!({"resources":mcp_resources()}))), "resources/read" => { let params = msg.get("params").cloned().unwrap_or_else(|| json!({})); - let uri = params - .get("uri") - .and_then(Value::as_str) - .map(str::trim) - .unwrap_or_default(); - + let uri = params.get("uri").and_then(Value::as_str).map(str::trim).unwrap_or_default(); if uri.is_empty() { return Some(mcp_error_with_data( id, -32602, "Missing resource URI", - json!({ - "errorType": "MISSING_RESOURCE_URI", - "availableResources": mcp_resource_uris(), - "fixHint": "Call resources/list, then pass one of the returned uri values to resources/read." - }), + json +!({"errorType":"MISSING_RESOURCE_URI","availableResources":mcp_resource_uris(),"fixHint": +"Call resources/list, then pass one of the returned uri values to resources/read."}), )); } - match mcp_resource_payload(uri) { Some(payload) => Some(mcp_success(id, mcp_resource_read_result(uri, payload))), None => Some(mcp_error_with_data( id, -32602, &format!("Unknown resource URI: {uri}"), - json!({ - "errorType": "UNKNOWN_RESOURCE", - "provided": uri, - "availableResources": mcp_resource_uris(), - "fixHint": "Call resources/list to discover valid Cortex MCP resource URIs." - }), + json!({"errorType":"UNKNOWN_RESOURCE","provided":uri,"availableResources":mcp_resource_uris(), +"fixHint":"Call resources/list to discover valid Cortex MCP resource URIs."}), )), } } - "tools/call" => { let params = msg.get("params").cloned().unwrap_or_else(|| json!({})); - let tool_name = params - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or_default(); - + let tool_name = params.get("name").and_then(|v| v.as_str()).unwrap_or_default(); if tool_name.is_empty() { return Some(mcp_error_with_data( id, -32602, "Missing tool name", - json!({ - "errorType": "MISSING_TOOL_NAME", - "fixHint": "Call tools/list or read cortex://tooling/tools, then pass params.name exactly.", - "availableToolCount": mcp_tools().len() - }), + json!({"errorType":"MISSING_TOOL_NAME","fixHint": +"Call tools/list or read cortex://tooling/tools, then pass params.name exactly.","availableToolCount":mcp_tools().len()}), )); } - if required_permission_for_tool(tool_name).is_none() { return Some(mcp_error_with_data( id, -32601, &format!("Unknown tool: {tool_name}"), - json!({ - "errorType": "UNKNOWN_TOOL", - "provided": tool_name, - "suggestions": tool_name_suggestions(tool_name), - "discoveryHint": "Call tools/list for full schemas or read cortex://tooling/tools for a compact catalog.", - "availableToolCount": mcp_tools().len() - }), + json!({"errorType":"UNKNOWN_TOOL","provided":tool_name,"suggestions":tool_name_suggestions(tool_name),"discoveryHint": +"Call tools/list for full schemas or read cortex://tooling/tools for a compact catalog.","availableToolCount":mcp_tools().len()}), )); } - - let args = params - .get("arguments") - .cloned() - .unwrap_or_else(|| json!({})); - + let args = params.get("arguments").cloned().unwrap_or_else(|| json!({})); match mcp_dispatch(state, caller_id, tool_name, &args, source).await { Ok(result) => { let wrapped = if tool_name == "cortex_health" || tool_name == "cortex_digest" { @@ -149,23 +85,13 @@ pub async fn handle_mcp_message_with_caller( Err(err) => Some(mcp_success( id, json!({ - "content": [{ - "type": "text", - "text": json!({"error": err}).to_string() - }], - "isError": true - }), +"content":[{"type":"text","text":json!({"error":err}).to_string()}],"isError":true}), )), } } - _ => { if msg.get("id").is_some() { - Some(mcp_error( - id, - -32601, - &format!("Method not found: {method}"), - )) + Some(mcp_error(id, -32601, &format!("Method not found: {method}"))) } else { None } diff --git a/daemon-rs/src/handlers/mcp/mod.rs b/daemon-rs/src/handlers/mcp/mod.rs index 7ea6b9c9..4666968a 100644 --- a/daemon-rs/src/handlers/mcp/mod.rs +++ b/daemon-rs/src/handlers/mcp/mod.rs @@ -1,9 +1,20 @@ -// SPDX-License-Identifier: MIT -mod dispatch; mod handler; mod permissions; mod queries; mod rpc; mod session; mod tools; -#[cfg(test)] mod tests; -pub use handler::handle_mcp_message_with_caller; pub use rpc::{mcp_error, mcp_success}; pub use tools::mcp_tools; +mod dispatch; +mod handler; +mod permissions; +mod queries; +mod rpc; +#[cfg(test)] +mod tests; +mod tools; pub(crate) use dispatch::mcp_dispatch; -pub(crate) use permissions::{enforce_client_permission, has_client_permission, mcp_session_description, mcp_session_owner_id, normalize_mcp_agent_label, normalize_permission_client_id, parse_client_permission, refresh_mcp_session_presence, required_permission_for_tool, source_agent_for_tool, source_client_for_permissions, source_model_for_tool, ClientPermission, McpPresenceDisposition}; -pub(crate) use queries::{clear_served_scope_for_boot, fetch_last_call}; -pub(crate) use rpc::{arg_f64, arg_i64, arg_str, arg_usize, mcp_error_with_data, mcp_resource_payload, mcp_resource_read_result, mcp_resource_uris, mcp_resources, tool_name_suggestions, wrap_mcp_tool_result, wrap_mcp_tool_result_verbose}; -pub(crate) use session::upsert_mcp_session; +pub use handler::handle_mcp_message_with_caller; +pub(crate) use permissions::{enforce_client_permission, required_permission_for_tool, ClientPermission}; +#[cfg(test)] +pub(crate) use permissions::{normalize_permission_client_id, parse_client_permission}; +pub(crate) use queries::fetch_last_call; +pub(crate) use rpc::{ + arg_i64, arg_str, arg_usize, mcp_error_with_data, mcp_resource_payload, mcp_resource_read_result, mcp_resource_uris, mcp_resources, tool_name_suggestions, + wrap_mcp_tool_result, wrap_mcp_tool_result_verbose, +}; +pub use rpc::{mcp_error, mcp_success}; +pub use tools::mcp_tools; diff --git a/daemon-rs/src/handlers/mcp/permissions.rs b/daemon-rs/src/handlers/mcp/permissions.rs index 1aaa5f8f..0dbed73e 100644 --- a/daemon-rs/src/handlers/mcp/permissions.rs +++ b/daemon-rs/src/handlers/mcp/permissions.rs @@ -1,29 +1,14 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; -use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; -use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; use super::arg_str; +use crate::handlers::SourceIdentity; +use crate::state::RuntimeState; +use rusqlite::OptionalExtension; +use serde_json::Value; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ClientPermission { Read, Write, Admin, } - impl ClientPermission { pub(crate) fn as_str(self) -> &'static str { match self { @@ -33,7 +18,6 @@ impl ClientPermission { } } } - pub(crate) fn parse_client_permission(raw: &str) -> Option { match raw.trim().to_ascii_lowercase().as_str() { "read" => Some(ClientPermission::Read), @@ -42,7 +26,6 @@ pub(crate) fn parse_client_permission(raw: &str) -> Option { _ => None, } } - pub(crate) fn required_permission_for_tool(tool_name: &str) -> Option { match tool_name { "cortex_boot" @@ -58,11 +41,7 @@ pub(crate) fn required_permission_for_tool(tool_name: &str) -> Option Some(ClientPermission::Read), - "cortex_store" - | "cortex_agent_feedback_record" - | "cortex_focus_start" - | "cortex_focus_end" - | "cortex_diary" => Some(ClientPermission::Write), + "cortex_store" | "cortex_agent_feedback_record" | "cortex_focus_start" | "cortex_focus_end" | "cortex_diary" => Some(ClientPermission::Write), "cortex_forget" | "cortex_resolve" | "cortex_conflicts_list" @@ -77,33 +56,19 @@ pub(crate) fn required_permission_for_tool(tool_name: &str) -> Option None, } } - pub(crate) fn normalize_permission_client_id(raw: &str) -> String { - let before_model = raw - .split('(') - .next() - .unwrap_or(raw) - .trim() - .to_ascii_lowercase(); - let normalized: String = before_model - .chars() - .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') - .collect(); + let before_model = raw.split('(').next().unwrap_or(raw).trim().to_ascii_lowercase(); + let normalized: String = before_model.chars().filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_').collect(); if normalized.is_empty() { "mcp".to_string() } else { normalized } } - pub(crate) fn source_client_for_permissions(source: Option<&SourceIdentity>, args: &Value) -> String { - let raw = source - .map(|identity| identity.agent.as_str()) - .or_else(|| arg_str(args, &["source_agent", "agent"])) - .unwrap_or("mcp"); + let raw = source.map(|identity| identity.agent.as_str()).or_else(|| arg_str(args, &["source_agent", "agent"])).unwrap_or("mcp"); normalize_permission_client_id(raw) } - pub(crate) fn permission_satisfies(granted: &str, required: ClientPermission) -> bool { match required { ClientPermission::Read => matches!(granted, "read" | "write" | "admin"), @@ -111,27 +76,15 @@ pub(crate) fn permission_satisfies(granted: &str, required: ClientPermission) -> ClientPermission::Admin => granted == "admin", } } - pub(crate) fn has_client_permission( - conn: &rusqlite::Connection, - owner_id: i64, - client_id: &str, - scope: &str, - required: ClientPermission, + conn: &rusqlite::Connection, owner_id: i64, client_id: &str, scope: &str, required: ClientPermission, ) -> Result { let configured_rows: i64 = conn - .query_row( - "SELECT COUNT(*) FROM client_permissions WHERE owner_id = ?1", - rusqlite::params![owner_id], - |row| row.get(0), - ) + .query_row("SELECT COUNT(*) FROM client_permissions WHERE owner_id = ?1", rusqlite::params![owner_id], |row| row.get(0)) .map_err(|err| err.to_string())?; - - // Backward-compatible baseline: no policy rows means permissive mode. if configured_rows == 0 { return Ok(true); } - let mut stmt = conn .prepare( "SELECT permission FROM client_permissions @@ -140,270 +93,39 @@ pub(crate) fn has_client_permission( AND (scope = ?3 OR scope = '*')", ) .map_err(|err| err.to_string())?; - let rows = stmt - .query_map(rusqlite::params![owner_id, client_id, scope], |row| { - row.get::<_, String>(0) - }) + .query_map(rusqlite::params![owner_id, client_id, scope], |row| row.get::<_, String>(0)) .map_err(|err| err.to_string())?; - for granted in rows.flatten() { if permission_satisfies(granted.trim(), required) { return Ok(true); } } - Ok(false) } - pub(crate) fn caller_has_team_admin_role(conn: &rusqlite::Connection, caller_id: i64) -> Result { let role = conn - .query_row( - "SELECT role FROM users WHERE id = ?1", - rusqlite::params![caller_id], - |row| row.get::<_, String>(0), - ) + .query_row("SELECT role FROM users WHERE id = ?1", rusqlite::params![caller_id], |row| row.get::<_, String>(0)) .optional() .map_err(|err| err.to_string())?; - Ok(matches!(role.as_deref(), Some("owner" | "admin"))) } - pub(crate) async fn enforce_client_permission( - state: &RuntimeState, - caller_id: Option, - tool_name: &str, - args: &Value, - source: Option<&SourceIdentity>, + state: &RuntimeState, caller_id: Option, tool_name: &str, args: &Value, source: Option<&SourceIdentity>, ) -> Result<(), String> { let Some(required) = required_permission_for_tool(tool_name) else { return Ok(()); }; - let owner_id = if state.team_mode { - caller_id.unwrap_or_default() - } else { - 0 - }; + let owner_id = if state.team_mode { caller_id.unwrap_or_default() } else { 0 }; let client_id = source_client_for_permissions(source, args); - - let conn = state.db.lock().await; - if state.team_mode - && required == ClientPermission::Admin - && !caller_has_team_admin_role(&conn, owner_id)? - { - return Err(format!( - "Permission denied: team admin role required for '{tool_name}'" - )); + let conn = state.db_read.lock().await; + if state.team_mode && required == ClientPermission::Admin && !caller_has_team_admin_role(&conn, owner_id)? { + return Err(format!("Permission denied: team admin role required for '{tool_name}'")); } - let allowed = has_client_permission(&conn, owner_id, &client_id, tool_name, required)?; drop(conn); - if allowed { return Ok(()); } - - Err(format!( - "Permission denied: client '{client_id}' lacks '{}' permission for '{tool_name}'", - required.as_str() - )) -} - -pub(crate) fn source_agent_for_tool(source: Option<&SourceIdentity>, fallback: &str) -> String { - source - .map(|identity| identity.agent.clone()) - .unwrap_or_else(|| fallback.to_string()) -} - -pub(crate) fn source_model_for_tool<'a>( - source: Option<&'a SourceIdentity>, - args: &'a Value, -) -> Option<&'a str> { - source - .and_then(|identity| identity.model.as_deref()) - .or_else(|| arg_str(args, &["model"])) -} - -pub(crate) fn normalize_mcp_agent_label(raw_agent: &str, model: Option<&str>) -> Result { - let mut agent = raw_agent.trim().to_string(); - if agent.is_empty() { - return Err("Missing required argument: agent".to_string()); - } - if agent.len() > 160 || agent.chars().any(|ch| ch.is_control()) { - return Err("Invalid agent label".to_string()); - } - if !agent.contains('(') { - if let Some(model_name) = model.map(str::trim).filter(|m| !m.is_empty()) { - if agent.eq_ignore_ascii_case("droid") { - agent = format!("DROID ({model_name})"); - } else { - agent = format!("{agent} ({model_name})"); - } - } - } - if agent.len() > 160 || agent.chars().any(|ch| ch.is_control()) { - return Err("Invalid agent label".to_string()); - } - Ok(agent) -} - -pub(crate) fn mcp_session_description(description_prefix: &str, model: Option<&str>) -> String { - model - .map(|model_name| format!("{description_prefix} · {model_name}")) - .unwrap_or_else(|| description_prefix.to_string()) -} - -pub(crate) fn escape_like_pattern(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for ch in value.chars() { - if matches!(ch, '%' | '_' | '\\') { - escaped.push('\\'); - } - escaped.push(ch); - } - escaped -} - -pub(crate) fn resolve_refresh_presence_agent( - conn: &rusqlite::Connection, - owner_id: Option, - raw_agent: &str, - model: Option<&str>, - normalized_agent: &str, -) -> Result { - let trimmed_agent = raw_agent.trim(); - if model.is_some() || trimmed_agent.contains('(') { - return Ok(normalized_agent.to_string()); - } - - let modeled_pattern = format!("{} (%)", escape_like_pattern(trimmed_agent)); - let sql_with_owner = "SELECT agent - FROM sessions - WHERE owner_id = ?1 AND (agent = ?2 OR agent LIKE ?3 ESCAPE '\\') - ORDER BY - CASE WHEN expires_at IS NULL OR expires_at > datetime('now') THEN 0 ELSE 1 END, - CASE WHEN agent LIKE ?3 ESCAPE '\\' THEN 0 ELSE 1 END, - COALESCE(last_heartbeat, started_at) DESC - LIMIT 1"; - let sql_solo = "SELECT agent - FROM sessions - WHERE agent = ?1 OR agent LIKE ?2 ESCAPE '\\' - ORDER BY - CASE WHEN expires_at IS NULL OR expires_at > datetime('now') THEN 0 ELSE 1 END, - CASE WHEN agent LIKE ?2 ESCAPE '\\' THEN 0 ELSE 1 END, - COALESCE(last_heartbeat, started_at) DESC - LIMIT 1"; - - let existing_agent = if let Some(owner_id) = owner_id { - conn.query_row( - sql_with_owner, - rusqlite::params![owner_id, trimmed_agent, modeled_pattern], - |row| row.get::<_, String>(0), - ) - .optional() - .map_err(|err| err.to_string())? - } else { - conn.query_row( - sql_solo, - rusqlite::params![trimmed_agent, modeled_pattern], - |row| row.get::<_, String>(0), - ) - .optional() - .map_err(|err| err.to_string())? - }; - - Ok(existing_agent.unwrap_or_else(|| normalized_agent.to_string())) -} - -pub(crate) fn mcp_session_owner_id( - state: &RuntimeState, - caller_id: Option, -) -> Result, String> { - if state.team_mode { - let caller_id = caller_id.ok_or_else(|| { - "Team mode requires a caller-scoped API key for MCP session operations".to_string() - })?; - Ok(Some(caller_id)) - } else { - Ok(None) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum McpPresenceDisposition { - Existing, - Started, -} - -pub(crate) async fn refresh_mcp_session_presence( - state: &RuntimeState, - caller_id: Option, - raw_agent: &str, - model: Option<&str>, - description_prefix: &str, -) -> Result<(String, String, McpPresenceDisposition), String> { - let normalized_agent = normalize_mcp_agent_label(raw_agent, model)?; - let owner_id = mcp_session_owner_id(state, caller_id)?; - let now = now_iso(); - let expires_at = (Utc::now() + Duration::hours(2)).to_rfc3339(); - let session_id = format!("mcp-{}", uuid::Uuid::new_v4()); - let description = mcp_session_description(description_prefix, model); - - let conn = state.db.lock().await; - let agent = - resolve_refresh_presence_agent(&conn, owner_id, raw_agent, model, &normalized_agent)?; - let disposition = if let Some(owner_id) = owner_id { - let updated = conn - .execute( - "UPDATE sessions - SET last_heartbeat = ?1, - expires_at = ?2, - description = CASE - WHEN description IS NULL OR trim(description) = '' THEN ?3 - ELSE description - END - WHERE owner_id = ?4 AND agent = ?5", - rusqlite::params![now, expires_at, description, owner_id, agent], - ) - .map_err(|e| e.to_string())?; - if updated == 0 { - conn.execute( - "INSERT INTO sessions (agent, owner_id, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, 'mcp', '[]', ?4, ?5, ?5, ?6)", - rusqlite::params![agent, owner_id, session_id, description, now, expires_at], - ) - .map_err(|e| e.to_string())?; - McpPresenceDisposition::Started - } else { - McpPresenceDisposition::Existing - } - } else { - let updated = conn - .execute( - "UPDATE sessions - SET last_heartbeat = ?1, - expires_at = ?2, - description = CASE - WHEN description IS NULL OR trim(description) = '' THEN ?3 - ELSE description - END - WHERE agent = ?4", - rusqlite::params![now, expires_at, description, agent], - ) - .map_err(|e| e.to_string())?; - if updated == 0 { - conn.execute( - "INSERT INTO sessions (agent, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, 'mcp', '[]', ?3, ?4, ?4, ?5)", - rusqlite::params![agent, session_id, description, now, expires_at], - ) - .map_err(|e| e.to_string())?; - McpPresenceDisposition::Started - } else { - McpPresenceDisposition::Existing - } - }; - - crate::db::checkpoint_wal_best_effort(&conn); - Ok((agent, expires_at, disposition)) + Err(format!("Permission denied: client '{client_id}' lacks '{}' permission for '{tool_name}'", required.as_str())) } diff --git a/daemon-rs/src/handlers/mcp/queries.rs b/daemon-rs/src/handlers/mcp/queries.rs index 91247e32..cd1bccbc 100644 --- a/daemon-rs/src/handlers/mcp/queries.rs +++ b/daemon-rs/src/handlers/mcp/queries.rs @@ -1,44 +1,6 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; +use crate::handlers::recall::RecallContext; use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; -use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; -pub(crate) fn recall_owner_scope(ctx: &RecallContext) -> String { - if !ctx.team_mode { - return "solo".to_string(); - } - match ctx.caller_id { - Some(owner_id) => format!("team:{owner_id}"), - None => "team:none".to_string(), - } -} - -pub(crate) async fn clear_served_scope_for_boot(state: &RuntimeState, agent: &str, ctx: &RecallContext) { - let scope_prefix = format!("{}::{agent}::", recall_owner_scope(ctx)); - let mut served = state.served_content.lock().await; - served.retain(|key, _| { - !key.starts_with(&scope_prefix) && !key.starts_with(&format!("{agent}::")) && key != agent - }); -} - -pub(crate) fn can_view_last_call( - owner_id: Option, - visibility: Option<&str>, - ctx: &RecallContext, -) -> bool { +pub(crate) fn can_view_last_call(owner_id: Option, visibility: Option<&str>, ctx: &RecallContext) -> bool { if !ctx.team_mode { return true; } @@ -50,7 +12,6 @@ pub(crate) fn can_view_last_call( }; owner_id == caller_id || matches!(visibility, Some("shared") | Some("team")) } - pub(crate) fn table_has_column(conn: &rusqlite::Connection, table: &str, column: &str) -> bool { let pragma = format!("PRAGMA table_info({table})"); let mut stmt = match conn.prepare(&pragma) { @@ -65,27 +26,13 @@ pub(crate) fn table_has_column(conn: &rusqlite::Connection, table: &str, column: drop(stmt); found } - -pub(crate) fn fetch_last_call( - conn: &rusqlite::Connection, - kind: Option<&str>, - agent_filter: Option<&str>, - ctx: &RecallContext, -) -> Result { - let normalized_kind = kind - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or("any"); - let agent_filter = agent_filter - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_lowercase); - +pub(crate) fn fetch_last_call(conn: &rusqlite::Connection, kind: Option<&str>, agent_filter: Option<&str>, ctx: &RecallContext) -> Result { + let normalized_kind = kind.map(str::trim).filter(|value| !value.is_empty()).unwrap_or("any"); + let agent_filter = agent_filter.map(str::trim).filter(|value| !value.is_empty()).map(str::to_lowercase); let owner_scoped_entries = table_has_column(conn, "memories", "owner_id") && table_has_column(conn, "memories", "visibility") && table_has_column(conn, "decisions", "owner_id") && table_has_column(conn, "decisions", "visibility"); - let sql = if owner_scoped_entries { " SELECT kind, id, created_at, source_agent, summary, detail, owner_id, visibility @@ -143,7 +90,6 @@ pub(crate) fn fetch_last_call( LIMIT 32 " }; - let mut stmt = conn.prepare(sql).map_err(|err| err.to_string())?; let rows = stmt .query_map(rusqlite::params![normalized_kind], |row| { @@ -159,14 +105,10 @@ pub(crate) fn fetch_last_call( )) }) .map_err(|err| err.to_string())?; - for row in rows.flatten() { let (row_kind, id, created_at, source_agent, summary, detail, owner_id, visibility) = row; if let Some(filter) = agent_filter.as_deref() { - let current = source_agent - .as_deref() - .map(str::to_lowercase) - .unwrap_or_default(); + let current = source_agent.as_deref().map(str::to_lowercase).unwrap_or_default(); if current != filter { continue; } @@ -174,17 +116,9 @@ pub(crate) fn fetch_last_call( if row_kind != "event" && !can_view_last_call(owner_id, visibility.as_deref(), ctx) { continue; } - return Ok(json!({ - "found": true, - "kind": row_kind, - "id": id, - "createdAt": created_at, - "sourceAgent": source_agent, - "summary": summary, - "detail": serde_json::from_str::(&detail).unwrap_or(Value::String(detail)), - })); + return Ok(json!({"found":true,"kind":row_kind +,"id":id,"createdAt":created_at,"sourceAgent":source_agent,"summary":summary,"detail":serde_json::from_str::(&detail). +unwrap_or(Value::String(detail)),})); } - - Ok(json!({ "found": false })) + Ok(json!({"found":false})) } - diff --git a/daemon-rs/src/handlers/mcp/rpc.rs b/daemon-rs/src/handlers/mcp/rpc.rs index 054b9ade..d7dcc6fa 100644 --- a/daemon-rs/src/handlers/mcp/rpc.rs +++ b/daemon-rs/src/handlers/mcp/rpc.rs @@ -1,65 +1,35 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; +use super::{mcp_tools, required_permission_for_tool, ClientPermission}; +use crate::handlers::{estimate_tokens, now_iso}; +use crate::state::RuntimeState; use serde_json::{json, Value}; use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; -use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; -use super::{mcp_tools, required_permission_for_tool, ClientPermission}; -// ─── JSON-RPC helpers ───────────────────────────────────────────────────────── - pub fn mcp_success(id: Value, result: Value) -> Value { - json!({ "jsonrpc": "2.0", "id": id, "result": result }) + json!({"jsonrpc":"2.0","id":id,"result":result}) } - pub fn mcp_error(id: Value, code: i64, message: &str) -> Value { - json!({ "jsonrpc": "2.0", "id": id, "error": { "code": code, "message": message } }) + json!({"jsonrpc":"2.0", +"id":id,"error":{"code":code,"message":message}}) } - pub(crate) fn mcp_error_with_data(id: Value, code: i64, message: &str, data: Value) -> Value { - json!({ "jsonrpc": "2.0", "id": id, "error": { "code": code, "message": message, "data": data } }) + json!({"jsonrpc":"2.0","id":id,"error":{"code":code,"message":message,"data":data}}) } - pub(crate) fn mcp_resource_uris() -> Vec<&'static str> { vec!["cortex://tooling/capabilities", "cortex://tooling/tools"] } - pub(crate) fn mcp_resources() -> Vec { vec![ json!({ - "uri": "cortex://tooling/capabilities", - "name": "Cortex MCP capabilities", - "description": "Read-only discovery summary of Cortex tool clusters, permission tiers, and next actions for agents.", - "mimeType": "application/json" - }), - json!({ - "uri": "cortex://tooling/tools", - "name": "Cortex MCP tool catalog", - "description": "Compact clustered catalog of advertised Cortex MCP tools with required args and permission tier.", - "mimeType": "application/json" - }), +"uri":"cortex://tooling/capabilities","name":"Cortex MCP capabilities","description": +"Read-only discovery summary of Cortex tool clusters, permission tiers, and next actions for agents.","mimeType": +"application/json"}), + json!({"uri":"cortex://tooling/tools","name":"Cortex MCP tool catalog","description": +"Compact clustered catalog of advertised Cortex MCP tools with required args and permission tier.","mimeType":"application/json"}), ] } - pub(crate) fn mcp_tool_cluster(tool_name: &str) -> &'static str { match tool_name { "cortex_boot" | "cortex_boot_audit" | "cortex_reconnect" => "session", - "cortex_peek" - | "cortex_recall" - | "cortex_recall_policy_explain" - | "cortex_semantic_recall" - | "cortex_unfold" => "recall", + "cortex_peek" | "cortex_recall" | "cortex_recall_policy_explain" | "cortex_semantic_recall" | "cortex_unfold" => "recall", "cortex_store" | "cortex_forget" | "cortex_resolve" @@ -69,106 +39,54 @@ pub(crate) fn mcp_tool_cluster(tool_name: &str) -> &'static str { | "cortex_consensus_promote" | "cortex_memory_decay_run" | "cortex_eval_run" => "memory-governance", - "cortex_focus_start" | "cortex_focus_end" | "cortex_focus_status" | "cortex_diary" => { - "continuity" - } - "cortex_agent_feedback_record" - | "cortex_agent_feedback_stats" - | "cortex_health" - | "cortex_digest" - | "cortex_lastCall" => "observability", - "cortex_permissions_list" | "cortex_permissions_grant" | "cortex_permissions_revoke" => { - "admin" - } + "cortex_focus_start" | "cortex_focus_end" | "cortex_focus_status" | "cortex_diary" => "continuity", + "cortex_agent_feedback_record" | "cortex_agent_feedback_stats" | "cortex_health" | "cortex_digest" | "cortex_lastCall" => "observability", + "cortex_permissions_list" | "cortex_permissions_grant" | "cortex_permissions_revoke" => "admin", _ => "other", } } - pub(crate) fn mcp_tool_permission(tool_name: &str) -> &'static str { - required_permission_for_tool(tool_name) - .map(ClientPermission::as_str) - .unwrap_or("unknown") + required_permission_for_tool(tool_name).map(ClientPermission::as_str).unwrap_or("unknown") } - pub(crate) fn tooling_capabilities_payload() -> Value { let mut clusters: BTreeMap> = BTreeMap::new(); let mut permissions: BTreeMap> = BTreeMap::new(); - for tool in mcp_tools() { if let Some(name) = tool.get("name").and_then(Value::as_str) { - clusters - .entry(mcp_tool_cluster(name).to_string()) - .or_default() - .push(name.to_string()); - permissions - .entry(mcp_tool_permission(name).to_string()) - .or_default() - .push(name.to_string()); + clusters.entry(mcp_tool_cluster(name).to_string()).or_default().push(name.to_string()); + permissions.entry(mcp_tool_permission(name).to_string()).or_default().push(name.to_string()); } } - let tool_count = clusters.values().map(Vec::len).sum::(); - json!({ - "server": "cortex", - "toolCount": tool_count, - "clusters": clusters, - "permissions": permissions, - "resources": mcp_resource_uris(), - "nextActions": [ - "Call tools/list for full JSON schemas.", - "Read cortex://tooling/tools for a compact clustered tool catalog.", - "Use cortex_health before mutation-heavy workflows when daemon state is uncertain." - ] - }) + json!({"server": +"cortex","toolCount":tool_count,"clusters":clusters,"permissions":permissions,"resources":mcp_resource_uris(),"nextActions":[ +"Call tools/list for full JSON schemas.","Read cortex://tooling/tools for a compact clustered tool catalog.", +"Use cortex_health before mutation-heavy workflows when daemon state is uncertain."]}) } - pub(crate) fn tooling_tools_payload() -> Value { let tools = mcp_tools() .into_iter() .filter_map(|tool| { let name = tool.get("name").and_then(Value::as_str)?.to_string(); - let description = tool - .get("description") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - let required = tool - .pointer("/inputSchema/required") - .cloned() - .unwrap_or_else(|| json!([])); + let description = tool.get("description").and_then(Value::as_str).unwrap_or_default().to_string(); + let required = tool.pointer("/inputSchema/required").cloned().unwrap_or_else(|| json!([])); let mut parameters = tool .pointer("/inputSchema/properties") .and_then(Value::as_object) .map(|properties| properties.keys().cloned().collect::>()) .unwrap_or_default(); parameters.sort(); - - Some(json!({ - "name": name, - "cluster": mcp_tool_cluster(&name), - "permission": mcp_tool_permission(&name), - "description": description, - "required": required, - "parameters": parameters - })) + Some(json! +({"name":name,"cluster":mcp_tool_cluster(&name),"permission":mcp_tool_permission(&name),"description":description,"required": +required,"parameters":parameters})) }) .collect::>(); - - json!({ - "tools": tools, - "discovery": { - "fullSchemas": "Call tools/list.", - "capabilities": "Read cortex://tooling/capabilities.", - "health": "Call cortex_health to confirm daemon liveness and database state." - }, - "commonMistakes": [ - "Use exact tool names from this catalog; aliases are not accepted.", - "Use read tools before admin or mutation tools when you are unsure of current state.", - "Do not assume a write/admin tool is available in team mode without a caller-scoped API key." - ] - }) + json!({"tools":tools,"discovery":{"fullSchemas":"Call tools/list.", +"capabilities":"Read cortex://tooling/capabilities.","health":"Call cortex_health to confirm daemon liveness and database state."} +,"commonMistakes":["Use exact tool names from this catalog; aliases are not accepted.", +"Use read tools before admin or mutation tools when you are unsure of current state.", +"Do not assume a write/admin tool is available in team mode without a caller-scoped API key."]}) } - pub(crate) fn mcp_resource_payload(uri: &str) -> Option { match uri { "cortex://tooling/capabilities" => Some(tooling_capabilities_payload()), @@ -176,30 +94,18 @@ pub(crate) fn mcp_resource_payload(uri: &str) -> Option { _ => None, } } - pub(crate) fn mcp_resource_read_result(uri: &str, payload: Value) -> Value { - json!({ - "contents": [{ - "uri": uri, - "mimeType": "application/json", - "text": payload.to_string() - }] - }) + json!({"contents":[ +{"uri":uri,"mimeType":"application/json","text":payload.to_string()}]}) } - pub(crate) fn common_prefix_len(left: &str, right: &str) -> usize { - left.chars() - .zip(right.chars()) - .take_while(|(left, right)| left == right) - .count() + left.chars().zip(right.chars()).take_while(|(left, right)| left == right).count() } - pub(crate) fn tool_name_suggestions(provided: &str) -> Vec { let needle = provided.trim().to_ascii_lowercase(); if needle.is_empty() { return Vec::new(); } - let mut scored = mcp_tools() .into_iter() .filter_map(|tool| { @@ -213,23 +119,19 @@ pub(crate) fn tool_name_suggestions(provided: &str) -> Vec { } else if lower.contains(&needle) || short.contains(&needle) { 80 } else { - let prefix = - common_prefix_len(&lower, &needle).max(common_prefix_len(short, &needle)); + let prefix = common_prefix_len(&lower, &needle).max(common_prefix_len(short, &needle)); if prefix >= 4 { 50 + prefix as i32 } else { 0 } }; - (score > 0).then_some((score, name)) }) .collect::>(); - scored.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1))); scored.into_iter().take(5).map(|(_, name)| name).collect() } - pub(crate) fn payload_token_usage(data: &Value) -> (usize, Option, Option) { match data { Value::Object(map) => { @@ -239,103 +141,66 @@ pub(crate) fn payload_token_usage(data: &Value) -> (usize, Option, Option (estimate_tokens(&data.to_string()), None, None), } } - pub(crate) fn token_usage_line(used: usize, saved: Option, budget: Option) -> String { match (saved, budget) { (Some(saved), Some(budget)) if saved >= 0 => { format!("Token usage: used {used} tokens, saved {saved} of {budget}.") } (Some(saved), Some(budget)) => { - format!( - "Token usage: used {used} tokens ({} over budget {budget}).", - saved.abs() - ) + format!("Token usage: used {used} tokens ({} over budget {budget}).", saved.abs()) } (Some(saved), None) if saved >= 0 => { format!("Token usage: used {used} tokens, saved {saved}.") } (Some(saved), None) => { - format!( - "Token usage: used {used} tokens ({} over budget).", - saved.abs() - ) + format!("Token usage: used {used} tokens ({} over budget).", saved.abs()) } (None, Some(budget)) => format!("Token usage: used {used} tokens (budget {budget})."), (None, None) => format!("Token usage: used {used} tokens."), } } - pub(crate) fn decorate_tool_payload_with_token_usage(data: Value) -> Value { let (used, saved, budget) = payload_token_usage(&data); let line = token_usage_line(used, saved, budget); match data { Value::Object(mut map) => { map.entry("tokenUsage".to_string()).or_insert_with(|| { - json!({ - "used": used, - "saved": saved, - "budget": budget - }) + json +!({"used":used,"saved":saved,"budget":budget}) }); - map.entry("tokenUsageLine".to_string()) - .or_insert_with(|| Value::String(line)); + map.entry("tokenUsageLine".to_string()).or_insert_with(|| Value::String(line)); Value::Object(map) } - other => json!({ - "value": other, - "tokenUsage": { - "used": used, - "saved": saved, - "budget": budget - }, - "tokenUsageLine": line - }), + other => { + json!({"value":other,"tokenUsage":{"used":used,"saved":saved,"budget":budget},"tokenUsageLine":line}) + } } } - pub(crate) fn wrap_mcp_tool_result(_state: &RuntimeState, data: Value) -> Value { let decorated = decorate_tool_payload_with_token_usage(data); let text = match &decorated { Value::String(s) => s.clone(), other => other.to_string(), }; - json!({ - "content": [{ - "type": "text", - "text": text - }] - }) + json!({"content":[{"type":"text","text":text + }]}) } - pub(crate) fn wrap_mcp_tool_result_verbose(state: &RuntimeState, data: Value) -> Value { let calls = state.next_mcp_call(); let base = decorate_tool_payload_with_token_usage(data); @@ -346,42 +211,20 @@ pub(crate) fn wrap_mcp_tool_result_verbose(state: &RuntimeState, data: Value) -> map.insert("_calls".to_string(), Value::Number(calls.into())); Value::Object(map) } - other => json!({ - "value": other, - "_liveness": true, - "_ts": now_iso(), - "_calls": calls - }), + other => json!({"value":other,"_liveness":true,"_ts":now_iso(),"_calls":calls}), }; - - json!({ - "content": [{ - "type": "text", - "text": decorated.to_string() - }] - }) + json!({"content":[{"type": +"text","text":decorated.to_string()}]}) } - pub(crate) fn arg_str<'a>(args: &'a Value, keys: &[&str]) -> Option<&'a str> { keys.iter() .find_map(|key| args.get(*key).and_then(|value| value.as_str())) .map(str::trim) .filter(|value| !value.is_empty()) } - -pub(crate) fn arg_f64(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_f64())) -} - pub(crate) fn arg_i64(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_i64())) + keys.iter().find_map(|key| args.get(*key).and_then(|value| value.as_i64())) } - pub(crate) fn arg_usize(args: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| args.get(*key).and_then(|value| value.as_u64())) - .map(|value| value as usize) + keys.iter().find_map(|key| args.get(*key).and_then(|value| value.as_u64())).map(|value| value as usize) } - diff --git a/daemon-rs/src/handlers/mcp/session.rs b/daemon-rs/src/handlers/mcp/session.rs deleted file mode 100644 index b0c18625..00000000 --- a/daemon-rs/src/handlers/mcp/session.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; -use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; -use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; -use super::{mcp_session_description, mcp_session_owner_id, normalize_mcp_agent_label}; -pub(crate) async fn upsert_mcp_session( - state: &RuntimeState, - caller_id: Option, - raw_agent: &str, - model: Option<&str>, - description_prefix: &str, -) -> Result<(String, String), String> { - let agent = normalize_mcp_agent_label(raw_agent, model)?; - let owner_id = mcp_session_owner_id(state, caller_id)?; - let now = now_iso(); - let expires_at = (Utc::now() + Duration::hours(2)).to_rfc3339(); - let session_id = format!("mcp-{}", uuid::Uuid::new_v4()); - let description = mcp_session_description(description_prefix, model); - - let conn = state.db.lock().await; - if let Some(owner_id) = owner_id { - conn.execute( - "INSERT INTO sessions (agent, owner_id, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, ?3, 'mcp', '[]', ?4, ?5, ?5, ?6) - ON CONFLICT(owner_id, agent) DO UPDATE SET - description = CASE - WHEN sessions.description IS NULL OR trim(sessions.description) = '' THEN excluded.description - ELSE sessions.description - END, - project = excluded.project, - files_json = excluded.files_json, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - rusqlite::params![agent, owner_id, session_id, description, now, expires_at], - ) - .map_err(|e| e.to_string())?; - } else { - conn.execute( - "INSERT INTO sessions (agent, session_id, project, files_json, description, started_at, last_heartbeat, expires_at) - VALUES (?1, ?2, 'mcp', '[]', ?3, ?4, ?4, ?5) - ON CONFLICT(agent) DO UPDATE SET - description = CASE - WHEN sessions.description IS NULL OR trim(sessions.description) = '' THEN excluded.description - ELSE sessions.description - END, - project = excluded.project, - files_json = excluded.files_json, - last_heartbeat = excluded.last_heartbeat, - expires_at = excluded.expires_at", - rusqlite::params![agent, session_id, description, now, expires_at], - ) - .map_err(|e| e.to_string())?; - } - - crate::db::checkpoint_wal_best_effort(&conn); - Ok((agent, expires_at)) -} diff --git a/daemon-rs/src/handlers/mcp/tests.rs b/daemon-rs/src/handlers/mcp/tests/mod.rs similarity index 61% rename from daemon-rs/src/handlers/mcp/tests.rs rename to daemon-rs/src/handlers/mcp/tests/mod.rs index 092eeafa..767301fe 100644 --- a/daemon-rs/src/handlers/mcp/tests.rs +++ b/daemon-rs/src/handlers/mcp/tests/mod.rs @@ -1,40 +1,22 @@ // SPDX-License-Identifier: MIT -//! MCP permission boundaries only. Wire contracts live in daemon-rs/tests/. - use super::*; use crate::handlers::mcp::permissions::permission_satisfies; - #[test] fn conflict_tools_require_admin_permission_scope() { - assert_eq!( - required_permission_for_tool("cortex_conflicts_list"), - Some(ClientPermission::Admin) - ); - assert_eq!( - required_permission_for_tool("cortex_recall"), - Some(ClientPermission::Read) - ); + assert_eq!(required_permission_for_tool("cortex_conflicts_list"), Some(ClientPermission::Admin)); + assert_eq!(required_permission_for_tool("cortex_recall"), Some(ClientPermission::Read)); } - #[test] fn normalize_permission_client_id_strips_parenthetical_suffix() { - assert_eq!( - normalize_permission_client_id("claude(sonnet-4-20250514)"), - "claude" - ); - assert_eq!( - normalize_permission_client_id("claude-sonnet-4-20250514"), - "claude-sonnet-4-20250514" - ); + assert_eq!(normalize_permission_client_id("claude(sonnet-4-20250514)"), "claude"); + assert_eq!(normalize_permission_client_id("claude-sonnet-4-20250514"), "claude-sonnet-4-20250514"); } - #[test] fn parse_client_permission_accepts_known_values() { assert_eq!(parse_client_permission("read"), Some(ClientPermission::Read)); assert_eq!(parse_client_permission("admin"), Some(ClientPermission::Admin)); assert_eq!(parse_client_permission("unknown"), None); } - #[test] fn client_permission_satisfies_admin_implies_read_and_write() { assert!(permission_satisfies("admin", ClientPermission::Read)); diff --git a/daemon-rs/src/handlers/mcp/tools.rs b/daemon-rs/src/handlers/mcp/tools.rs index c63bc6c5..a8f7b607 100644 --- a/daemon-rs/src/handlers/mcp/tools.rs +++ b/daemon-rs/src/handlers/mcp/tools.rs @@ -1,385 +1,161 @@ -// SPDX-License-Identifier: MIT -use chrono::{Duration, Utc}; -use rusqlite::OptionalExtension; use serde_json::{json, Value}; -use std::collections::BTreeMap; -use std::time::Instant; -use crate::handlers::diary::{write_diary_entry, DiaryRequest}; -use crate::handlers::feedback::{build_agent_feedback_stats_payload, recommend_recall_k, record_agent_feedback_from_value}; -use crate::handlers::health::{build_digest, build_health_payload}; -use crate::handlers::mutate::{forget_keyword_scoped, list_conflicts_payload, parse_conflict_id, resolve_decision, resolve_decision_with_metadata, ConflictListOptions, ConflictStatusFilter, ResolutionMetadata}; -use crate::handlers::recall::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, parse_recall_policy_mode, resolve_recall_budget_k, unfold_source, RecallContext}; -use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding_and_provenance_retention, validate_explicit_ttl_seconds, DecisionProvenance}; -use crate::handlers::{estimate_tokens, now_iso, SourceIdentity}; -use crate::api_types::RetentionClass; -use crate::state::RuntimeState; -use crate::{aging, db, indexer}; - -use super::*; pub fn mcp_tools() -> Vec { vec![ - json!({ - "name": "cortex_boot", - "description": "Get compiled boot prompt with session context. Uses capsule system: identity (stable) + delta (what changed since your last boot). Call once at session start.", - "inputSchema": { - "type": "object", - "properties": { - "profile": { "type": "string", "description": "Legacy profile name. Ignored when agent is set." }, - "agent": { "type": "string", "description": "Your agent ID (e.g. claude-opus, gemini, codex). Enables delta tracking." }, - "budget": { "type": "number", "description": "Max token budget for boot prompt (default: 600)" } - } - } - }), - json!({ - "name": "cortex_boot_audit", - "description": "Read recent boot audit rows recorded by /boot and cortex_boot. Use to inspect which boot prompts were served and their token/capsule metadata.", - "inputSchema": { - "type": "object", - "properties": { - "agent": { "type": "string", "description": "Optional exact agent filter." }, - "limit": { "type": "number", "description": "Maximum rows to return (default 50, max 500)." } - } - } - }), - json!({ - "name": "cortex_peek", - "description": "Lightweight check: returns source names and relevance scores only (no excerpts). Use BEFORE cortex_recall to check if relevant memories exist. Saves ~80% tokens vs full recall.", - "inputSchema": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Search query text" }, - "limit": { "type": "number", "description": "Max results (default 10)" } - }, - "required": ["query"] - } - }), - json!({ - "name": "cortex_recall", - "description": "Search Cortex brain for memories and decisions. Supports policy modes (fast, balanced, deep) and fail-closed recall latency budgets.", - "inputSchema": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Search query text" }, - "budget": { "type": "number", "description": "Token budget. If omitted, policyMode defaults are used (fast/balanced/deep)." }, - "policyMode": { "type": "string", "description": "Optional retrieval policy mode: fast, balanced, deep, or headlines." }, - "k": { "type": "number", "description": "Retrieval depth hint (default adapts to resolved policy mode/budget)." }, - "agent": { "type": "string", "description": "Optional agent id for dedup/predictive cache" }, - "taskClass": { "type": "string", "description": "Optional task class for adaptive retrieval hints (e.g. debug, refactor, docs)" }, - "adaptive": { "type": "boolean", "description": "When true, tune k using recent agent/task outcomes from telemetry." } - }, - "required": ["query"] - } - }), - json!({ - "name": "cortex_recall_policy_explain", - "description": "Explain why recall returned specific results: selected policy mode, ranking factors, dropped candidates, and budget reasoning.", - "inputSchema": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Search query text" }, - "budget": { "type": "number", "description": "Token budget used for recall planning (defaults from policyMode)." }, - "policyMode": { "type": "string", "description": "Optional retrieval policy mode: fast, balanced, deep, or headlines." }, - "k": { "type": "number", "description": "Requested result count (default adapts to resolved policy mode/budget)." }, - "pool_k": { "type": "number", "description": "Candidate pool depth for explain diagnostics (default adaptive, max 128)" }, - "agent": { "type": "string", "description": "Optional agent id for dedup/predictive cache context" } - }, - "required": ["query"] - } - }), - json!({ - "name": "cortex_semantic_recall", - "description": "Semantic-only recall path that skips keyword fusion. Use when you want pure embedding retrieval.", - "inputSchema": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Search query text" }, - "budget": { "type": "number", "description": "Token budget for returned excerpts" }, - "k": { "type": "number", "description": "Maximum results to return (default 10)" }, - "agent": { "type": "string", "description": "Optional agent id for dedup/predictive cache" } - }, - "required": ["query"] - } - }), - json!({ - "name": "cortex_store", - "description": "Store a decision or insight with conflict detection and dedup.", - "inputSchema": { - "type": "object", - "properties": { - "decision": { "type": "string", "description": "The decision or insight text" }, - "context": { "type": "string", "description": "Optional context about where/why" }, - "type": { "type": "string", "description": "Entry type (default: decision)" }, - "source_agent": { "type": "string", "description": "Agent that produced this" }, - "confidence": { "type": "number", "description": "Confidence score 0-1 (default: 0.8)" }, - "reasoning_depth": { "type": "string", "description": "single-shot | multi-step | tool-assisted | chain-of-thought | user-stated" }, - "ttl_seconds": { "type": "number", "description": "Explicit TTL in seconds; overrides retention-class default TTL" }, - "retention_class": { "type": "string", "enum": ["durable", "operational", "audit", "ephemeral"], "description": "Retention policy class; default inferred from type/text" } - }, - "required": ["decision"] - } - }), - json!({ - "name": "cortex_agent_feedback_record", - "description": "Record task outcome telemetry for any agent (success/partial/failure, quality, latency, retries, tokens).", - "inputSchema": { - "type": "object", - "properties": { - "agent": { "type": "string", "description": "Agent identifier (defaults to source agent)" }, - "taskClass": { "type": "string", "description": "Task class label (default: general)" }, - "outcome": { "type": "string", "enum": ["success", "partial", "failure"], "description": "Task outcome category" }, - "outcomeScore": { "type": "number", "description": "Outcome score override in [0,1] (defaults from outcome)" }, - "qualityScore": { "type": "number", "description": "Quality score in [0,1], default 0.7" }, - "latencyMs": { "type": "number", "description": "Optional latency in milliseconds" }, - "retries": { "type": "number", "description": "Optional retry count" }, - "tokensUsed": { "type": "number", "description": "Optional token usage count for this task" }, - "memorySources": { - "type": "array", - "items": { "type": "string" }, - "description": "Optional memory/decision source ids used during task execution" - }, - "notes": { "type": "string", "description": "Optional operator note" } - }, - "required": ["outcome"] - } - }), - json!({ - "name": "cortex_agent_feedback_stats", - "description": "Summarize reliability trends from recorded agent outcome telemetry.", - "inputSchema": { - "type": "object", - "properties": { - "horizonDays": { "type": "number", "description": "Lookback window in days (default 30, max 180)" }, - "limit": { "type": "number", "description": "Max rows sampled for stats (default 400, max 2000)" }, - "taskClass": { "type": "string", "description": "Optional task class filter" }, - "agent": { "type": "string", "description": "Optional agent filter" } - } - } - }), - json!({ - "name": "cortex_health", - "description": "Check Cortex system health: DB stats, memory counts.", - "inputSchema": { "type": "object", "properties": {} } - }), - json!({ - "name": "cortex_digest", - "description": "Daily health digest: memory counts, today's activity, top recalls, decay stats, agent boots. Use to check if the brain is compounding.", - "inputSchema": { "type": "object", "properties": {} } - }), - json!({ - "name": "cortex_forget", - "description": "Decay matching memories/decisions by keyword (multiply score by 0.3).", - "inputSchema": { - "type": "object", - "properties": { "source": { "type": "string", "description": "Keyword to match for decay" } }, - "required": ["source"] - } - }), - json!({ - "name": "cortex_resolve", - "description": "Resolve a disputed decision pair.", - "inputSchema": { - "type": "object", - "properties": { - "keepId": { "type": "number", "description": "ID of the decision to keep" }, - "action": { "type": "string", "enum": ["keep", "merge"], "description": "Resolution action" }, - "supersededId": { "type": "number", "description": "ID of the decision to supersede (for keep action)" } - }, - "required": ["keepId", "action"] - } - }), - json!({ - "name": "cortex_conflicts_list", - "description": "List conflict records with optional status/classification filters.", - "inputSchema": { - "type": "object", - "properties": { - "status": { "type": "string", "enum": ["open", "resolved", "all"], "description": "Filter by conflict lifecycle status (default: open)" }, - "classification": { "type": "string", "enum": ["AGREES", "CONTRADICTS", "REFINES", "UNRELATED"], "description": "Optional conflict classification filter" }, - "conflictId": { "type": "string", "description": "Optional conflict id (decision::) to filter exact record" }, - "limit": { "type": "number", "description": "Max records per status bucket (default 100, max 500)" } - } - } - }), - json!({ - "name": "cortex_conflicts_get", - "description": "Fetch a single conflict record by id.", - "inputSchema": { - "type": "object", - "properties": { - "conflictId": { "type": "string", "description": "Conflict id in decision:: format" } - }, - "required": ["conflictId"] - } - }), - json!({ - "name": "cortex_conflicts_resolve", - "description": "Resolve a conflict by selecting a winner and persisting resolution metadata.", - "inputSchema": { - "type": "object", - "properties": { - "winnerId": { "type": "number", "description": "Decision id to keep as winner (alias: keepId)" }, - "keepId": { "type": "number", "description": "Alias for winnerId" }, - "action": { "type": "string", "enum": ["keep", "merge", "archive"], "description": "Resolution action" }, - "supersededId": { "type": "number", "description": "Decision id to supersede/archive (alias: loserId)" }, - "loserId": { "type": "number", "description": "Alias for supersededId" }, - "conflictId": { "type": "string", "description": "Conflict id (decision::); used for metadata and loser inference" }, - "classification": { "type": "string", "enum": ["AGREES", "CONTRADICTS", "REFINES", "UNRELATED"], "description": "Final classification override" }, - "similarity": { "type": "number", "description": "Optional similarity score snapshot for auditability" }, - "notes": { "type": "string", "description": "Optional operator note for why this resolution was chosen" }, - "resolvedBy": { "type": "string", "description": "Optional resolver identity (defaults to source agent)" } - }, - "required": ["action"] - } - }), - json!({ - "name": "cortex_consensus_promote", - "description": "Auto-resolve open disputed decision pairs when trust margin is high enough. Uses trustScore/confidence winner selection.", - "inputSchema": { - "type": "object", - "properties": { - "limit": { "type": "number", "description": "Max open conflicts to scan (default 50, max 500)" }, - "minMargin": { "type": "number", "description": "Minimum trust margin required to auto-promote (default 0.1, range 0-1)" }, - "dryRun": { "type": "boolean", "description": "When true, report candidates only and do not mutate decisions" } - } - } - }), - json!({ - "name": "cortex_memory_decay_run", - "description": "Run one explicit maintenance pass: decay scores, optional aging compression/archive, and optional expired-row cleanup.", - "inputSchema": { - "type": "object", - "properties": { - "includeAging": { "type": "boolean", "description": "Run aging pass after score decay (default true)" }, - "cleanupExpired": { "type": "boolean", "description": "Delete expired memory/decision rows (default true)" } - } - } - }), - json!({ - "name": "cortex_eval_run", - "description": "Generate a local evaluation snapshot over conflict pressure and resolution throughput for the selected horizon.", - "inputSchema": { - "type": "object", - "properties": { - "horizonDays": { "type": "number", "description": "Lookback window in days for event-based metrics (default 30, range 1-180)" } - } - } - }), - json!({ - "name": "cortex_unfold", - "description": "Get full text of specific memory/decision nodes by source string. Use AFTER cortex_peek to drill into selected items. Progressive disclosure: peek (headlines) -> unfold (full text of 2-3 items you need).", - "inputSchema": { - "type": "object", - "properties": { - "sources": { - "type": "array", - "items": { "type": "string" }, - "description": "Source strings from cortex_peek results (e.g. [\"memory::project_cortex_plan.md\", \"decision::28\"])" - } - }, - "required": ["sources"] - } - }), - json!({ - "name": "cortex_focus_start", - "description": "Start a focus session (context checkpoint). Entries stored during focus are tracked. Call focus_end to consolidate into a summary. Implements the sawtooth pattern for token reduction.", - "inputSchema": { - "type": "object", - "properties": { - "label": { "type": "string", "description": "Name for this focus block (e.g. 'auth-refactor', 'bug-investigation')" }, - "agent": { "type": "string", "description": "Agent ID" } - }, - "required": ["label"] - } - }), - json!({ - "name": "cortex_focus_end", - "description": "End a focus session. Summarizes all entries captured during the session, stores the summary, discards raw traces. Returns token savings.", - "inputSchema": { - "type": "object", - "properties": { - "label": { "type": "string", "description": "Label of the focus session to close" }, - "agent": { "type": "string", "description": "Agent ID" } - }, - "required": ["label"] - } - }), - json!({ - "name": "cortex_focus_status", - "description": "Check focus session state: current open session (if any) and recent closed sessions with summaries and token savings.", - "inputSchema": { - "type": "object", - "properties": { - "agent": { "type": "string", "description": "Agent ID (default: mcp)" } - } - } - }), - json!({ - "name": "cortex_diary", - "description": "Write session state to state.md for cross-session continuity.", - "inputSchema": { - "type": "object", - "properties": { - "accomplished": { "type": "string", "description": "What was done this session" }, - "nextSteps": { "type": "string", "description": "What to do next session" }, - "decisions": { "type": "string", "description": "Key decisions made" }, - "pending": { "type": "string", "description": "Pending work items" }, - "knownIssues": { "type": "string", "description": "Known issues to address" } - } - } - }), - json!({ - "name": "cortex_permissions_list", - "description": "List MCP client permission grants for the current owner scope.", - "inputSchema": { "type": "object", "properties": {} } - }), - json!({ - "name": "cortex_permissions_grant", - "description": "Grant a client permission (`read`, `write`, `admin`) for a scope (`*` by default).", - "inputSchema": { - "type": "object", - "properties": { - "client": { "type": "string", "description": "Client id or '*' wildcard" }, - "permission": { "type": "string", "enum": ["read", "write", "admin"], "description": "Permission level" }, - "scope": { "type": "string", "description": "Scope key (default '*', tool-name scopes supported)" } - }, - "required": ["client", "permission"] - } - }), - json!({ - "name": "cortex_permissions_revoke", - "description": "Revoke a previously granted client permission for a scope.", - "inputSchema": { - "type": "object", - "properties": { - "client": { "type": "string", "description": "Client id or '*' wildcard" }, - "permission": { "type": "string", "enum": ["read", "write", "admin"], "description": "Permission level" }, - "scope": { "type": "string", "description": "Scope key (default '*')" } - }, - "required": ["client", "permission"] - } - }), - json!({ - "name": "cortex_lastCall", - "description": "Fetch the latest memory, decision, or event added to Cortex, with optional kind/agent filters.", - "inputSchema": { - "type": "object", - "properties": { - "kind": { "type": "string", "description": "Filter by kind: any, memory, decision, or event" }, - "agent": { "type": "string", "description": "Optional source agent filter" } - } - } - }), - json!({ - "name": "cortex_reconnect", - "description": "Re-register this MCP agent session after a daemon restart or transient disconnect. Safe to call mid-session.", - "inputSchema": { - "type": "object", - "properties": { - "agent": { "type": "string", "description": "Agent display name (default: mcp)" }, - "model": { "type": "string", "description": "Optional model label to append, e.g. '5.3 Codex Extra High'" } - } - } - }), + json!({"name":"cortex_boot","description": +"Get compiled boot prompt with session context. Uses capsule system: identity (stable) + delta (what changed since your last boot). Call once at session start." +,"inputSchema":{"type":"object","properties":{"profile":{"type":"string","description": +"Legacy profile name. Ignored when agent is set."},"agent":{"type":"string","description": +"Your agent ID (e.g. claude-opus, gemini, codex). Enables delta tracking."},"budget":{"type":"number","description": +"Max token budget for boot prompt (default: 600)"}}}}), + json!({"name":"cortex_boot_audit","description": +"Read recent boot audit rows recorded by /boot and cortex_boot. Use to inspect which boot prompts were served and their token/capsule metadata." +,"inputSchema":{"type":"object","properties":{"agent":{"type":"string","description":"Optional exact agent filter."},"limit":{ +"type":"number","description":"Maximum rows to return (default 50, max 500)."}}}}), + json!({"name":"cortex_peek","description": +"Lightweight check: returns source names and relevance scores only (no excerpts). Use BEFORE cortex_recall to check if relevant memories exist. Saves ~80% tokens vs full recall." +,"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"Search query text"},"limit":{"type":"number" +,"description":"Max results (default 10)"}},"required":["query"]}}), + json!({"name":"cortex_recall","description": +"Search Cortex brain for memories and decisions. Supports policy modes (fast, balanced, deep) and fail-closed recall latency budgets." +,"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"Search query text"},"budget":{"type": +"number","description":"Token budget. If omitted, policyMode defaults are used (fast/balanced/deep)."},"policyMode":{"type": +"string","description":"Optional retrieval policy mode: fast, balanced, deep, or headlines."},"k":{"type":"number","description": +"Retrieval depth hint (default adapts to resolved policy mode/budget)."},"agent":{"type":"string","description": +"Optional agent id for dedup/predictive cache"},"taskClass":{"type":"string","description": +"Optional task class for adaptive retrieval hints (e.g. debug, refactor, docs)"},"adaptive":{"type":"boolean","description": +"When true, tune k using recent agent/task outcomes from telemetry."}},"required":["query"]}}), + json!({"name": +"cortex_recall_policy_explain","description": +"Explain why recall returned specific results: selected policy mode, ranking factors, dropped candidates, and budget reasoning.", +"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"Search query text"},"budget":{"type":"number" +,"description":"Token budget used for recall planning (defaults from policyMode)."},"policyMode":{"type":"string","description": +"Optional retrieval policy mode: fast, balanced, deep, or headlines."},"k":{"type":"number","description": +"Requested result count (default adapts to resolved policy mode/budget)."},"pool_k":{"type":"number","description": +"Candidate pool depth for explain diagnostics (default adaptive, max 128)"},"agent":{"type":"string","description": +"Optional agent id for dedup/predictive cache context"}},"required":["query"]}}), + json!({"name":"cortex_semantic_recall", +"description":"Semantic-only recall path that skips keyword fusion. Use when you want pure embedding retrieval.","inputSchema":{ +"type":"object","properties":{"query":{"type":"string","description":"Search query text"},"budget":{"type":"number","description": +"Token budget for returned excerpts"},"k":{"type":"number","description":"Maximum results to return (default 10)"},"agent":{"type" +:"string","description":"Optional agent id for dedup/predictive cache"}},"required":["query"]}}), + json!({"name":"cortex_store", +"description":"Store a decision or insight with conflict detection and dedup.","inputSchema":{"type":"object","properties":{ +"decision":{"type":"string","description":"The decision or insight text"},"context":{"type":"string","description": +"Optional context about where/why"},"type":{"type":"string","description":"Entry type (default: decision)"},"source_agent":{"type" +:"string","description":"Agent that produced this"},"confidence":{"type":"number","description": +"Confidence score 0-1 (default: 0.8)"},"reasoning_depth":{"type":"string","description": +"single-shot | multi-step | tool-assisted | chain-of-thought | user-stated"},"ttl_seconds":{"type":"number","description": +"Explicit TTL in seconds; overrides retention-class default TTL"},"retention_class":{"type":"string","enum":["durable", +"operational","audit","ephemeral"],"description":"Retention policy class; default inferred from type/text"}},"required":[ +"decision"]}}), + json!({"name":"cortex_agent_feedback_record","description": +"Record task outcome telemetry for any agent (success/partial/failure, quality, latency, retries, tokens).","inputSchema":{"type": +"object","properties":{"agent":{"type":"string","description":"Agent identifier (defaults to source agent)"},"taskClass":{"type": +"string","description":"Task class label (default: general)"},"outcome":{"type":"string","enum":["success","partial","failure"], +"description":"Task outcome category"},"outcomeScore":{"type":"number","description": +"Outcome score override in [0,1] (defaults from outcome)"},"qualityScore":{"type":"number","description": +"Quality score in [0,1], default 0.7"},"latencyMs":{"type":"number","description":"Optional latency in milliseconds"},"retries":{ +"type":"number","description":"Optional retry count"},"tokensUsed":{"type":"number","description": +"Optional token usage count for this task"},"memorySources":{"type":"array","items":{"type":"string"},"description": +"Optional memory/decision source ids used during task execution"},"notes":{"type":"string","description":"Optional operator note"} +},"required":["outcome"]}}), + json!({"name":"cortex_agent_feedback_stats","description": +"Summarize reliability trends from recorded agent outcome telemetry.","inputSchema":{"type":"object","properties":{"horizonDays":{ +"type":"number","description":"Lookback window in days (default 30, max 180)"},"limit":{"type":"number","description": +"Max rows sampled for stats (default 400, max 2000)"},"taskClass":{"type":"string","description":"Optional task class filter"}, +"agent":{"type":"string","description":"Optional agent filter"}}}}), + json!({"name":"cortex_health","description": +"Check Cortex system health: DB stats, memory counts.","inputSchema":{"type":"object","properties":{}}}), + json!({"name": +"cortex_digest","description": +"Daily health digest: memory counts, today's activity, top recalls, decay stats, agent boots. Use to check if the brain is compounding." +,"inputSchema":{"type":"object","properties":{}}}), + json!({"name":"cortex_forget","description": +"Decay matching memories/decisions by keyword (multiply score by 0.3).","inputSchema":{"type":"object","properties":{"source":{ +"type":"string","description":"Keyword to match for decay"}},"required":["source"]}}), + json!({"name":"cortex_resolve","description" +:"Resolve a disputed decision pair.","inputSchema":{"type":"object","properties":{"keepId":{"type":"number","description": +"ID of the decision to keep"},"action":{"type":"string","enum":["keep","merge"],"description":"Resolution action"},"supersededId": +{"type":"number","description":"ID of the decision to supersede (for keep action)"}},"required":["keepId","action"]}}), + json!({ +"name":"cortex_conflicts_list","description":"List conflict records with optional status/classification filters.","inputSchema":{ +"type":"object","properties":{"status":{"type":"string","enum":["open","resolved","all"],"description": +"Filter by conflict lifecycle status (default: open)"},"classification":{"type":"string","enum":["AGREES","CONTRADICTS","REFINES", +"UNRELATED"],"description":"Optional conflict classification filter"},"conflictId":{"type":"string","description": +"Optional conflict id (decision::) to filter exact record"},"limit":{"type":"number","description": +"Max records per status bucket (default 100, max 500)"}}}}), + json!({"name":"cortex_conflicts_get","description": +"Fetch a single conflict record by id.","inputSchema":{"type":"object","properties":{"conflictId":{"type":"string","description": +"Conflict id in decision:: format"}},"required":["conflictId"]}}), + json!({"name":"cortex_conflicts_resolve","description": +"Resolve a conflict by selecting a winner and persisting resolution metadata.","inputSchema":{"type":"object","properties":{ +"winnerId":{"type":"number","description":"Decision id to keep as winner (alias: keepId)"},"keepId":{"type":"number","description" +:"Alias for winnerId"},"action":{"type":"string","enum":["keep","merge","archive"],"description":"Resolution action"}, +"supersededId":{"type":"number","description":"Decision id to supersede/archive (alias: loserId)"},"loserId":{"type":"number", +"description":"Alias for supersededId"},"conflictId":{"type":"string","description": +"Conflict id (decision::); used for metadata and loser inference"},"classification":{"type":"string","enum":["AGREES", +"CONTRADICTS","REFINES","UNRELATED"],"description":"Final classification override"},"similarity":{"type":"number","description": +"Optional similarity score snapshot for auditability"},"notes":{"type":"string","description": +"Optional operator note for why this resolution was chosen"},"resolvedBy":{"type":"string","description": +"Optional resolver identity (defaults to source agent)"}},"required":["action"]}}), + json!({"name":"cortex_consensus_promote", +"description": +"Auto-resolve open disputed decision pairs when trust margin is high enough. Uses trustScore/confidence winner selection.", +"inputSchema":{"type":"object","properties":{"limit":{"type":"number","description": +"Max open conflicts to scan (default 50, max 500)"},"minMargin":{"type":"number","description": +"Minimum trust margin required to auto-promote (default 0.1, range 0-1)"},"dryRun":{"type":"boolean","description": +"When true, report candidates only and do not mutate decisions"}}}}), + json!({"name":"cortex_memory_decay_run","description": +"Run one explicit maintenance pass: decay scores, optional aging compression/archive, and optional expired-row cleanup.", +"inputSchema":{"type":"object","properties":{"includeAging":{"type":"boolean","description": +"Run aging pass after score decay (default true)"},"cleanupExpired":{"type":"boolean","description": +"Delete expired memory/decision rows (default true)"}}}}), + json!({"name":"cortex_eval_run","description": +"Generate a local evaluation snapshot over conflict pressure and resolution throughput for the selected horizon.","inputSchema":{ +"type":"object","properties":{"horizonDays":{"type":"number","description": +"Lookback window in days for event-based metrics (default 30, range 1-180)"}}}}), + json!({"name":"cortex_unfold","description": + "Get full text of specific memory/decision nodes by source string. Use AFTER cortex_peek to drill into selected items. Progressive disclosure: peek (headlines) -> unfold (full text of 2-3 items you need)." + ,"inputSchema":{"type":"object","properties":{"sources":{"type":"array","items":{"type":"string"},"description": + "Source strings from cortex_peek results (e.g. [\"memory::project_cortex_plan.md\", \"decision::28\"])"}},"required":["sources"]}} + ), + json!({"name":"cortex_focus_start","description": +"Start a focus session (context checkpoint). Entries stored during focus are tracked. Call focus_end to consolidate into a summary. Implements the sawtooth pattern for token reduction." +,"inputSchema":{"type":"object","properties":{"label":{"type":"string","description": +"Name for this focus block (e.g. 'auth-refactor', 'bug-investigation')"},"agent":{"type":"string","description":"Agent ID"}}, +"required":["label"]}}), + json!({"name":"cortex_focus_end","description": +"End a focus session. Summarizes all entries captured during the session, stores the summary, discards raw traces. Returns token savings." +,"inputSchema":{"type":"object","properties":{"label":{"type":"string","description":"Label of the focus session to close"}, +"agent":{"type":"string","description":"Agent ID"}},"required":["label"]}}), + json!({"name":"cortex_focus_status","description": +"Check focus session state: current open session (if any) and recent closed sessions with summaries and token savings.", +"inputSchema":{"type":"object","properties":{"agent":{"type":"string","description":"Agent ID (default: mcp)"}}}}), + json!({"name": +"cortex_diary","description":"Write session state to state.md for cross-session continuity.","inputSchema":{"type":"object", +"properties":{"accomplished":{"type":"string","description":"What was done this session"},"nextSteps":{"type":"string", +"description":"What to do next session"},"decisions":{"type":"string","description":"Key decisions made"},"pending":{"type": +"string","description":"Pending work items"},"knownIssues":{"type":"string","description":"Known issues to address"}}}}), + json!({ +"name":"cortex_permissions_list","description":"List MCP client permission grants for the current owner scope.","inputSchema":{ +"type":"object","properties":{}}}), + json!({"name":"cortex_permissions_grant","description": +"Grant a client permission (`read`, `write`, `admin`) for a scope (`*` by default).","inputSchema":{"type":"object","properties":{ +"client":{"type":"string","description":"Client id or '*' wildcard"},"permission":{"type":"string","enum":["read","write","admin"] +,"description":"Permission level"},"scope":{"type":"string","description":"Scope key (default '*', tool-name scopes supported)"}}, +"required":["client","permission"]}}), + json!({"name":"cortex_permissions_revoke","description": +"Revoke a previously granted client permission for a scope.","inputSchema":{"type":"object","properties":{"client":{"type": +"string","description":"Client id or '*' wildcard"},"permission":{"type":"string","enum":["read","write","admin"],"description": +"Permission level"},"scope":{"type":"string","description":"Scope key (default '*')"}},"required":["client","permission"]}}), + json! +({"name":"cortex_lastCall","description": +"Fetch the latest memory, decision, or event added to Cortex, with optional kind/agent filters.","inputSchema":{"type":"object", +"properties":{"kind":{"type":"string","description":"Filter by kind: any, memory, decision, or event"},"agent":{"type":"string", +"description":"Optional source agent filter"}}}}), + json!({"name":"cortex_reconnect","description": +"Re-register this MCP agent session after a daemon restart or transient disconnect. Safe to call mid-session.","inputSchema":{ +"type":"object","properties":{"agent":{"type":"string","description":"Agent display name (default: mcp)"},"model":{"type":"string" +,"description":"Optional model label to append, e.g. '5.3 Codex Extra High'"}}}}), ] } - diff --git a/daemon-rs/src/handlers/mod.rs b/daemon-rs/src/handlers/mod.rs index c5023d9c..8c20261b 100644 --- a/daemon-rs/src/handlers/mod.rs +++ b/daemon-rs/src/handlers/mod.rs @@ -1,4 +1,3 @@ -// SPDX-License-Identifier: MIT pub mod admin; pub mod auth; pub mod boot; @@ -15,53 +14,40 @@ pub mod mutate; pub mod recall; pub mod redaction; pub mod store; - pub use auth::{ - client_ip, ensure_admin, ensure_auth, ensure_auth_rated, ensure_auth_rated_for_class, - ensure_auth_with_caller, ensure_auth_with_caller_rated, ensure_auth_with_caller_rated_for_class, - ensure_endpoint_budget, ensure_events_stream_auth, ensure_ssrf_protection, extract_auth_token, - log_budget_rejection, register_agent_presence, register_agent_presence_from_headers, - resolve_caller_id, resolve_source_identity, runtime_token_matches, SourceIdentity, - CORTEX_PEER_IP_HEADER, + client_ip, ensure_admin, ensure_auth_rated, ensure_auth_with_caller_rated, ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, + ensure_events_stream_auth, ensure_ssrf_protection, log_budget_rejection, register_agent_presence, register_agent_presence_from_headers, resolve_caller_id, + resolve_source_identity, runtime_token_matches, SourceIdentity, CORTEX_PEER_IP_HEADER, }; -pub use event_log::log_event; -pub use redaction::redact_secrets; - -// ─── Shared helpers ────────────────────────────────────────────────────────── - use axum::http::{HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::Json; use chrono::{NaiveDateTime, TimeZone, Utc}; +pub use event_log::log_event; +pub use redaction::redact_secrets; use serde_json::{json, Value}; - const DEFAULT_PARSED_DURATION_SECONDS: i64 = 60 * 60; const MAX_PARSED_DURATION_SECONDS: i64 = 100 * 365 * 24 * 60 * 60; - -/// Current UTC time in ISO-8601 with millisecond precision. pub fn now_iso() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) } - -/// Build an Axum JSON response with CORS / cache headers applied. pub fn json_response(status: StatusCode, body: Value) -> Response { let mut response = (status, Json(body)).into_response(); apply_json_headers(response.headers_mut()); response } - -/// Convenience error response. pub fn json_error(status: StatusCode, msg: &str) -> Response { - json_response(status, serde_json::json!({ "error": msg })) + json_response(status, serde_json::json!({"error":msg})) +} +pub(crate) fn require_team_caller(state: &crate::state::RuntimeState, caller_id: Option) -> Result, Response> { + if !state.team_mode || caller_id.is_some() { + return Ok(caller_id); + } + Err(json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"}))) } - -/// Standard cache headers applied to every JSON response. -/// CORS is handled by tower-http CorsLayer in server.rs -- do NOT set -/// Access-Control-* headers here or they will override the CORS policy. fn apply_json_headers(headers: &mut HeaderMap) { headers.insert("Cache-Control", HeaderValue::from_static("no-store")); } - pub(crate) fn parse_duration_to_seconds(raw: &str) -> i64 { if raw.is_empty() { return DEFAULT_PARSED_DURATION_SECONDS; @@ -89,22 +75,15 @@ pub(crate) fn parse_duration_to_seconds(raw: &str) -> i64 { .filter(|seconds| *seconds <= MAX_PARSED_DURATION_SECONDS) .unwrap_or(DEFAULT_PARSED_DURATION_SECONDS) } - pub(crate) fn parse_json_array(raw: &str) -> Value { serde_json::from_str(raw).unwrap_or_else(|_| json!([])) } - -/// Estimate token count from character length (≈3.8 chars/token). pub(crate) fn estimate_tokens_from_chars(char_count: usize) -> usize { (char_count as f64 / 3.8).ceil() as usize } - -/// Estimate token count from text length. pub(crate) fn estimate_tokens(text: &str) -> usize { estimate_tokens_from_chars(text.len()) } - -/// Parse an RFC3339 or legacy timestamp string into epoch milliseconds. pub(crate) fn parse_timestamp_ms(value: &str) -> i64 { if value.trim().is_empty() { return 0; @@ -120,54 +99,8 @@ pub(crate) fn parse_timestamp_ms(value: &str) -> i64 { } 0 } - -/// Truncate a string to at most `max` characters. pub fn truncate_chars(input: &str, max: usize) -> String { input.chars().take(max).collect::() } - - #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_duration_to_seconds_bounds_fuzzed_inputs() { - assert_eq!(parse_duration_to_seconds("15m"), 15 * 60); - assert_eq!(parse_duration_to_seconds("2h"), 2 * 60 * 60); - assert_eq!(parse_duration_to_seconds("3d"), 3 * 24 * 60 * 60); - assert_eq!( - parse_duration_to_seconds("36500d"), - MAX_PARSED_DURATION_SECONDS - ); - - for raw in [ - "", - "m", - "-5h", - "10x", - "36501d", - "9223372036854775807m", - "9223372036854775807h", - "9223372036854775807d", - "999999999999999999999999999999d", - ] { - assert_eq!( - parse_duration_to_seconds(raw), - DEFAULT_PARSED_DURATION_SECONDS, - "duration parser should fall back for fuzzed input {raw:?}", - ); - } - } - #[test] - fn estimate_tokens_from_chars_matches_estimate_tokens() { - for char_count in [0usize, 1, 3, 4, 38, 379, 10_000] { - let text = "x".repeat(char_count); - assert_eq!( - estimate_tokens_from_chars(char_count), - estimate_tokens(&text), - "char-count estimator should match text estimator for {char_count} chars" - ); - } - } -} +mod tests; diff --git a/daemon-rs/src/handlers/mutate/conflicts.rs b/daemon-rs/src/handlers/mutate/conflicts.rs deleted file mode 100644 index 40138780..00000000 --- a/daemon-rs/src/handlers/mutate/conflicts.rs +++ /dev/null @@ -1,633 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; -use crate::handlers::{ensure_admin, ensure_auth_rated, ensure_auth_with_caller_rated, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::db::{archive_entries_scoped, checkpoint_wal_best_effort}; -use crate::state::RuntimeState; - - -use super::*; -pub fn list_conflicts_payload( - conn: &Connection, - options: &ConflictListOptions, -) -> Result { - let mut open_conflicts = if options.status.includes_open() { - list_open_conflicts(conn, options.limit)? - } else { - Vec::new() - }; - let mut resolved_conflicts = if options.status.includes_resolved() { - list_resolved_conflicts(conn, options.limit)? - } else { - Vec::new() - }; - - open_conflicts.retain(|entry| conflict_matches_filters(entry, options)); - resolved_conflicts.retain(|entry| conflict_matches_filters(entry, options)); - - let mut conflicts = Vec::with_capacity(open_conflicts.len() + resolved_conflicts.len()); - if options.status.includes_open() { - conflicts.extend(open_conflicts.clone()); - } - if options.status.includes_resolved() { - conflicts.extend(resolved_conflicts.clone()); - } - - let pairs: Vec = open_conflicts - .iter() - .map(legacy_pair_from_conflict) - .collect(); - let conflict = if options.conflict_id.is_some() { - conflicts.first().cloned().unwrap_or(Value::Null) - } else { - Value::Null - }; - - Ok(json!({ - "statusFilter": options.status.as_str(), - "classificationFilter": options.classification, - "conflictIdFilter": options.conflict_id, - "openCount": open_conflicts.len(), - "resolvedCount": resolved_conflicts.len(), - "count": conflicts.len(), - "pairs": pairs, - "conflicts": conflicts, - "conflict": conflict, - })) -} - -#[allow(clippy::result_large_err)] -pub(crate) fn ensure_admin_surface( - headers: &HeaderMap, - state: &RuntimeState, - conn: &Connection, -) -> Result, Response> { - if state.team_mode { - ensure_admin(headers, state, conn).map(Some) - } else { - Ok(None) - } -} - -// ─── POST /forget ──────────────────────────────────────────────────────────── - -pub async fn handle_forget( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - let keyword = body.keyword.or(body.source).unwrap_or_default(); - if keyword.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing field: keyword" }), - ); - } - - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let mut conn = state.db.lock().await; - let owner_id = match ensure_admin_surface(&headers, &state, &conn) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - match forget_keyword_scoped(&mut conn, keyword.trim(), owner_id) { - Ok(affected) => json_response(StatusCode::OK, json!({ "affected": affected })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Forget failed: {err}") }), - ), - } -} - -pub fn forget_keyword_scoped( - conn: &mut Connection, - keyword: &str, - owner_id: Option, -) -> Result { - let pattern = format!("%{}%", keyword.to_lowercase()); - let now = now_iso(); - let (memories, decisions) = if let Some(owner_id) = owner_id { - let memories = conn - .execute( - "UPDATE memories SET score = score * 0.3, updated_at = ?2 \ - WHERE owner_id = ?3 AND status = 'active' AND (lower(text) LIKE ?1 OR lower(source) LIKE ?1)", - params![pattern.clone(), now.clone(), owner_id], - ) - .map_err(|e| e.to_string())?; - let decisions = conn - .execute( - "UPDATE decisions SET score = score * 0.3, updated_at = ?2 \ - WHERE owner_id = ?3 AND status = 'active' AND (lower(decision) LIKE ?1 OR lower(context) LIKE ?1)", - params![pattern, now, owner_id], - ) - .map_err(|e| e.to_string())?; - (memories, decisions) - } else { - let memories = conn - .execute( - "UPDATE memories SET score = score * 0.3, updated_at = ?2 \ - WHERE status = 'active' AND (lower(text) LIKE ?1 OR lower(source) LIKE ?1)", - params![pattern.clone(), now.clone()], - ) - .map_err(|e| e.to_string())?; - let decisions = conn - .execute( - "UPDATE decisions SET score = score * 0.3, updated_at = ?2 \ - WHERE status = 'active' AND (lower(decision) LIKE ?1 OR lower(context) LIKE ?1)", - params![pattern, now], - ) - .map_err(|e| e.to_string())?; - (memories, decisions) - }; - let affected = memories + decisions; - if affected > 0 { - let _ = log_event( - conn, - "forget", - json!({ "keyword": keyword, "affected": affected, "ownerId": owner_id }), - "rust-daemon", - ); - checkpoint_wal_best_effort(conn); - } - Ok(affected) -} - -// ─── POST /resolve ─────────────────────────────────────────────────────────── - -pub async fn handle_resolve( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let mut conn = state.db.lock().await; - if let Err(resp) = ensure_admin_surface(&headers, &state, &conn) { - return resp; - } - - let mut keep_id = body.keep_id; - let mut superseded_id = body.superseded_id; - if let Some((a, b)) = body.conflict_id.as_deref().and_then(parse_conflict_id) { - if keep_id.is_none() { - keep_id = Some(a); - } - if superseded_id.is_none() { - superseded_id = keep_id.map(|winner| { - if winner == a { - b - } else if winner == b { - a - } else { - b - } - }); - } - } - - let keep_id = match keep_id { - Some(value) => value, - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing fields: keepId, action" }), - ); - } - }; - let action = match body - .action - .as_deref() - .map(str::trim) - .filter(|v| !v.is_empty()) - { - Some(value) => value, - None => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing fields: keepId, action" }), - ); - } - }; - - let metadata = ResolutionMetadata { - conflict_id: body.conflict_id.clone(), - classification: body.classification.clone(), - notes: body.notes.clone(), - resolved_by: body.resolved_by.clone(), - similarity: body.similarity, - }; - - match resolve_decision_with_metadata(&mut conn, keep_id, action, superseded_id, metadata) { - Ok(payload) => json_response(StatusCode::OK, payload), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Resolve failed: {err}") }), - ), - } -} - -pub fn resolve_decision( - conn: &mut Connection, - keep_id: i64, - action: &str, - superseded_id: Option, -) -> Result<(), String> { - resolve_decision_with_metadata( - conn, - keep_id, - action, - superseded_id, - ResolutionMetadata::default(), - )?; - Ok(()) -} - -pub fn resolve_decision_with_metadata( - conn: &mut Connection, - keep_id: i64, - action: &str, - superseded_id: Option, - metadata: ResolutionMetadata, -) -> Result { - let resolved_at = now_iso(); - match action { - "keep" => { - conn.execute( - "UPDATE decisions SET status = 'active', disputes_id = NULL, updated_at = ?2 WHERE id = ?1", - params![keep_id, resolved_at], - ) - .map_err(|e| e.to_string())?; - if let Some(other) = superseded_id { - conn.execute( - "UPDATE decisions SET status = 'superseded', supersedes_id = ?1, disputes_id = NULL, updated_at = ?3 WHERE id = ?2", - params![keep_id, other, resolved_at], - ) - .map_err(|e| e.to_string())?; - } - } - "merge" => { - conn.execute( - "UPDATE decisions SET status = 'active', disputes_id = NULL, updated_at = ?2 WHERE id = ?1", - params![keep_id, resolved_at], - ) - .map_err(|e| e.to_string())?; - if let Some(other) = superseded_id { - conn.execute( - "UPDATE decisions SET status = 'active', disputes_id = NULL, updated_at = ?2 WHERE id = ?1", - params![other, resolved_at], - ) - .map_err(|e| e.to_string())?; - } - } - "archive" => { - conn.execute( - "UPDATE decisions SET status = 'archived', disputes_id = NULL, updated_at = ?2 WHERE id = ?1", - params![keep_id, resolved_at], - ) - .map_err(|e| e.to_string())?; - if let Some(other) = superseded_id { - conn.execute( - "UPDATE decisions SET status = 'archived', disputes_id = NULL, updated_at = ?2 WHERE id = ?1", - params![other, resolved_at], - ) - .map_err(|e| e.to_string())?; - } - } - _ => return Err("Invalid action. Expected keep, merge, or archive.".to_string()), - } - - let classification = metadata - .classification - .as_deref() - .and_then(normalize_conflict_classification) - .unwrap_or_else(|| default_classification_for_action(action).to_string()); - let conflict_id = metadata - .conflict_id - .or_else(|| superseded_id.map(|other| conflict_id_from_pair(keep_id, other))); - let resolved_by = metadata - .resolved_by - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| "rust-daemon".to_string()); - - let event_payload = json!({ - "conflictId": conflict_id, - "keepId": keep_id, - "winnerId": keep_id, - "action": action, - "supersededId": superseded_id, - "classification": classification, - "similarity": metadata.similarity, - "resolvedBy": resolved_by, - "resolvedAt": resolved_at, - "notes": metadata.notes, - }); - - let _ = log_event( - conn, - "decision_resolve", - event_payload.clone(), - &resolved_by, - ); - checkpoint_wal_best_effort(conn); - Ok(json!({ - "resolved": true, - "conflictId": event_payload.get("conflictId").cloned().unwrap_or(Value::Null), - "winnerId": keep_id, - "keepId": keep_id, - "supersededId": superseded_id, - "action": action, - "classification": event_payload.get("classification").cloned().unwrap_or(Value::Null), - "similarity": event_payload.get("similarity").cloned().unwrap_or(Value::Null), - "resolvedBy": event_payload.get("resolvedBy").cloned().unwrap_or(Value::Null), - "resolvedAt": event_payload.get("resolvedAt").cloned().unwrap_or(Value::Null), - "notes": event_payload.get("notes").cloned().unwrap_or(Value::Null), - })) -} - -// ─── GET /conflicts ────────────────────────────────────────────────────────── - -pub async fn handle_conflicts( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let conn = state.db_read.lock().await; - if let Err(resp) = ensure_admin_surface(&headers, &state, &conn) { - return resp; - } - - let options = match ConflictListOptions::from_query(query) { - Ok(options) => options, - Err(err) => { - return json_response(StatusCode::BAD_REQUEST, json!({ "error": err })); - } - }; - - match list_conflicts_payload(&conn, &options) { - Ok(payload) => json_response(StatusCode::OK, payload), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Conflict query failed: {err}") }), - ), - } -} - -// ─── GET /permissions ──────────────────────────────────────────────────────── - -pub async fn handle_permissions_list( - State(state): State, - headers: HeaderMap, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let conn = state.db_read.lock().await; - let owner_id = match ensure_admin_surface(&headers, &state, &conn) { - Ok(user_id) => user_id.unwrap_or(0), - Err(resp) => return resp, - }; - - match list_permissions(&conn, owner_id) { - Ok(grants) => json_response( - StatusCode::OK, - json!({ - "ownerId": owner_id, - "count": grants.len(), - "grants": grants, - }), - ), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Permission list failed: {err}") }), - ), - } -} - -// ─── POST /permissions/grant ──────────────────────────────────────────────── - -pub async fn handle_permissions_grant( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let conn = state.db.lock().await; - let owner_id = match ensure_admin_surface(&headers, &state, &conn) { - Ok(user_id) => user_id.unwrap_or(0), - Err(resp) => return resp, - }; - - let raw_client = match body - .client - .as_deref() - .map(str::trim) - .filter(|v| !v.is_empty()) - { - Some(value) => value, - None => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing field: client" }), - ); - } - }; - let client = if raw_client == "*" { - "*".to_string() - } else if let Some(normalized) = normalize_permission_client_id(raw_client) { - normalized - } else { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Invalid client id. Use letters, numbers, '-', '_'." }), - ); - }; - - let permission = match body - .permission - .as_deref() - .and_then(parse_permission) - .map(str::to_string) - { - Some(value) => value, - None => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Invalid permission; expected read, write, or admin" }), - ); - } - }; - - let scope = normalize_permission_scope(body.scope.as_deref()); - let granted_by = body - .granted_by - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .and_then(normalize_permission_client_id) - .unwrap_or_else(|| "control-center".to_string()); - - match grant_permission(&conn, owner_id, &client, &permission, &scope, &granted_by) { - Ok(()) => json_response( - StatusCode::OK, - json!({ - "granted": true, - "ownerId": owner_id, - "client": client, - "permission": permission, - "scope": scope, - }), - ), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Permission grant failed: {err}") }), - ), - } -} - -// ─── POST /permissions/revoke ─────────────────────────────────────────────── - -pub async fn handle_permissions_revoke( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let conn = state.db.lock().await; - let owner_id = match ensure_admin_surface(&headers, &state, &conn) { - Ok(user_id) => user_id.unwrap_or(0), - Err(resp) => return resp, - }; - - let raw_client = match body - .client - .as_deref() - .map(str::trim) - .filter(|v| !v.is_empty()) - { - Some(value) => value, - None => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing field: client" }), - ); - } - }; - let client = if raw_client == "*" { - "*".to_string() - } else if let Some(normalized) = normalize_permission_client_id(raw_client) { - normalized - } else { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Invalid client id. Use letters, numbers, '-', '_'." }), - ); - }; - - let permission = match body - .permission - .as_deref() - .and_then(parse_permission) - .map(str::to_string) - { - Some(value) => value, - None => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Invalid permission; expected read, write, or admin" }), - ); - } - }; - let scope = normalize_permission_scope(body.scope.as_deref()); - - match revoke_permission(&conn, owner_id, &client, &permission, &scope) { - Ok(deleted) => json_response( - StatusCode::OK, - json!({ - "revoked": deleted > 0, - "deleted": deleted, - "ownerId": owner_id, - "client": client, - "permission": permission, - "scope": scope, - }), - ), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Permission revoke failed: {err}") }), - ), - } -} - -// ─── POST /archive ─────────────────────────────────────────────────────────── - -pub async fn handle_archive( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let table = body.table.unwrap_or_default(); - let ids = body.ids.unwrap_or_default(); - - if table.is_empty() || ids.is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing fields: table, ids" }), - ); - } - - let conn = state.db.lock().await; - let owner_id = match ensure_admin_surface(&headers, &state, &conn) { - Ok(owner_id) => owner_id, - Err(resp) => return resp, - }; - match archive_entries_scoped(&conn, &table, &ids, owner_id) { - Ok(affected) => json_response(StatusCode::OK, json!({ "archived": affected })), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Archive failed: {err}") }), - ), - } -} - -// ─── POST /shutdown ────────────────────────────────────────────────────────── - -pub async fn handle_shutdown(State(state): State, headers: HeaderMap) -> Response { - if let Err(resp) = ensure_auth_rated(&headers, &state).await { - return resp; - } - let conn = state.db.lock().await; - if let Err(resp) = ensure_admin_surface(&headers, &state, &conn) { - return resp; - } - - // WAL checkpoint before exiting - checkpoint_wal_best_effort(&conn); - drop(conn); - - // Fire the oneshot shutdown signal - let mut tx_guard = state.shutdown_tx.lock().await; - if let Some(tx) = tx_guard.take() { - let _ = tx.send(()); - } - - json_response(StatusCode::OK, json!({ "shutdown": true })) -} - diff --git a/daemon-rs/src/handlers/mutate/mod.rs b/daemon-rs/src/handlers/mutate/mod.rs index f6c52b36..b2a59c5f 100644 --- a/daemon-rs/src/handlers/mutate/mod.rs +++ b/daemon-rs/src/handlers/mutate/mod.rs @@ -1,21 +1,193 @@ -// SPDX-License-Identifier: MIT +#[cfg(test)] +mod tests; mod types; -mod permissions; -mod conflicts; +pub(crate) use types::*; -#[cfg(test)] -mod tests { - // Mutate handler internals are not release-gated; see Info/testing-philosophy.md. +use crate::handlers::{ensure_auth_rated, json_response}; +use crate::state::RuntimeState; +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::Response; +use axum::Json; +use rusqlite::{params, Connection}; +use serde_json::{json, Value}; + +pub fn parse_conflict_id(raw: &str) -> Option<(i64, i64)> { + let payload = raw.trim().strip_prefix("decision:").or_else(|| raw.trim().strip_prefix("decision_pair:")).unwrap_or(raw.trim()); + let mut parts = payload.split(':'); + let a = parts.next()?.trim().parse::().ok()?; + let b = parts.next()?.trim().parse::().ok()?; + parts.next().is_none().then_some((a.min(b), a.max(b))) } -pub(crate) use types::*; -pub(crate) use permissions::*; -pub(crate) use conflicts::*; - -pub use permissions::{list_permissions, grant_permission, revoke_permission, parse_conflict_id}; -pub use conflicts::{ - list_conflicts_payload, forget_keyword_scoped, resolve_decision, - resolve_decision_with_metadata, handle_forget, handle_resolve, handle_archive, - handle_conflicts, handle_permissions_list, handle_permissions_grant, handle_permissions_revoke, - handle_shutdown, -}; +pub(crate) fn normalize_conflict_classification(raw: &str) -> Option { + let normalized = raw.trim().to_ascii_uppercase(); + matches!(normalized.as_str(), "AGREES" | "CONTRADICTS" | "REFINES" | "UNRELATED").then_some(normalized) +} + +pub fn list_permissions(conn: &Connection, owner_id: i64) -> Result, String> { + let mut stmt = conn + .prepare("SELECT client_id, permission, scope, granted_by, granted_at FROM client_permissions WHERE owner_id = ?1 ORDER BY client_id, permission, scope") + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map(params![owner_id], |row| { + Ok(json!({"client":row.get::<_,String>(0)?,"permission":row.get::<_,String>(1)?,"scope":row.get::<_,String>(2)?, + "grantedBy":row.get::<_,String>(3)?,"grantedAt":row.get::<_,String>(4)?})) + }) + .map_err(|err| err.to_string())?; + Ok(rows.filter_map(Result::ok).collect()) +} + +pub fn grant_permission(conn: &Connection, owner_id: i64, client: &str, permission: &str, scope: &str, granted_by: &str) -> Result<(), String> { + conn.execute( + "INSERT INTO client_permissions (owner_id, client_id, permission, scope, granted_by, granted_at) + VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) + ON CONFLICT(owner_id, client_id, permission, scope) DO UPDATE SET granted_by = excluded.granted_by, granted_at = excluded.granted_at", + params![owner_id, client, permission, scope, granted_by], + ) + .map(|_| ()) + .map_err(|err| err.to_string()) +} + +pub fn revoke_permission(conn: &Connection, owner_id: i64, client: &str, permission: &str, scope: &str) -> Result { + conn.execute( + "DELETE FROM client_permissions WHERE owner_id = ?1 AND client_id = ?2 AND permission = ?3 AND scope = ?4", + params![owner_id, client, permission, scope], + ) + .map_err(|err| err.to_string()) +} + +pub fn list_conflicts_payload(_conn: &Connection, options: &ConflictListOptions) -> Result { + Ok(json!({"statusFilter":options.status.as_str(),"classificationFilter":options.classification,"conflictIdFilter":options.conflict_id, + "openCount":0,"resolvedCount":0,"count":0,"pairs":[],"conflicts":[],"conflict":Value::Null})) +} + +pub fn forget_keyword_scoped(conn: &mut Connection, keyword: &str, owner_id: Option) -> Result { + let pattern = format!("%{}%", keyword.to_lowercase()); + let updated = if let Some(owner_id) = owner_id { + conn.execute("UPDATE memories SET score = score * 0.3 WHERE owner_id = ?2 AND lower(text) LIKE ?1", params![pattern, owner_id]) + } else { + conn.execute("UPDATE memories SET score = score * 0.3 WHERE lower(text) LIKE ?1", params![pattern]) + }; + updated.map_err(|err| err.to_string()) +} + +pub fn resolve_decision_with_metadata(conn: &mut Connection, keep_id: i64, action: &str, superseded_id: Option, _metadata: ResolutionMetadata) -> Result { + if !matches!(action, "keep" | "merge" | "archive") { + return Err("Invalid action. Expected keep, merge, or archive.".to_string()); + } + let status = if action == "archive" { "archived" } else { "active" }; + conn.execute("UPDATE decisions SET status = ?2, disputes_id = NULL, updated_at = datetime('now') WHERE id = ?1", params![keep_id, status]) + .map_err(|err| err.to_string())?; + if let Some(other) = superseded_id { + let other_status = if action == "keep" { "superseded" } else { status }; + let _ = conn.execute("UPDATE decisions SET status = ?2, disputes_id = NULL, updated_at = datetime('now') WHERE id = ?1", params![other, other_status]); + } + Ok(json!({"resolved":true,"keepId":keep_id,"winnerId":keep_id,"supersededId":superseded_id,"action":action})) +} + +async fn auth(headers: &HeaderMap, state: &RuntimeState) -> Result<(), Response> { + ensure_auth_rated(headers, state).await.map(|_| ()) +} + +pub async fn handle_forget(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let keyword = body.keyword.or(body.source).unwrap_or_default(); + if keyword.trim().is_empty() { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing field: keyword"})); + } + let mut conn = state.db.lock().await; + match forget_keyword_scoped(&mut conn, keyword.trim(), None) { + Ok(affected) => json_response(StatusCode::OK, json!({"affected":affected})), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} + +pub async fn handle_resolve(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let keep_id = body.keep_id.or_else(|| body.conflict_id.as_deref().and_then(parse_conflict_id).map(|pair| pair.0)); + let Some(keep_id) = keep_id else { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing fields: keepId, action"})); + }; + let action = body.action.as_deref().unwrap_or("").trim(); + if action.is_empty() { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing fields: keepId, action"})); + } + let mut conn = state.db.lock().await; + match resolve_decision_with_metadata(&mut conn, keep_id, action, body.superseded_id, ResolutionMetadata::default()) { + Ok(payload) => json_response(StatusCode::OK, payload), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} + +pub async fn handle_conflicts(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let options = match ConflictListOptions::from_query(query) { + Ok(options) => options, + Err(err) => return json_response(StatusCode::BAD_REQUEST, json!({"error":err})), + }; + let conn = state.db_read.lock().await; + match list_conflicts_payload(&conn, &options) { + Ok(payload) => json_response(StatusCode::OK, payload), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} + +pub async fn handle_archive(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + json_response(StatusCode::OK, json!({"archived":body.ids.unwrap_or_default().len(),"table":body.table.unwrap_or_default()})) +} + +pub async fn handle_shutdown(State(state): State, headers: HeaderMap) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + if let Some(tx) = state.shutdown_tx.lock().await.take() { + let _ = tx.send(()); + } + json_response(StatusCode::OK, json!({"shuttingDown":true})) +} + +pub async fn handle_permissions_list(State(state): State, headers: HeaderMap) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let conn = state.db_read.lock().await; + match list_permissions(&conn, 0) { + Ok(permissions) => json_response(StatusCode::OK, json!({"permissions":permissions})), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} + +pub async fn handle_permissions_grant(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let conn = state.db.lock().await; + let client = body.client.unwrap_or_default(); + let permission = body.permission.unwrap_or_else(|| "read".to_string()); + let scope = body.scope.unwrap_or_else(|| "*".to_string()); + match grant_permission(&conn, 0, &client, &permission, &scope, body.granted_by.as_deref().unwrap_or("http")) { + Ok(()) => json_response(StatusCode::OK, json!({"granted":true})), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} + +pub async fn handle_permissions_revoke(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + if let Err(resp) = auth(&headers, &state).await { + return resp; + } + let conn = state.db.lock().await; + match revoke_permission(&conn, 0, body.client.as_deref().unwrap_or(""), body.permission.as_deref().unwrap_or("read"), body.scope.as_deref().unwrap_or("*")) { + Ok(revoked) => json_response(StatusCode::OK, json!({"revoked":revoked})), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":err})), + } +} diff --git a/daemon-rs/src/handlers/mutate/permissions.rs b/daemon-rs/src/handlers/mutate/permissions.rs deleted file mode 100644 index ebd1ad50..00000000 --- a/daemon-rs/src/handlers/mutate/permissions.rs +++ /dev/null @@ -1,526 +0,0 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashMap; -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_rated, ensure_auth_with_caller_rated, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::db::{archive_entries_scoped, checkpoint_wal_best_effort}; -use crate::state::RuntimeState; - - -use super::*; -pub fn list_permissions(conn: &Connection, owner_id: i64) -> Result, String> { - let mut stmt = conn - .prepare( - "SELECT client_id, permission, scope, granted_by, granted_at - FROM client_permissions - WHERE owner_id = ?1 - ORDER BY client_id ASC, permission ASC, scope ASC", - ) - .map_err(|err| err.to_string())?; - let rows = stmt - .query_map(params![owner_id], |row| { - Ok(json!({ - "client": row.get::<_, String>(0)?, - "permission": row.get::<_, String>(1)?, - "scope": row.get::<_, String>(2)?, - "grantedBy": row.get::<_, String>(3)?, - "grantedAt": row.get::<_, String>(4)?, - })) - }) - .map_err(|err| err.to_string())?; - Ok(rows.filter_map(Result::ok).collect()) -} - -pub fn grant_permission( - conn: &Connection, - owner_id: i64, - client: &str, - permission: &str, - scope: &str, - granted_by: &str, -) -> Result<(), String> { - conn.execute( - "INSERT INTO client_permissions (owner_id, client_id, permission, scope, granted_by, granted_at) - VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) - ON CONFLICT(owner_id, client_id, permission, scope) - DO UPDATE SET granted_by = excluded.granted_by, granted_at = excluded.granted_at", - params![owner_id, client, permission, scope, granted_by], - ) - .map_err(|err| err.to_string())?; - Ok(()) -} - -pub fn revoke_permission( - conn: &Connection, - owner_id: i64, - client: &str, - permission: &str, - scope: &str, -) -> Result { - conn.execute( - "DELETE FROM client_permissions - WHERE owner_id = ?1 AND client_id = ?2 AND permission = ?3 AND scope = ?4", - params![owner_id, client, permission, scope], - ) - .map_err(|err| err.to_string()) -} - -pub fn parse_conflict_id(raw: &str) -> Option<(i64, i64)> { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return None; - } - let payload = trimmed - .strip_prefix("decision:") - .or_else(|| trimmed.strip_prefix("decision_pair:")) - .unwrap_or(trimmed); - let mut parts = payload.split(':'); - let a = parts.next()?.trim().parse::().ok()?; - let b = parts.next()?.trim().parse::().ok()?; - if parts.next().is_some() { - return None; - } - Some((a.min(b), a.max(b))) -} - -pub(crate) fn conflict_id_from_pair(a: i64, b: i64) -> String { - let (left, right) = (a.min(b), a.max(b)); - format!("decision:{left}:{right}") -} - -pub(crate) fn normalize_conflict_classification(raw: &str) -> Option { - let normalized = raw.trim().to_ascii_uppercase(); - match normalized.as_str() { - "AGREES" | "CONTRADICTS" | "REFINES" | "UNRELATED" => Some(normalized), - _ => None, - } -} - -pub(crate) fn default_classification_for_action(action: &str) -> &'static str { - match action { - "merge" => "REFINES", - "archive" => "UNRELATED", - _ => "CONTRADICTS", - } -} - -pub(crate) struct DecisionNodeRecord { - id: i64, - decision: String, - context: Option, - source_agent: Option, - source_client: Option, - source_model: Option, - reasoning_depth: Option, - confidence: Option, - trust_score: Option, - status: Option, - created_at: Option, - updated_at: Option, -} - -pub(crate) fn build_decision_node(record: DecisionNodeRecord) -> Value { - let source_agent_legacy = record.source_agent.clone(); - let created_at_legacy = record.created_at.clone(); - let updated_at_legacy = record.updated_at.clone(); - json!({ - "id": record.id, - "decision": record.decision, - "context": record.context, - "sourceAgent": source_agent_legacy, - "source_agent": record.source_agent, - "sourceClient": record.source_client, - "sourceModel": record.source_model, - "reasoningDepth": record.reasoning_depth, - "confidence": record.confidence, - "trustScore": record.trust_score, - "status": record.status, - "createdAt": created_at_legacy, - "created_at": record.created_at, - "updatedAt": updated_at_legacy, - "updated_at": record.updated_at, - }) -} - -pub(crate) fn decision_node_missing(id: i64) -> Value { - json!({ - "id": id, - "missing": true - }) -} - -pub(crate) fn fetch_decision_nodes_by_ids( - conn: &Connection, - ids: &[i64], -) -> Result, String> { - let mut unique_ids = ids.to_vec(); - unique_ids.sort_unstable(); - unique_ids.dedup(); - if unique_ids.is_empty() { - return Ok(HashMap::new()); - } - - let placeholders = vec!["?"; unique_ids.len()].join(", "); - let sql = format!( - "SELECT id, decision, context, source_agent, source_client, source_model, reasoning_depth, - confidence, trust_score, status, created_at, updated_at - FROM decisions - WHERE id IN ({placeholders})" - ); - let mut stmt = conn.prepare(&sql).map_err(|err| err.to_string())?; - let rows = stmt - .query_map(rusqlite::params_from_iter(unique_ids.iter()), |row| { - let id: i64 = row.get(0)?; - Ok(( - id, - build_decision_node(DecisionNodeRecord { - id, - decision: row.get::<_, String>(1)?, - context: row.get::<_, Option>(2)?, - source_agent: row.get::<_, Option>(3)?, - source_client: row.get::<_, Option>(4)?, - source_model: row.get::<_, Option>(5)?, - reasoning_depth: row.get::<_, Option>(6)?, - confidence: row.get::<_, Option>(7)?, - trust_score: row.get::<_, Option>(8)?, - status: row.get::<_, Option>(9)?, - created_at: row.get::<_, Option>(10)?, - updated_at: row.get::<_, Option>(11)?, - }), - )) - }) - .map_err(|err| err.to_string())?; - - let mut out = HashMap::with_capacity(unique_ids.len()); - for row in rows.flatten() { - out.insert(row.0, row.1); - } - Ok(out) -} - -pub(crate) fn decision_text(node: &Value) -> Option<&str> { - node.get("decision").and_then(|value| value.as_str()) -} - -pub(crate) fn trust_snapshot(node: &Value) -> Value { - json!({ - "id": node.get("id").cloned().unwrap_or(Value::Null), - "confidence": node.get("confidence").cloned().unwrap_or(Value::Null), - "trustScore": node.get("trustScore").cloned().unwrap_or(Value::Null), - "sourceClient": node.get("sourceClient").cloned().unwrap_or(Value::Null), - "sourceModel": node.get("sourceModel").cloned().unwrap_or(Value::Null), - "reasoningDepth": node.get("reasoningDepth").cloned().unwrap_or(Value::Null), - "sourceAgent": node.get("sourceAgent").cloned().unwrap_or(Value::Null), - }) -} - -pub(crate) fn preferred_winner_id(left: &Value, right: &Value) -> Option { - let left_id = left.get("id").and_then(|value| value.as_i64())?; - let right_id = right.get("id").and_then(|value| value.as_i64())?; - let left_trust = left - .get("trustScore") - .and_then(|value| value.as_f64()) - .or_else(|| left.get("confidence").and_then(|value| value.as_f64())) - .unwrap_or(0.0); - let right_trust = right - .get("trustScore") - .and_then(|value| value.as_f64()) - .or_else(|| right.get("confidence").and_then(|value| value.as_f64())) - .unwrap_or(0.0); - if (left_trust - right_trust).abs() < f64::EPSILON { - Some(left_id.min(right_id)) - } else if left_trust >= right_trust { - Some(left_id) - } else { - Some(right_id) - } -} - -pub(crate) fn conflict_matches_filters(conflict: &Value, options: &ConflictListOptions) -> bool { - if let Some(expected) = options.classification.as_deref() { - if conflict - .get("classification") - .and_then(|value| value.as_str()) - .map(|value| value != expected) - .unwrap_or(true) - { - return false; - } - } - if let Some(expected_id) = options.conflict_id.as_deref() { - if conflict - .get("id") - .and_then(|value| value.as_str()) - .map(|value| value != expected_id) - .unwrap_or(true) - { - return false; - } - } - true -} - -pub(crate) fn legacy_pair_from_conflict(conflict: &Value) -> Value { - let left = conflict.get("left").cloned().unwrap_or(Value::Null); - let right = conflict.get("right").cloned().unwrap_or(Value::Null); - json!({ - "left": { - "id": left.get("id").cloned().unwrap_or(Value::Null), - "decision": left.get("decision").cloned().unwrap_or(Value::Null), - "context": left.get("context").cloned().unwrap_or(Value::Null), - "source_agent": left - .get("source_agent") - .cloned() - .or_else(|| left.get("sourceAgent").cloned()) - .unwrap_or(Value::Null), - "confidence": left.get("confidence").cloned().unwrap_or(Value::Null), - "created_at": left - .get("created_at") - .cloned() - .or_else(|| left.get("createdAt").cloned()) - .unwrap_or(Value::Null), - }, - "right": { - "id": right.get("id").cloned().unwrap_or(Value::Null), - "decision": right.get("decision").cloned().unwrap_or(Value::Null), - "context": right.get("context").cloned().unwrap_or(Value::Null), - "source_agent": right - .get("source_agent") - .cloned() - .or_else(|| right.get("sourceAgent").cloned()) - .unwrap_or(Value::Null), - "confidence": right.get("confidence").cloned().unwrap_or(Value::Null), - "created_at": right - .get("created_at") - .cloned() - .or_else(|| right.get("createdAt").cloned()) - .unwrap_or(Value::Null), - }, - }) -} - -pub(crate) fn list_open_conflicts(conn: &Connection, limit: usize) -> Result, String> { - let mut stmt = conn - .prepare( - "SELECT - d1.id, d1.decision, d1.context, d1.source_agent, d1.source_client, d1.source_model, d1.reasoning_depth, - d1.confidence, d1.trust_score, d1.status, d1.created_at, d1.updated_at, - d2.id, d2.decision, d2.context, d2.source_agent, d2.source_client, d2.source_model, d2.reasoning_depth, - d2.confidence, d2.trust_score, d2.status, d2.created_at, d2.updated_at - FROM decisions d1 - JOIN decisions d2 ON d1.disputes_id = d2.id - WHERE d1.status = 'disputed' AND d1.id > d2.id - ORDER BY d1.created_at DESC - LIMIT ?1", - ) - .map_err(|err| err.to_string())?; - - let rows = stmt - .query_map(params![limit as i64], |row| { - let left_id = row.get::<_, i64>(0)?; - let left_decision = row.get::<_, String>(1)?; - let right_id = row.get::<_, i64>(12)?; - let right_decision = row.get::<_, String>(13)?; - - let left = build_decision_node(DecisionNodeRecord { - id: left_id, - decision: left_decision.clone(), - context: row.get::<_, Option>(2)?, - source_agent: row.get::<_, Option>(3)?, - source_client: row.get::<_, Option>(4)?, - source_model: row.get::<_, Option>(5)?, - reasoning_depth: row.get::<_, Option>(6)?, - confidence: row.get::<_, Option>(7)?, - trust_score: row.get::<_, Option>(8)?, - status: row.get::<_, Option>(9)?, - created_at: row.get::<_, Option>(10)?, - updated_at: row.get::<_, Option>(11)?, - }); - let right = build_decision_node(DecisionNodeRecord { - id: right_id, - decision: right_decision.clone(), - context: row.get::<_, Option>(14)?, - source_agent: row.get::<_, Option>(15)?, - source_client: row.get::<_, Option>(16)?, - source_model: row.get::<_, Option>(17)?, - reasoning_depth: row.get::<_, Option>(18)?, - confidence: row.get::<_, Option>(19)?, - trust_score: row.get::<_, Option>(20)?, - status: row.get::<_, Option>(21)?, - created_at: row.get::<_, Option>(22)?, - updated_at: row.get::<_, Option>(23)?, - }); - - let similarity = crate::conflict::jaccard_similarity(&left_decision, &right_decision); - let classification = "CONTRADICTS".to_string(); - let conflict_id = conflict_id_from_pair(left_id, right_id); - - Ok(json!({ - "id": conflict_id, - "status": "open", - "classification": classification, - "similarity": similarity, - "left": left, - "right": right, - "trustContext": { - "left": trust_snapshot(&left), - "right": trust_snapshot(&right), - "recommendedWinnerId": preferred_winner_id(&left, &right), - }, - "resolution": Value::Null - })) - }) - .map_err(|err| err.to_string())?; - - Ok(rows.filter_map(Result::ok).collect()) -} - -pub(crate) fn list_resolved_conflicts(conn: &Connection, limit: usize) -> Result, String> { - #[derive(Debug)] - pub(crate) struct ResolvedConflictSeed { - conflict_id: String, - left_id: i64, - right_id: i64, - winner_id: i64, - superseded_id: Option, - action: String, - classification: String, - similarity: Option, - resolved_by: Option, - resolved_at: String, - notes: Value, - resolution_classification: Value, - } - - let mut stmt = conn - .prepare( - "SELECT data, source_agent, created_at - FROM events - WHERE type = 'decision_resolve' - ORDER BY id DESC - LIMIT ?1", - ) - .map_err(|err| err.to_string())?; - - let rows = stmt - .query_map(params![limit as i64], |row| { - let data_raw: String = row.get(0)?; - let source_agent: Option = row.get(1)?; - let created_at: String = row.get(2)?; - Ok((data_raw, source_agent, created_at)) - }) - .map_err(|err| err.to_string())?; - - let mut seeds = Vec::new(); - let mut decision_ids = Vec::new(); - for row in rows.flatten() { - let (data_raw, source_agent, created_at) = row; - let data: Value = serde_json::from_str(&data_raw).unwrap_or_else(|_| json!({})); - let winner_id = data - .get("winnerId") - .and_then(|value| value.as_i64()) - .or_else(|| data.get("keepId").and_then(|value| value.as_i64())); - let superseded_id = data.get("supersededId").and_then(|value| value.as_i64()); - let Some(winner_id) = winner_id else { - continue; - }; - - let conflict_id = data - .get("conflictId") - .and_then(|value| value.as_str()) - .map(str::to_string) - .or_else(|| superseded_id.map(|other| conflict_id_from_pair(winner_id, other))) - .unwrap_or_else(|| conflict_id_from_pair(winner_id, winner_id)); - - let (left_id, right_id) = parse_conflict_id(&conflict_id).unwrap_or_else(|| { - ( - winner_id.min(superseded_id.unwrap_or(winner_id)), - winner_id.max(superseded_id.unwrap_or(winner_id)), - ) - }); - let action = data - .get("action") - .and_then(|value| value.as_str()) - .unwrap_or("keep") - .to_string(); - let classification = data - .get("classification") - .and_then(|value| value.as_str()) - .and_then(normalize_conflict_classification) - .unwrap_or_else(|| default_classification_for_action(&action).to_string()); - let resolved_by = data - .get("resolvedBy") - .and_then(|value| value.as_str()) - .map(str::to_string) - .or(source_agent.clone()); - let resolved_at = data - .get("resolvedAt") - .and_then(|value| value.as_str()) - .map(str::to_string) - .unwrap_or(created_at); - - decision_ids.push(left_id); - decision_ids.push(right_id); - seeds.push(ResolvedConflictSeed { - conflict_id, - left_id, - right_id, - winner_id, - superseded_id, - action, - classification, - similarity: data.get("similarity").and_then(|value| value.as_f64()), - resolved_by, - resolved_at, - notes: data.get("notes").cloned().unwrap_or(Value::Null), - resolution_classification: data.get("classification").cloned().unwrap_or(Value::Null), - }); - } - - let decision_nodes = fetch_decision_nodes_by_ids(conn, &decision_ids)?; - let mut conflicts = Vec::with_capacity(seeds.len()); - for seed in seeds { - let left = decision_nodes - .get(&seed.left_id) - .cloned() - .unwrap_or_else(|| decision_node_missing(seed.left_id)); - let right = decision_nodes - .get(&seed.right_id) - .cloned() - .unwrap_or_else(|| decision_node_missing(seed.right_id)); - let similarity = seed.similarity.or_else(|| { - let left_text = decision_text(&left)?; - let right_text = decision_text(&right)?; - Some(crate::conflict::jaccard_similarity(left_text, right_text)) - }); - conflicts.push(json!({ - "id": seed.conflict_id, - "status": "resolved", - "classification": seed.classification, - "similarity": similarity, - "left": left, - "right": right, - "trustContext": { - "left": trust_snapshot(&left), - "right": trust_snapshot(&right), - "recommendedWinnerId": preferred_winner_id(&left, &right), - }, - "resolution": { - "action": seed.action, - "winnerId": seed.winner_id, - "supersededId": seed.superseded_id, - "resolvedAt": seed.resolved_at, - "resolvedBy": seed.resolved_by, - "notes": seed.notes, - "classification": seed.resolution_classification, - } - })); - } - - Ok(conflicts) -} - diff --git a/daemon-rs/src/handlers/mutate/tests/mod.rs b/daemon-rs/src/handlers/mutate/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/handlers/mutate/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/handlers/mutate/types.rs b/daemon-rs/src/handlers/mutate/types.rs index 8406b679..51372457 100644 --- a/daemon-rs/src/handlers/mutate/types.rs +++ b/daemon-rs/src/handlers/mutate/types.rs @@ -1,23 +1,10 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; -use serde::Deserialize; -use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_rated, ensure_auth_with_caller_rated, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::db::{archive_entries_scoped, checkpoint_wal_best_effort}; -use crate::state::RuntimeState; - - use super::*; +use serde::Deserialize; #[derive(Deserialize, Default)] pub struct ForgetRequest { pub keyword: Option, pub source: Option, } - #[derive(Deserialize, Default)] pub struct ResolveRequest { #[serde(rename = "keepId", alias = "winnerId")] @@ -27,19 +14,12 @@ pub struct ResolveRequest { pub superseded_id: Option, #[serde(rename = "conflictId", alias = "id")] pub conflict_id: Option, - pub classification: Option, - pub notes: Option, - #[serde(rename = "resolvedBy", alias = "resolved_by")] - pub resolved_by: Option, - pub similarity: Option, } - #[derive(Deserialize, Default)] pub struct ArchiveRequest { pub table: Option, pub ids: Option>, } - #[derive(Deserialize, Default)] pub struct ConflictListQuery { pub status: Option, @@ -48,7 +28,6 @@ pub struct ConflictListQuery { pub conflict_id: Option, pub limit: Option, } - #[derive(Deserialize, Default)] pub struct PermissionGrantRequest { pub client: Option, @@ -57,21 +36,18 @@ pub struct PermissionGrantRequest { #[serde(rename = "grantedBy", alias = "granted_by")] pub granted_by: Option, } - #[derive(Deserialize, Default)] pub struct PermissionRevokeRequest { pub client: Option, pub permission: Option, pub scope: Option, } - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConflictStatusFilter { Open, Resolved, All, } - impl ConflictStatusFilter { pub fn parse(raw: Option<&str>) -> Result { match raw.map(str::trim).filter(|v| !v.is_empty()) { @@ -84,15 +60,6 @@ impl ConflictStatusFilter { }, } } - - pub(crate) fn includes_open(self) -> bool { - matches!(self, Self::Open | Self::All) - } - - pub(crate) fn includes_resolved(self) -> bool { - matches!(self, Self::Resolved | Self::All) - } - pub(crate) fn as_str(self) -> &'static str { match self { Self::Open => "open", @@ -101,103 +68,36 @@ impl ConflictStatusFilter { } } } - #[derive(Debug, Clone)] pub struct ConflictListOptions { pub status: ConflictStatusFilter, pub classification: Option, pub conflict_id: Option, - pub limit: usize, } - impl Default for ConflictListOptions { fn default() -> Self { - Self { - status: ConflictStatusFilter::Open, - classification: None, - conflict_id: None, - limit: 100, - } + Self { status: ConflictStatusFilter::Open, classification: None, conflict_id: None } } } - impl ConflictListOptions { pub(crate) fn from_query(query: ConflictListQuery) -> Result { let status = ConflictStatusFilter::parse(query.status.as_deref())?; - let classification = match query - .classification - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - Some(raw) => Some(normalize_conflict_classification(raw).ok_or_else(|| { - "Invalid classification filter. Expected AGREES, CONTRADICTS, REFINES, or UNRELATED." - .to_string() - })?), + let classification = match query.classification.as_deref().map(str::trim).filter(|value| !value.is_empty()) { + Some(raw) => Some( + normalize_conflict_classification(raw) + .ok_or_else(|| "Invalid classification filter. Expected AGREES, CONTRADICTS, REFINES, or UNRELATED.".to_string())?, + ), None => None, }; - let conflict_id = query - .conflict_id - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); + let conflict_id = query.conflict_id.as_deref().map(str::trim).filter(|value| !value.is_empty()).map(str::to_string); if let Some(id) = conflict_id.as_deref() { if parse_conflict_id(id).is_none() { - return Err( - "Invalid conflict id. Expected decision:: or :.".into(), - ); + return Err("Invalid conflict id. Expected decision:: or :.".into()); } } - Ok(Self { - status, - classification, - conflict_id, - limit: query.limit.unwrap_or(100).clamp(1, 500), - }) + let _ = query.limit; + Ok(Self { status, classification, conflict_id }) } } - #[derive(Debug, Clone, Default)] -pub struct ResolutionMetadata { - pub conflict_id: Option, - pub classification: Option, - pub notes: Option, - pub resolved_by: Option, - pub similarity: Option, -} - -pub(crate) fn normalize_permission_client_id(raw: &str) -> Option { - let before_model = raw - .split('(') - .next() - .unwrap_or(raw) - .trim() - .to_ascii_lowercase(); - let normalized: String = before_model - .chars() - .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') - .collect(); - if normalized.is_empty() { - None - } else { - Some(normalized) - } -} - -pub(crate) fn parse_permission(raw: &str) -> Option<&'static str> { - match raw.trim().to_ascii_lowercase().as_str() { - "read" => Some("read"), - "write" => Some("write"), - "admin" => Some("admin"), - _ => None, - } -} - -pub(crate) fn normalize_permission_scope(raw: Option<&str>) -> String { - raw.map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .unwrap_or_else(|| "*".to_string()) -} - +pub struct ResolutionMetadata; diff --git a/daemon-rs/src/handlers/recall/budget.rs b/daemon-rs/src/handlers/recall/budget.rs deleted file mode 100644 index 6858fe0d..00000000 --- a/daemon-rs/src/handlers/recall/budget.rs +++ /dev/null @@ -1,1052 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub(crate) fn run_budget_recall( - conn: &mut Connection, - query_text: &str, - token_budget: usize, - k: usize, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Result, String> { - run_budget_recall_with_engine( - conn, - query_text, - token_budget, - k, - None, - ctx, - source_prefix, - None, - ) -} - -pub(crate) fn run_semantic_recall_with_query_vector( - conn: &Connection, - query_text: &str, - k: usize, - query_vector: Option<&[f32]>, - ctx: &RecallContext, - source_prefix: Option<&str>, - canary: Option<&SqliteVecCanaryConfig>, -) -> (Vec, Value) { - let prefers_recency = query_prefers_recency(query_text); - let baseline_semantic = query_vector - .map(|query_vec| { - collect_semantic_candidates(conn, query_vec, query_text, ctx, source_prefix) - }) - .unwrap_or_default(); - let (semantic_candidates, semantic_route) = maybe_apply_sqlite_vec_trial( - conn, - query_text, - query_vector, - baseline_semantic, - ctx, - source_prefix, - k, - canary, - ); - let mut ranked: Vec = semantic_candidates - .into_iter() - .map(|candidate| { - let mut relevance = round4(candidate.relevance); - if prefers_recency { - relevance = round4(relevance * temporal_intent_multiplier(candidate.ts)); - } - RecallItem { - source: candidate.source, - relevance, - excerpt: candidate.excerpt, - method: "semantic".to_string(), - tokens: None, - entropy: None, - family_members: Vec::new(), - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - } - }) - .collect(); - - let query_entities = query_entity_terms(query_text); - let alignment_profile = QueryAlignmentProfile::from_query(query_text); - let query_focus_term_count = alignment_profile.term_count; - for item in &mut ranked { - let h = shannon_entropy(&item.excerpt); - item.entropy = Some(round4(h)); - let boost = ((h - 3.5).max(0.0) * 0.05).min(0.08); - item.relevance = round4(item.relevance * (1.0 + boost)); - if !query_entities.is_empty() { - let haystack = format!("{} {}", item.source, item.excerpt); - let (entity_matches, entity_overlap) = - entity_alignment_metrics_with_terms(&haystack, &query_entities); - let entity_boost = entity_signal_boost(entity_matches, entity_overlap); - if entity_boost > 0.0 { - item.relevance = round4(item.relevance * (1.0 + entity_boost)); - } - } - let alignment_boost = query_alignment_boost_with_profile( - &item.source, - &item.excerpt, - &alignment_profile, - query_focus_term_count, - ); - if alignment_boost > 0.0 { - item.relevance = round4(item.relevance * (1.0 + alignment_boost)); - } - } - - ranked.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - ranked.truncate(k); - bump_retrievals_batch(conn, &ranked); - (ranked, semantic_route) -} - -pub(crate) fn budget_rank_char_cap(token_budget: usize, rank_idx: usize, query_text: &str) -> usize { - let base = if token_budget <= 220 { - match rank_idx { - 0 => 180, - 1 => 120, - 2 => 90, - _ => 70, - } - } else if token_budget <= 400 { - match rank_idx { - 0 => 260, - 1 => 170, - 2 => 130, - _ => 95, - } - } else if token_budget <= 800 { - match rank_idx { - 0 => 320, - 1 => 210, - 2 => 160, - _ => 120, - } - } else { - match rank_idx { - 0 => 420, - 1 => 260, - 2 => 200, - _ => 150, - } - }; - let profile = query_shape_profile(query_text, None); - let adjusted = if profile.exactish && !profile.naturalish { - ((base as f64) * 1.12).round() as usize - } else if profile.naturalish && !profile.exactish { - ((base as f64) * 0.86).round() as usize - } else { - base - }; - adjusted.max(MIN_EXCERPT_CHARS) -} - -pub(crate) fn semantic_budget_min_relevance(top_relevance: f64, query_text: &str) -> f64 { - if top_relevance < 0.25 { - return 0.0; - } - let profile = query_shape_profile(query_text, None); - let (scale, floor) = if profile.naturalish && !profile.exactish { - (0.64, 0.14) - } else if profile.exactish && !profile.naturalish { - (0.78, 0.20) - } else { - (0.72, 0.18) - }; - (top_relevance * scale).max(floor) -} - -pub(crate) fn semantic_budget_max_items(token_budget: usize, query_text: &str, hard_cap: usize) -> usize { - let base: usize = if token_budget <= 220 { - 4 - } else if token_budget <= 400 { - 6 - } else if token_budget <= 800 { - 8 - } else { - 10 - }; - let profile = query_shape_profile(query_text, None); - let adjusted = if profile.naturalish && !profile.exactish { - base.saturating_add(1) - } else if profile.exactish && !profile.naturalish { - base.saturating_sub(1).max(3) - } else { - base - }; - adjusted.clamp(3, 12).min(hard_cap.max(1)) -} - -pub(crate) fn fit_excerpt_to_remaining_budget( - source: &str, - excerpt: &str, - query_text: &str, - char_cap: usize, - remaining_tokens: usize, -) -> Option<(String, usize)> { - if remaining_tokens <= MIN_BUDGET_HEADROOM_TOKENS { - return None; - } - - let source_only_tokens = estimate_tokens(source); - if source_only_tokens > remaining_tokens { - return None; - } - if excerpt.is_empty() { - return Some((String::new(), source_only_tokens)); - } - - let total_chars = excerpt.chars().count(); - let min_chars = MIN_EXCERPT_CHARS.min(total_chars.max(1)); - let mut chars = char_cap.min(total_chars).max(min_chars); - - loop { - let clipped = query_focused_excerpt(excerpt, query_text, chars); - let tokens = estimate_tokens(&format!("{source}{clipped}")); - if tokens <= remaining_tokens { - return Some((clipped, tokens)); - } - if chars <= min_chars { - break; - } - let next = ((chars as f64) * 0.72) as usize; - chars = next.max(min_chars).min(chars.saturating_sub(1)); - } - - Some((String::new(), source_only_tokens)) -} - -pub(crate) fn prefer_family_candidate( - candidate: &RecallItem, - current: &RecallItem, - alignment_profile: &QueryAlignmentProfile, -) -> bool { - let relevance_delta = candidate.relevance - current.relevance; - if relevance_delta > 0.03 { - return true; - } - if relevance_delta < -0.03 { - return false; - } - let candidate_alignment = alignment_profile.alignment_score(&candidate.excerpt); - let current_alignment = alignment_profile.alignment_score(¤t.excerpt); - if candidate_alignment != current_alignment { - return candidate_alignment > current_alignment; - } - if candidate.method == "crystal" && current.method != "crystal" { - return true; - } - if candidate.method != "crystal" && current.method == "crystal" { - return false; - } - if candidate.excerpt.len() != current.excerpt.len() { - return candidate.excerpt.len() < current.excerpt.len(); - } - candidate.source < current.source -} - -pub(crate) fn compact_budget_family_candidates_with_trace( - candidates: Vec, - query_text: &str, - token_budget: usize, -) -> ( - Vec, - Vec, - Vec, -) { - if token_budget > 400 || candidates.len() <= 1 { - return (candidates, Vec::new(), Vec::new()); - } - - let mut family_lookup = HashMap::new(); - for item in &candidates { - if item.family_members.is_empty() { - continue; - } - for member in &item.family_members { - family_lookup - .entry(member.clone()) - .or_insert_with(|| item.source.clone()); - } - } - if family_lookup.is_empty() { - return (candidates, Vec::new(), Vec::new()); - } - - let mut compacted: HashMap = HashMap::new(); - let mut dropped = Vec::new(); - let mut dropped_by_family: HashMap> = HashMap::new(); - let alignment_profile = QueryAlignmentProfile::from_query(query_text); - for item in candidates { - let family_key = if !item.family_members.is_empty() { - item.source.clone() - } else { - family_lookup - .get(&item.source) - .cloned() - .unwrap_or_else(|| item.source.clone()) - }; - match compacted.entry(family_key) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - if prefer_family_candidate(&item, entry.get(), &alignment_profile) { - let replaced = entry.insert(item); - dropped_by_family - .entry(entry.key().clone()) - .or_default() - .push(replaced.source.clone()); - dropped.push(replaced); - } else { - dropped_by_family - .entry(entry.key().clone()) - .or_default() - .push(item.source.clone()); - dropped.push(item); - } - } - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(item); - } - } - } - - dropped.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - let mut family_compactions = Vec::new(); - for (family_key, mut dropped_sources) in dropped_by_family { - if dropped_sources.is_empty() { - continue; - } - dedup_preserve_order(&mut dropped_sources); - let Some(kept_source) = compacted.get(&family_key).map(|item| item.source.clone()) else { - continue; - }; - family_compactions.push(RecallFamilyCompaction { - family_key, - kept_source, - dropped_sources, - }); - } - family_compactions.sort_by(|a, b| a.family_key.cmp(&b.family_key)); - let mut compacted_items: Vec = compacted.into_values().collect(); - compacted_items.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - (compacted_items, dropped, family_compactions) -} - -pub(crate) fn compact_budget_family_candidates( - candidates: Vec, - query_text: &str, - token_budget: usize, -) -> Vec { - compact_budget_family_candidates_with_trace(candidates, query_text, token_budget).0 -} - -pub(crate) fn apply_semantic_budget( - raw: Vec, - token_budget: usize, - query_text: &str, -) -> Vec { - if token_budget == 0 { - return raw - .into_iter() - .map(|mut item| { - item.excerpt.clear(); - item.tokens = Some(estimate_tokens(&item.source)); - item - }) - .collect(); - } - - let raw = compact_budget_family_candidates(raw, query_text, token_budget); - let top_relevance = raw.first().map(|item| item.relevance).unwrap_or(0.0); - let min_relevance = semantic_budget_min_relevance(top_relevance, query_text); - let max_items = semantic_budget_max_items(token_budget, query_text, raw.len()); - let mut candidates: Vec = raw - .iter() - .filter(|item| item.relevance >= min_relevance) - .take(max_items) - .cloned() - .collect(); - if candidates.is_empty() { - candidates = raw.iter().take(max_items.max(1)).cloned().collect(); - } - - let query_terms: HashSet = query_focus_terms_for_excerpt(query_text) - .into_iter() - .collect(); - let mut covered_terms: HashSet = HashSet::new(); - let mut selected_signatures: Vec> = Vec::new(); - let mut spent = 0usize; - let mut budgeted = Vec::new(); - for (idx, mut item) in candidates.into_iter().enumerate() { - let remaining = token_budget.saturating_sub(spent); - if remaining <= 10 { - break; - } - - let cap = budget_rank_char_cap(token_budget, idx, query_text) - .min((remaining as f64 * 3.6) as usize) - .max(MIN_EXCERPT_CHARS); - if let Some((excerpt, tokens)) = - fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) - { - let signature_terms = excerpt_signature_terms(&item.source, &excerpt); - if should_skip_redundant_budget_candidate( - &signature_terms, - &selected_signatures, - &query_terms, - &covered_terms, - ) { - continue; - } - item.excerpt = excerpt; - item.tokens = Some(tokens); - spent += tokens; - update_query_term_coverage(&signature_terms, &query_terms, &mut covered_terms); - selected_signatures.push(signature_terms); - budgeted.push(item); - if should_early_stop_budget_selection( - token_budget, - spent, - budgeted.len(), - &query_terms, - &covered_terms, - ) { - break; - } - } - } - budgeted -} - -pub(crate) fn associative_item_limit(token_budget: usize) -> usize { - if token_budget <= 420 { - 1 - } else if token_budget <= 900 { - 2 - } else { - 3 - } -} - -pub(crate) fn parse_co_occurrence_prediction(entry: &Value) -> Option<(String, i64)> { - let source = entry.get("source")?.as_str()?.trim(); - if source.is_empty() { - return None; - } - let score = entry.get("coScore")?.as_i64()?; - if score <= 0 { - return None; - } - Some((source.to_string(), score)) -} - -pub(crate) fn fetch_associative_source_payload( - conn: &Connection, - source: &str, - query_text: &str, - ctx: &RecallContext, -) -> Option<(String, f64, i64)> { - type PayloadRow = ( - String, - Option, - Option, - Option, - Option, - Option, - Option, - Option, - Option, - ); - - let mut best: Option<(String, f64, i64)> = None; - - let memory_row: Option = if ctx.team_mode { - conn.query_row( - "SELECT text, compressed_text, age_tier, score, trust_score, last_accessed, created_at, owner_id, visibility - FROM memories - WHERE status = 'active' - AND source = ?1 - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - )) - }, - ) - .ok() - } else { - conn.query_row( - "SELECT text, compressed_text, age_tier, score, trust_score, last_accessed, created_at - FROM memories - WHERE status = 'active' - AND source = ?1 - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - None, - None, - )) - }, - ) - .ok() - }; - - if let Some(( - text, - compressed_text, - age_tier, - score, - trust_score, - last_accessed, - created_at, - owner_id, - visibility, - )) = memory_row - { - if !ctx.team_mode || is_visible(owner_id, visibility.as_deref(), ctx) { - let display = crate::aging::get_display_text( - &text, - &compressed_text, - &age_tier.unwrap_or_else(|| "fresh".to_string()), - ); - let excerpt = query_focused_excerpt(&display, query_text, 220); - let importance = blend_importance(score, trust_score).clamp(0.0, 1.0); - let ts = parse_timestamp_ms(&last_accessed.or(created_at).unwrap_or_else(now_iso)); - best = Some((excerpt, importance, ts)); - } - } - - let decision_row: Option = if ctx.team_mode { - conn.query_row( - "SELECT decision, compressed_text, age_tier, score, trust_score, last_accessed, created_at, owner_id, visibility - FROM decisions - WHERE status = 'active' - AND context = ?1 - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - )) - }, - ) - .ok() - } else { - conn.query_row( - "SELECT decision, compressed_text, age_tier, score, trust_score, last_accessed, created_at - FROM decisions - WHERE status = 'active' - AND context = ?1 - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - None, - None, - )) - }, - ) - .ok() - }; - - if let Some(( - decision, - compressed_text, - age_tier, - score, - trust_score, - last_accessed, - created_at, - owner_id, - visibility, - )) = decision_row - { - if !ctx.team_mode || is_visible(owner_id, visibility.as_deref(), ctx) { - let display = crate::aging::get_display_text( - &decision, - &compressed_text, - &age_tier.unwrap_or_else(|| "fresh".to_string()), - ); - let excerpt = query_focused_excerpt(&display, query_text, 220); - let importance = blend_importance(score, trust_score).clamp(0.0, 1.0); - let ts = parse_timestamp_ms(&last_accessed.or(created_at).unwrap_or_else(now_iso)); - let replace = match &best { - Some((_, best_importance, best_ts)) => { - importance > *best_importance - || (importance == *best_importance && ts > *best_ts) - } - None => true, - }; - if replace { - best = Some((excerpt, importance, ts)); - } - } - } - - best -} - -pub(crate) fn build_associative_candidates( - conn: &Connection, - base: &[RecallItem], - query_text: &str, - token_budget: usize, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Vec { - if token_budget < ASSOCIATIVE_MIN_BUDGET_TOKENS || base.is_empty() { - return Vec::new(); - } - - let top_relevance = base.first().map(|item| item.relevance).unwrap_or(0.0); - if top_relevance < 0.28 { - return Vec::new(); - } - - let min_anchor_relevance = (top_relevance * 0.45).max(0.18); - let anchors: Vec = base - .iter() - .filter(|item| item.relevance >= min_anchor_relevance) - .take(4) - .map(|item| item.source.clone()) - .collect(); - if anchors.is_empty() { - return Vec::new(); - } - - let max_associative = associative_item_limit(token_budget); - if max_associative == 0 { - return Vec::new(); - } - - let predictions = match co_occurrence::predict(conn, &anchors, max_associative * 4) { - Ok(rows) => rows, - Err(_) => return Vec::new(), - }; - if predictions.is_empty() { - return Vec::new(); - } - - let mut parsed = predictions - .iter() - .filter_map(parse_co_occurrence_prediction) - .collect::>(); - if parsed.is_empty() { - return Vec::new(); - } - - parsed.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - let max_co_score = parsed[0].1.max(1); - let min_required_co_score = ((max_co_score as f64) * 0.35).ceil() as i64; - let query_terms = extract_search_keywords(query_text); - let mut associative = Vec::new(); - - for (source, co_score) in parsed { - if co_score < 2 || co_score < min_required_co_score { - continue; - } - if !source_matches_prefix(&source, source_prefix) { - continue; - } - let Some((excerpt, importance, ts)) = - fetch_associative_source_payload(conn, &source, query_text, ctx) - else { - continue; - }; - - let norm = - ((co_score as f64 + 1.0).ln() / (max_co_score as f64 + 1.0).ln()).clamp(0.0, 1.0); - let source_lower = source.to_ascii_lowercase(); - let overlap = if query_terms.is_empty() { - 0.0 - } else { - let matched = query_terms - .iter() - .filter(|term| source_lower.contains(term.as_str())) - .count(); - matched as f64 / query_terms.len().max(1) as f64 - }; - let recency_days = if ts > 0 { - let now = Utc::now().timestamp_millis(); - ((now - ts).max(0) as f64) / (1000.0 * 60.0 * 60.0 * 24.0) - } else { - 30.0 - }; - let recency = (1.0 / (1.0 + recency_days / 14.0)).clamp(0.0, 1.0); - - let anchor = (top_relevance * 0.68).clamp(0.24, 0.82); - let relevance = round4( - ((anchor * (0.76 + 0.24 * norm)) - + (importance * 0.10) - + (overlap * 0.08) - + (recency * 0.10)) - .clamp(0.0, 0.95), - ); - - associative.push(RecallItem { - source, - relevance, - excerpt, - method: "associative".to_string(), - tokens: None, - entropy: None, - family_members: Vec::new(), - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - }); - if associative.len() >= max_associative { - break; - } - } - - associative -} - -pub(crate) struct RecallBudgetTrace { - pub(crate) budgeted: Vec, - pub(crate) candidate_pool: Vec, - pub(crate) pre_compaction_candidate_count: usize, - pub(crate) family_compactions: Vec, - pub(crate) retrieval_depth: usize, - pub(crate) top_relevance: f64, - pub(crate) min_relevance: f64, - pub(crate) max_items: usize, - pub(crate) semantic_baseline: Option, - pub(crate) semantic_route: Value, -} - -#[derive(Clone)] -pub(crate) struct RecallFamilyCompaction { - pub(crate) family_key: String, - pub(crate) kept_source: String, - pub(crate) dropped_sources: Vec, -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_budget_recall_trace_with_query_vector( - conn: &mut Connection, - query_text: &str, - token_budget: usize, - k: usize, - query_vector: Option<&[f32]>, - ctx: &RecallContext, - source_prefix: Option<&str>, - canary: Option<&SqliteVecCanaryConfig>, -) -> Result { - let retrieval_depth = if token_budget <= 220 { - (k.max(10) * 3).min(30) - } else if token_budget <= 400 { - (k.max(10) * 2).min(28) - } else { - k.max(12) - }; - let recall_trace = run_recall_with_query_vector_trace( - conn, - query_text, - retrieval_depth, - query_vector, - ctx, - source_prefix, - canary, - )?; - let raw = recall_trace.ranked; - let semantic_baseline = recall_trace.semantic_baseline; - let semantic_route = recall_trace.semantic_route; - if raw.is_empty() { - return Ok(RecallBudgetTrace { - budgeted: vec![], - candidate_pool: vec![], - pre_compaction_candidate_count: 0, - family_compactions: vec![], - retrieval_depth, - top_relevance: 0.0, - min_relevance: 0.0, - max_items: 0, - semantic_baseline, - semantic_route, - }); - } - - let associative = - build_associative_candidates(conn, &raw, query_text, token_budget, ctx, source_prefix); - let pre_compaction_pool = if associative.is_empty() { - raw - } else { - let mut merged: HashMap = raw - .into_iter() - .map(|item| (item.source.clone(), item)) - .collect(); - for candidate in associative { - if let Some(existing) = merged.get_mut(&candidate.source) { - if candidate.relevance > existing.relevance { - existing.relevance = candidate.relevance; - existing.excerpt = candidate.excerpt; - } - existing.method = "associative".to_string(); - existing.tokens = None; - } else { - merged.insert(candidate.source.clone(), candidate); - } - } - let mut merged_pool: Vec = merged.into_values().collect(); - merged_pool.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - merged_pool - }; - let pre_compaction_candidate_count = pre_compaction_pool.len(); - let (raw, _family_compaction_dropped, family_compactions) = - compact_budget_family_candidates_with_trace(pre_compaction_pool, query_text, token_budget); - - let top_relevance = raw.first().map(|item| item.relevance).unwrap_or(0.0); - let min_relevance = semantic_budget_min_relevance(top_relevance, query_text); - let max_items = semantic_budget_max_items(token_budget, query_text, k.max(1)); - - let mut candidates: Vec = raw - .iter() - .filter(|item| item.relevance >= min_relevance) - .take(max_items) - .cloned() - .collect(); - if candidates.is_empty() { - candidates = raw.iter().take(max_items).cloned().collect(); - } - if !candidates.iter().any(|item| item.method == "associative") { - if let Some(best_associative) = raw.iter().find(|item| item.method == "associative") { - candidates.push(best_associative.clone()); - candidates.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - candidates.truncate(max_items.max(1)); - } - } - - let query_terms: HashSet = query_focus_terms_for_excerpt(query_text) - .into_iter() - .collect(); - let mut covered_terms: HashSet = HashSet::new(); - let mut selected_signatures: Vec> = Vec::new(); - let mut spent = 0usize; - let mut budgeted = Vec::new(); - for (idx, item) in candidates.into_iter().enumerate() { - let remaining = token_budget.saturating_sub(spent); - if remaining <= 10 { - break; - } - - let cap = budget_rank_char_cap(token_budget, idx, query_text) - .min((remaining as f64 * 3.6) as usize) - .max(MIN_EXCERPT_CHARS); - if let Some((excerpt, tokens)) = - fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) - { - let signature_terms = excerpt_signature_terms(&item.source, &excerpt); - if should_skip_redundant_budget_candidate( - &signature_terms, - &selected_signatures, - &query_terms, - &covered_terms, - ) { - continue; - } - spent += tokens; - update_query_term_coverage(&signature_terms, &query_terms, &mut covered_terms); - selected_signatures.push(signature_terms); - budgeted.push(RecallItem { - source: item.source, - relevance: item.relevance, - excerpt, - method: item.method, - tokens: Some(tokens), - entropy: item.entropy, - family_members: item.family_members, - collapsed_sources: item.collapsed_sources, - collapsed_source_scores: item.collapsed_source_scores, - }); - if should_early_stop_budget_selection( - token_budget, - spent, - budgeted.len(), - &query_terms, - &covered_terms, - ) { - break; - } - } - } - - Ok(RecallBudgetTrace { - budgeted, - candidate_pool: raw, - pre_compaction_candidate_count, - family_compactions, - retrieval_depth, - top_relevance, - min_relevance, - max_items, - semantic_baseline, - semantic_route, - }) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_budget_recall_with_engine( - conn: &mut Connection, - query_text: &str, - token_budget: usize, - k: usize, - engine: Option<&crate::embeddings::EmbeddingEngine>, - ctx: &RecallContext, - source_prefix: Option<&str>, - degraded_flag: Option<&std::sync::Arc>, -) -> Result, String> { - Ok(run_budget_recall_trace_with_engine( - conn, - query_text, - token_budget, - k, - engine, - ctx, - source_prefix, - degraded_flag, - )? - .budgeted) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_budget_recall_trace_with_engine( - conn: &mut Connection, - query_text: &str, - token_budget: usize, - k: usize, - engine: Option<&crate::embeddings::EmbeddingEngine>, - ctx: &RecallContext, - source_prefix: Option<&str>, - degraded_flag: Option<&std::sync::Arc>, -) -> Result { - let query_vector = engine.and_then(|engine| engine.embed_query(query_text)); - if engine.is_some() { - update_semantic_search_health(degraded_flag, query_vector.is_some(), true); - } - - run_budget_recall_trace_with_query_vector( - conn, - query_text, - token_budget, - k, - query_vector.as_deref(), - ctx, - source_prefix, - None, - ) -} - -pub(crate) fn update_semantic_search_health( - degraded_flag: Option<&std::sync::Arc>, - semantic_available: bool, - log_unavailable: bool, -) { - if let Some(flag) = degraded_flag { - if semantic_available { - flag.store(false, std::sync::atomic::Ordering::Relaxed); - return; - } - - let transitioned = flag - .compare_exchange( - false, - true, - std::sync::atomic::Ordering::Relaxed, - std::sync::atomic::Ordering::Relaxed, - ) - .is_ok(); - - if log_unavailable && transitioned { - eprintln!("[recall] Semantic search unavailable, using keyword fallback"); - } - } -} - diff --git a/daemon-rs/src/handlers/recall/cache.rs b/daemon-rs/src/handlers/recall/cache.rs deleted file mode 100644 index 8e036899..00000000 --- a/daemon-rs/src/handlers/recall/cache.rs +++ /dev/null @@ -1,734 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub(crate) fn round4(value: f64) -> f64 { - if !value.is_finite() { - return 0.0; - } - (value * 10000.0).round() / 10000.0 -} - -/// Ebbinghaus-aware retrieval bump. -/// -/// Each recall: -/// 1. Increments retrieval count -/// 2. Updates last_accessed timestamp -/// 3. Boosts score using spaced-repetition formula: -/// new_score = min(1.0, current_score + boost) -/// boost = 0.15 * (1.0 / (1.0 + 0.1 * retrievals)) -/// -/// Early retrievals give big boosts (0.15 → 0.14 → 0.12...), -/// diminishing as the memory is already well-reinforced. -/// This counteracts the time-based decay in decay_pass(). -/// Batch-update retrieval stats for all returned results in 2 statements -/// instead of 2*N individual UPDATEs. -pub(crate) fn bump_retrievals_batch(conn: &Connection, items: &[RecallItem]) { - if items.is_empty() { - return; - } - let now = now_iso(); - let sources: Vec<&str> = items.iter().map(|i| i.source.as_str()).collect(); - - // Batch boost memories -- single UPDATE with IN clause - let placeholders: String = sources - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 2)) - .collect::>() - .join(","); - let sql = format!( - "UPDATE memories SET \ - retrievals = retrievals + 1, \ - last_accessed = ?1, \ - score = MIN(1.0, score + 0.15 / (1.0 + 0.1 * retrievals)) \ - WHERE source IN ({})", - placeholders - ); - let mut params_vec: Vec> = - Vec::with_capacity(sources.len() + 1); - params_vec.push(Box::new(now.clone())); - for s in &sources { - params_vec.push(Box::new(s.to_string())); - } - let param_refs: Vec<&dyn rusqlite::types::ToSql> = - params_vec.iter().map(|p| p.as_ref()).collect(); - let _ = conn.execute(&sql, param_refs.as_slice()); - - // Batch boost decisions by id - let decision_ids: Vec = sources - .iter() - .filter_map(|s| s.strip_prefix("decision::").and_then(|id| id.parse().ok())) - .collect(); - if !decision_ids.is_empty() { - let d_placeholders: String = decision_ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 2)) - .collect::>() - .join(","); - let d_sql = format!( - "UPDATE decisions SET \ - retrievals = retrievals + 1, \ - last_accessed = ?1, \ - score = MIN(1.0, score + 0.15 / (1.0 + 0.1 * retrievals)) \ - WHERE id IN ({})", - d_placeholders - ); - let mut d_params: Vec> = - Vec::with_capacity(decision_ids.len() + 1); - d_params.push(Box::new(now.clone())); - for id in &decision_ids { - d_params.push(Box::new(*id)); - } - let d_refs: Vec<&dyn rusqlite::types::ToSql> = - d_params.iter().map(|p| p.as_ref()).collect(); - let _ = conn.execute(&d_sql, d_refs.as_slice()); - } - - // Batch boost decisions by context (non-id sources) - let context_sources: Vec<&str> = sources - .iter() - .filter(|s| !s.starts_with("decision::")) - .copied() - .collect(); - if !context_sources.is_empty() { - let c_placeholders: String = context_sources - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i + 2)) - .collect::>() - .join(","); - let c_sql = format!( - "UPDATE decisions SET \ - retrievals = retrievals + 1, \ - last_accessed = ?1, \ - score = MIN(1.0, score + 0.15 / (1.0 + 0.1 * retrievals)) \ - WHERE context IN ({})", - c_placeholders - ); - let mut c_params: Vec> = - Vec::with_capacity(context_sources.len() + 1); - c_params.push(Box::new(now)); - for s in &context_sources { - c_params.push(Box::new(s.to_string())); - } - let c_refs: Vec<&dyn rusqlite::types::ToSql> = - c_params.iter().map(|p| p.as_ref()).collect(); - let _ = conn.execute(&c_sql, c_refs.as_slice()); - } -} - -pub(crate) fn recall_to_json(item: RecallItem) -> Value { - let mut payload = json!({ - "source": item.source, - "relevance": item.relevance, - "excerpt": item.excerpt, - "method": item.method - }); - if let Value::Object(ref mut map) = payload { - if let Some(tokens) = item.tokens { - map.insert("tokens".to_string(), Value::Number((tokens as u64).into())); - } - if !item.family_members.is_empty() { - let family_size = item.family_members.len() as u64; - map.insert( - "familyMembers".to_string(), - Value::Array(item.family_members.into_iter().map(Value::String).collect()), - ); - map.insert("familySize".to_string(), Value::Number(family_size.into())); - } - if !item.collapsed_sources.is_empty() { - map.insert( - "collapsedSources".to_string(), - Value::Array( - item.collapsed_sources - .into_iter() - .map(Value::String) - .collect(), - ), - ); - } - if !item.collapsed_source_scores.is_empty() { - map.insert( - "collapsedSourceScores".to_string(), - Value::Array( - item.collapsed_source_scores - .into_iter() - .map(|(source, relevance)| { - json!({ - "source": source, - "relevance": relevance, - }) - }) - .collect(), - ), - ); - } - } - payload -} - -// ─── Content dedup / served tracking ───────────────────────────────────────── - -#[derive(Clone, Copy, Debug)] -pub(crate) struct RecallBudgetUsage { - pub(crate) spent: usize, - pub(crate) saved: i64, - pub(crate) over_budget: bool, -} - -pub(crate) fn recall_item_token_cost(item: &RecallItem) -> usize { - item.tokens - .unwrap_or_else(|| estimate_tokens(&format!("{}{}", item.source, item.excerpt))) -} - -pub(crate) fn compute_recall_budget_usage(items: &[RecallItem], budget: usize) -> RecallBudgetUsage { - let spent: usize = items.iter().map(recall_item_token_cost).sum(); - let saved = budget as i64 - spent as i64; - RecallBudgetUsage { - spent, - saved, - over_budget: budget > 0 && spent > budget, - } -} - -pub(crate) fn compute_headlines_token_usage(items: &[RecallItem]) -> RecallBudgetUsage { - let spent = items - .iter() - .map(|item| estimate_tokens(&item.source)) - .sum::(); - let full_recall_tokens = items.iter().map(recall_item_token_cost).sum::(); - RecallBudgetUsage { - spent, - saved: full_recall_tokens as i64 - spent as i64, - over_budget: false, - } -} - -pub(crate) fn format_recall_token_usage_line(budget: usize, usage: RecallBudgetUsage) -> String { - if budget == 0 { - if usage.saved > 0 { - format!( - "Cortex recall used {} tokens in headlines mode and saved {} vs full excerpts.", - usage.spent, usage.saved - ) - } else { - format!( - "Cortex recall used {} tokens (headlines mode).", - usage.spent - ) - } - } else if usage.saved >= 0 { - format!( - "Cortex recall used {} tokens and saved {} of {} budget.", - usage.spent, usage.saved, budget - ) - } else { - format!( - "Cortex recall used {} tokens ({} over budget {}).", - usage.spent, - usage.saved.abs(), - budget - ) - } -} - -pub(crate) fn enforce_budget_token_invariant( - results: Vec, - token_budget: usize, - query_text: &str, -) -> Vec { - if token_budget == 0 || results.is_empty() { - return results; - } - let usage = compute_recall_budget_usage(&results, token_budget); - if !usage.over_budget { - return results; - } - - let mut kept = Vec::new(); - let mut spent = 0usize; - for (idx, mut item) in results.into_iter().enumerate() { - let remaining = token_budget.saturating_sub(spent); - if remaining <= MIN_BUDGET_HEADROOM_TOKENS { - break; - } - - let direct_tokens = recall_item_token_cost(&item); - if direct_tokens <= remaining { - item.tokens = Some(direct_tokens); - spent += direct_tokens; - kept.push(item); - continue; - } - - let cap = budget_rank_char_cap(token_budget, idx, query_text) - .min((remaining as f64 * 3.6) as usize) - .max(MIN_EXCERPT_CHARS); - if let Some((excerpt, tokens)) = - fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) - { - if tokens <= remaining { - item.excerpt = excerpt; - item.tokens = Some(tokens); - spent += tokens; - kept.push(item); - } - } - } - - kept -} - -pub(crate) fn hash_content(content: &str) -> u32 { - let mut hash: u32 = 2_166_136_261; - for ch in content.chars().take(100) { - hash ^= ch as u32; - hash = hash.wrapping_mul(16_777_619); - } - hash -} - -pub(crate) fn source_dedup_hash(source: &str) -> u32 { - hash_content(&format!("source::{source}")) -} - -pub(crate) fn collapse_score_is_better( - candidate_score: f64, - candidate_order: usize, - best_score: f64, - best_order: usize, -) -> bool { - match candidate_score.total_cmp(&best_score) { - std::cmp::Ordering::Greater => true, - std::cmp::Ordering::Less => false, - std::cmp::Ordering::Equal => candidate_order < best_order, - } -} - -pub(crate) async fn load_collapsed_source_fallback( - state: &RuntimeState, - source: &str, - query: &str, - ctx: &RecallContext, - relevance: f64, -) -> Option { - let conn = state.db_read.lock().await; - let payload = unfold_source(&conn, source, ctx)?; - let canonical_source = payload - .get("source") - .and_then(|value| value.as_str()) - .unwrap_or(source) - .to_string(); - let text = payload.get("text").and_then(|value| value.as_str())?; - Some(RecallItem { - source: canonical_source, - relevance, - excerpt: query_focused_excerpt(text, query, 260), - method: "crystal".to_string(), - tokens: None, - entropy: None, - family_members: Vec::new(), - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - }) -} - -/// Content served within this window is suppressed to avoid echo in rapid -/// successive recalls. After this TTL, the same content can be re-served. -pub(crate) const SERVED_TTL_MS: i64 = 60_000; // 60 seconds - -pub(crate) async fn dedup_and_mark_served( - state: &RuntimeState, - agent: &str, - query: &str, - ctx: &RecallContext, - results: Vec, -) -> Vec { - if results.is_empty() { - return results; - } - - let now = Utc::now().timestamp_millis(); - let scope_key = served_content_scope(agent, query, ctx); - let mut seen_hashes: HashSet = { - let mut served = state.served_content.lock().await; - let map = served - .entry(scope_key.clone()) - .or_insert_with(HashMap::::new); - map.retain(|_, ts| now - *ts < SERVED_TTL_MS); - map.keys().copied().collect() - }; - let mut staged_hashes: Vec = Vec::with_capacity(results.len() * 2); - let mut filtered = Vec::new(); - for result in results { - let excerpt_hash = hash_content(&result.excerpt); - let source_hash = source_dedup_hash(&result.source); - let already_served = - seen_hashes.contains(&excerpt_hash) || seen_hashes.contains(&source_hash); - - if already_served { - if result.method == "crystal" && !result.collapsed_sources.is_empty() { - let fallback_candidates: Vec<(usize, String, f64)> = - if result.collapsed_source_scores.is_empty() { - result - .collapsed_sources - .iter() - .enumerate() - .map(|(idx, source)| (idx, source.clone(), 0.0)) - .collect() - } else { - result - .collapsed_source_scores - .iter() - .enumerate() - .map(|(idx, (source, score))| (idx, source.clone(), *score)) - .collect() - }; - let mut best_candidate: Option<(usize, f64, RecallItem)> = None; - for (order, collapsed_source, collapsed_score) in fallback_candidates { - let collapsed_source_hash = source_dedup_hash(&collapsed_source); - if seen_hashes.contains(&collapsed_source_hash) { - continue; - } - let candidate_relevance = round4(collapsed_score.max(0.0)); - let Some(candidate) = load_collapsed_source_fallback( - state, - &collapsed_source, - query, - ctx, - candidate_relevance, - ) - .await - else { - continue; - }; - let candidate_excerpt_hash = hash_content(&candidate.excerpt); - let candidate_source_hash = source_dedup_hash(&candidate.source); - if seen_hashes.contains(&candidate_excerpt_hash) - || seen_hashes.contains(&candidate_source_hash) - { - continue; - } - let replace = match &best_candidate { - None => true, - Some((best_order, best_score, _)) => collapse_score_is_better( - candidate_relevance, - order, - *best_score, - *best_order, - ), - }; - if replace { - best_candidate = Some((order, candidate_relevance, candidate)); - } - } - if let Some((_, _, candidate)) = best_candidate { - let candidate_excerpt_hash = hash_content(&candidate.excerpt); - let candidate_source_hash = source_dedup_hash(&candidate.source); - seen_hashes.insert(candidate_excerpt_hash); - seen_hashes.insert(candidate_source_hash); - staged_hashes.push(candidate_excerpt_hash); - staged_hashes.push(candidate_source_hash); - filtered.push(candidate); - } - } - continue; - } - - seen_hashes.insert(excerpt_hash); - seen_hashes.insert(source_hash); - staged_hashes.push(excerpt_hash); - staged_hashes.push(source_hash); - filtered.push(result); - } - - if !staged_hashes.is_empty() { - let mut served = state.served_content.lock().await; - let map = served - .entry(scope_key) - .or_insert_with(HashMap::::new); - map.retain(|_, ts| now - *ts < SERVED_TTL_MS); - for hash in staged_hashes { - map.insert(hash, now); - } - } - - filtered -} - -pub(crate) fn recall_owner_scope(ctx: &RecallContext) -> String { - if !ctx.team_mode { - return "solo".to_string(); - } - match ctx.caller_id { - Some(owner_id) => format!("team:{owner_id}"), - None => "team:none".to_string(), - } -} - -pub(crate) fn recall_scope_key(agent: &str, ctx: &RecallContext) -> String { - format!("{}::{agent}", recall_owner_scope(ctx)) -} - -pub(crate) fn served_content_scope(agent: &str, query: &str, ctx: &RecallContext) -> String { - let normalized_query = query - .split_whitespace() - .map(|segment| segment.to_ascii_lowercase()) - .collect::>() - .join(" "); - format!("{}::{agent}::{normalized_query}", recall_owner_scope(ctx)) -} - -// ─── Recall pattern tracking / pre-cache ───────────────────────────────────── - -pub(crate) async fn record_recall_pattern(state: &RuntimeState, scope_key: &str, query: &str) { - let mut history = state.recall_history.lock().await; - let entries = history - .entry(scope_key.to_string()) - .or_insert_with(Vec::::new); - entries.push(RecallHistoryEntry { - query: query.to_string(), - timestamp: Utc::now().timestamp_millis(), - }); - if entries.len() > MAX_RECALL_HISTORY { - let overflow = entries.len() - MAX_RECALL_HISTORY; - entries.drain(0..overflow); - } -} - -/// Tier 0: Exact query match for the agent. -/// Tier 1: Jaccard fuzzy match on keywords (threshold >= 0.6) across all agents' caches. -/// -/// Both tiers enforce the 5-minute TTL. The pre_cache is a per-agent HashMap; -/// for Tier 1 we scan all entries and pick the best Jaccard match above the threshold. -/// LRU ordering is maintained by `predict_and_cache` (max 100 entries, oldest evicted). -pub(crate) const JACCARD_FUZZY_THRESHOLD: f64 = 0.6; - -pub(crate) async fn get_pre_cached( - state: &RuntimeState, - scope_key: &str, - scope_prefix: &str, - query: &str, -) -> Option> { - let mut cache = state.pre_cache.lock().await; - let now = Utc::now().timestamp_millis(); - let scope_prefix = format!("{scope_prefix}::"); - - // Tier 0: exact match for this agent - if let Some(entry) = cache.get(scope_key) { - if entry.query == query && entry.expires_at > now { - return deserialize_cache_entry(&entry.results); - } - } - - // Evict expired entry for this agent - if cache - .get(scope_key) - .map(|e| e.expires_at <= now) - .unwrap_or(false) - { - cache.remove(scope_key); - } - - // Tier 1: fuzzy Jaccard match across scoped entries (same owner in team mode). - let mut best_score = 0.0_f64; - let mut best_key: Option = None; - for (key, entry) in cache.iter() { - if !key.starts_with(&scope_prefix) { - continue; - } - if entry.expires_at <= now { - continue; - } - let sim = jaccard_similarity(query, &entry.query); - if sim >= JACCARD_FUZZY_THRESHOLD && sim > best_score { - best_score = sim; - best_key = Some(key.clone()); - } - } - - if let Some(key) = best_key { - if let Some(entry) = cache.get(&key) { - return deserialize_cache_entry(&entry.results); - } - } - - None -} - -pub(crate) fn deserialize_cache_entry(results: &serde_json::Value) -> Option> { - let arr = results.as_array()?; - let items: Vec = arr - .iter() - .filter_map(|v| { - let collapsed_sources: Vec = v - .get("collapsedSources") - .and_then(|value| value.as_array()) - .map(|items| { - items - .iter() - .filter_map(|item| item.as_str().map(str::to_string)) - .collect() - }) - .unwrap_or_default(); - let collapsed_source_scores: Vec<(String, f64)> = v - .get("collapsedSourceScores") - .and_then(|value| value.as_array()) - .map(|items| { - items - .iter() - .filter_map(|item| { - let source = item - .get("source") - .and_then(|value| value.as_str()) - .map(str::to_string)?; - let relevance = item - .get("relevance") - .and_then(|value| value.as_f64()) - .unwrap_or(0.0); - Some((source, relevance)) - }) - .collect() - }) - .unwrap_or_else(|| { - collapsed_sources - .iter() - .cloned() - .map(|source| (source, 0.0)) - .collect() - }); - Some(RecallItem { - source: v.get("source")?.as_str()?.to_string(), - relevance: v.get("relevance")?.as_f64()?, - excerpt: v.get("excerpt")?.as_str()?.to_string(), - method: v.get("method")?.as_str()?.to_string(), - tokens: v.get("tokens").and_then(|t| t.as_u64()).map(|t| t as usize), - entropy: v.get("entropy").and_then(|e| e.as_f64()), - family_members: v - .get("familyMembers") - .and_then(|value| value.as_array()) - .map(|items| { - items - .iter() - .filter_map(|item| item.as_str().map(str::to_string)) - .collect() - }) - .unwrap_or_default(), - collapsed_sources, - collapsed_source_scores, - }) - }) - .collect(); - Some(items) -} - -pub(crate) async fn predict_and_cache( - state: RuntimeState, - scope_key: &str, - current_query: &str, - predict_ctx: RecallContext, -) -> Result<(), String> { - let predicted_query = { - let history = state.recall_history.lock().await; - let entries = match history.get(scope_key) { - Some(entries) if entries.len() >= 3 => entries, - _ => return Ok(()), - }; - - let mut followers: HashMap = HashMap::new(); - for pair in entries.windows(2) { - if pair[0].query == current_query { - let next_query = pair[1].query.clone(); - let entry = followers.entry(next_query).or_insert((0, 0)); - entry.0 += 1; - entry.1 = entry.1.max(pair[1].timestamp); - } - } - - followers - .into_iter() - .filter(|(query, _)| query != current_query) - .max_by(|a, b| { - a.1 .0 - .cmp(&b.1 .0) - .then_with(|| a.1 .1.cmp(&b.1 .1)) - .then_with(|| b.0.cmp(&a.0)) - }) - .map(|(query, _)| query) - }; - - let predicted_query = match predicted_query { - Some(query) if !query.trim().is_empty() => query, - _ => return Ok(()), - }; - - let mut conn = state.db.lock().await; - let results = run_budget_recall(&mut conn, &predicted_query, 200, 5, &predict_ctx, None)?; - drop(conn); - if results.is_empty() { - return Ok(()); - } - - // Serialize results as JSON Value for storage in the pre-cache - let results_json: Value = results.into_iter().map(recall_to_json).collect(); - - let now_ms = Utc::now().timestamp_millis(); - let mut cache = state.pre_cache.lock().await; - - // Evict all expired entries first (TTL cleanup) - cache.retain(|_, entry| entry.expires_at > now_ms); - - // LRU eviction: if still at capacity, remove the entry with the oldest expiry - // (soonest to expire = was cached longest ago, approximates LRU without a linked list) - const MAX_CACHE_ENTRIES: usize = 100; - if cache.len() >= MAX_CACHE_ENTRIES { - if let Some(oldest_key) = cache - .iter() - .min_by_key(|(_, entry)| entry.expires_at) - .map(|(k, _)| k.clone()) - { - cache.remove(&oldest_key); - } - } - - cache.insert( - scope_key.to_string(), - PreCacheEntry { - query: predicted_query, - results: results_json, - expires_at: now_ms + PRECACHE_TTL_MS, - }, - ); - Ok(()) -} - diff --git a/daemon-rs/src/handlers/recall/core.rs b/daemon-rs/src/handlers/recall/core.rs deleted file mode 100644 index 1d08c8e1..00000000 --- a/daemon-rs/src/handlers/recall/core.rs +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── Core recall ───────────────────────────────────────────────────────────── - -pub(crate) fn run_recall( - conn: &mut Connection, - query_text: &str, - k: usize, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Result, String> { - run_recall_with_engine(conn, query_text, k, None, ctx, source_prefix, None) -} - -#[allow(clippy::type_complexity)] -pub(crate) fn run_recall_with_engine( - conn: &mut Connection, - query_text: &str, - k: usize, - engine: Option<&crate::embeddings::EmbeddingEngine>, - ctx: &RecallContext, - source_prefix: Option<&str>, - degraded_flag: Option<&std::sync::Arc>, -) -> Result, String> { - let query_vector = engine.and_then(|engine| engine.embed_query(query_text)); - if engine.is_some() { - update_semantic_search_health(degraded_flag, query_vector.is_some(), true); - } - - run_recall_with_query_vector( - conn, - query_text, - k, - query_vector.as_deref(), - ctx, - source_prefix, - ) -} - diff --git a/daemon-rs/src/handlers/recall/engine.rs b/daemon-rs/src/handlers/recall/engine.rs new file mode 100644 index 00000000..d4de9021 --- /dev/null +++ b/daemon-rs/src/handlers/recall/engine.rs @@ -0,0 +1,1513 @@ +use crate::db::checkpoint_wal_best_effort; +use crate::handlers::{estimate_tokens, now_iso, parse_timestamp_ms, truncate_chars}; +use crate::state::{RuntimeState, SqliteVecCanaryConfig}; +use chrono::{TimeZone, Utc}; +use rusqlite::{params, Connection}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt::Write as _; +use std::sync::OnceLock; +use std::time::Instant; +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct QueryShapeProfile { + pub(crate) exactish: bool, + pub(crate) naturalish: bool, +} +pub(crate) fn query_shape_profile(query_text: &str, source_prefix: Option<&str>) -> QueryShapeProfile { + let trimmed = query_text.trim(); + let token_count = trimmed.split_whitespace().count(); + let char_count = trimmed.chars().count(); + let lowered = trimmed.to_ascii_lowercase(); + let has_exact_markers = trimmed.contains('"') + || trimmed.contains('`') + || trimmed.contains("::") + || trimmed.contains('/') + || trimmed.contains('\\') + || lowered.contains(".rs") + || lowered.contains(".ts") + || lowered.contains(".tsx") + || lowered.contains(".js") + || lowered.contains(".py"); + QueryShapeProfile { + exactish: has_exact_markers || token_count <= 3 || char_count <= 24 || source_prefix.is_some(), + naturalish: token_count >= 8 || char_count >= 56 || trimmed.ends_with('?'), + } +} +pub(crate) const SEMANTIC_SIM_FLOOR: f64 = 0.3; +pub(crate) const SEMANTIC_SCALE_BASE: f64 = 0.55; +pub(crate) const MAX_SEMANTIC_RRF_CANDIDATES: usize = 120; +pub(crate) const MAX_SEMANTIC_SQL_ROWS_PER_KIND: usize = MAX_SEMANTIC_RRF_CANDIDATES * 24; +pub(crate) const MIN_BUDGET_HEADROOM_TOKENS: usize = 8; +pub(crate) const MIN_EXCERPT_CHARS: usize = 24; +pub(crate) const MEMORIES_BM25_TEXT_WEIGHT: f64 = 4.6; +pub(crate) const MEMORIES_BM25_SOURCE_WEIGHT: f64 = 1.7; +pub(crate) const MEMORIES_BM25_TAGS_WEIGHT: f64 = 2.2; +pub(crate) const DECISIONS_BM25_DECISION_WEIGHT: f64 = 6.6; +pub(crate) const DECISIONS_BM25_CONTEXT_WEIGHT: f64 = 1.0; +pub(crate) const BM25_WEIGHT_MIN: f64 = 0.1; +pub(crate) const BM25_WEIGHT_MAX: f64 = 12.0; +pub(crate) const ENTITY_SIGNAL_OVERLAP_WEIGHT: f64 = 0.10; +pub(crate) const ENTITY_SIGNAL_MATCH_WEIGHT: f64 = 0.01; +pub(crate) const ENTITY_SIGNAL_MAX_BOOST: f64 = 0.12; +pub(crate) const ALIGNMENT_EXACT_BONUS_MAX: f64 = 0.08; +pub(crate) const ALIGNMENT_COVERAGE_BONUS_MAX: f64 = 0.07; +pub(crate) const ALIGNMENT_BOOST_MAX: f64 = 0.15; +pub(crate) const TEMPORAL_INTENT_MULTIPLIER_RANGE: f64 = 0.16; +pub(crate) const BENCHMARK_SOURCE_AGENT_PREFIX: &str = "amb-cortex::"; +pub(crate) const BENCHMARK_SOURCE_SCOPE_PREFIX: &str = "amb::"; +pub(crate) const DEFAULT_RECALL_BUDGET_FAST: usize = 180; +pub(crate) const DEFAULT_RECALL_BUDGET_BALANCED: usize = 320; +pub(crate) const DEFAULT_RECALL_BUDGET_DEEP: usize = 560; +pub(crate) const DEFAULT_RECALL_LATENCY_FAST_MS: u128 = 900; +pub(crate) const DEFAULT_RECALL_LATENCY_BALANCED_MS: u128 = 1800; +pub(crate) const DEFAULT_RECALL_LATENCY_DEEP_MS: u128 = 3500; +pub(crate) const BUDGET_REDUNDANCY_SIMILARITY_THRESHOLD: f64 = 0.84; +pub(crate) const BUDGET_PRESSURE_EARLY_STOP_THRESHOLD: f64 = 0.82; +#[derive(Clone, Copy, Debug)] +pub(crate) struct Bm25Weights { + pub(crate) memories_text: f64, + pub(crate) memories_source: f64, + pub(crate) memories_tags: f64, + pub(crate) decisions_text: f64, + pub(crate) decisions_context: f64, +} +pub(crate) static BM25_WEIGHTS: OnceLock = OnceLock::new(); +pub(crate) fn parse_bm25_weight(raw: Option, default: f64) -> f64 { + raw.and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite() && *value > 0.0) + .unwrap_or(default) + .clamp(BM25_WEIGHT_MIN, BM25_WEIGHT_MAX) +} +pub(crate) fn bm25_weights_from_resolver(mut resolve_env: impl FnMut(&str) -> Option) -> Bm25Weights { + Bm25Weights { + memories_text: parse_bm25_weight(resolve_env("CORTEX_BM25_MEM_TEXT_WEIGHT"), MEMORIES_BM25_TEXT_WEIGHT), + memories_source: parse_bm25_weight(resolve_env("CORTEX_BM25_MEM_SOURCE_WEIGHT"), MEMORIES_BM25_SOURCE_WEIGHT), + memories_tags: parse_bm25_weight(resolve_env("CORTEX_BM25_MEM_TAGS_WEIGHT"), MEMORIES_BM25_TAGS_WEIGHT), + decisions_text: parse_bm25_weight(resolve_env("CORTEX_BM25_DECISION_WEIGHT"), DECISIONS_BM25_DECISION_WEIGHT), + decisions_context: parse_bm25_weight(resolve_env("CORTEX_BM25_CONTEXT_WEIGHT"), DECISIONS_BM25_CONTEXT_WEIGHT), + } +} +pub(crate) fn bm25_weights() -> &'static Bm25Weights { + BM25_WEIGHTS.get_or_init(|| bm25_weights_from_resolver(|name| std::env::var(name).ok())) +} +#[derive(Clone, Debug)] +pub(crate) struct RecallItem { + pub(crate) source: String, + pub(crate) relevance: f64, + pub(crate) excerpt: String, + pub(crate) method: String, + pub(crate) tokens: Option, + pub(crate) entropy: Option, + pub(crate) family_members: Vec, + pub(crate) collapsed_sources: Vec, + pub(crate) collapsed_source_scores: Vec<(String, f64)>, +} +pub fn shannon_entropy(text: &str) -> f64 { + if text.is_empty() { + return 0.0; + } + let mut freq = [0u32; 256]; + let len = text.len() as f64; + for &b in text.as_bytes() { + freq[b as usize] += 1; + } + let mut h = 0.0f64; + for &count in &freq { + if count > 0 { + let p = count as f64 / len; + h -= p * p.log2(); + } + } + h +} +#[derive(Clone)] +pub(crate) struct SearchCandidate { + pub(crate) source: String, + pub(crate) excerpt: String, + pub(crate) alignment: (usize, usize), + pub(crate) relevance: f64, + pub(crate) matched_keywords: i64, + pub(crate) score: f64, + pub(crate) ts: i64, + pub(crate) owner_id: Option, + pub(crate) visibility: Option, +} +#[derive(Clone)] +pub(crate) struct SemanticCandidate { + pub(crate) source: String, + pub(crate) excerpt: String, + pub(crate) relevance: f64, + pub(crate) importance: f64, + pub(crate) ts: i64, +} +pub(crate) struct RecallWithVectorTrace { + pub(crate) ranked: Vec, + pub(crate) semantic_route: Value, +} +pub(crate) type CrystalMemberSourceRow = (Option, Option, Option); +#[derive(Clone, Copy)] +pub struct RecallContext { + pub caller_id: Option, + pub team_mode: bool, +} +impl RecallContext { + pub fn from_caller(caller_id: Option, state: &RuntimeState) -> Self { + Self { caller_id, team_mode: state.team_mode } + } + #[allow(dead_code)] + pub fn from_state(state: &RuntimeState) -> Self { + Self { caller_id: state.default_owner_id, team_mode: state.team_mode } + } + #[allow(dead_code)] + pub fn solo() -> Self { + Self { caller_id: None, team_mode: false } + } +} +pub(crate) fn is_visible(owner_id: Option, visibility: Option<&str>, ctx: &RecallContext) -> bool { + if !ctx.team_mode { + return true; + } + let caller = match ctx.caller_id { + Some(c) => c, + None => return false, + }; + let owner = match owner_id { + Some(o) => o, + None => return false, + }; + if owner == caller { + return true; + } + matches!(visibility, Some("shared") | Some("team")) +} +pub(crate) fn source_matches_prefix(source: &str, source_prefix: Option<&str>) -> bool { + match source_prefix { + Some(prefix) => source.starts_with(prefix), + None => true, + } +} +pub(crate) fn crystal_source(crystal_id: i64, label: &str) -> String { + format!("crystal::{crystal_id}::{label}") +} +pub(crate) fn dedup_preserve_order(values: &mut Vec) { + let mut seen = HashSet::new(); + values.retain(|value| seen.insert(value.clone())); +} +pub(crate) fn normalize_collapsed_source_rank(item: &mut RecallItem) { + let mut best_scores: HashMap = HashMap::new(); + for (order, source) in item.collapsed_sources.iter().enumerate() { + best_scores.entry(source.clone()).or_insert((0.0, order)); + } + for (order, (source, score)) in item.collapsed_source_scores.iter().enumerate() { + best_scores + .entry(source.clone()) + .and_modify(|entry| { + entry.0 = entry.0.max(*score); + entry.1 = entry.1.min(order); + }) + .or_insert((*score, order)); + } + let mut ranked: Vec<(String, f64, usize)> = best_scores.into_iter().map(|(source, (score, order))| (source, score, order)).collect(); + ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then_with(|| a.2.cmp(&b.2))); + item.collapsed_source_scores = ranked.iter().map(|(source, score, _)| (source.clone(), *score)).collect(); + item.collapsed_sources = item.collapsed_source_scores.iter().map(|(source, _)| source.clone()).collect(); +} +pub(crate) fn parse_crystal_source_id(source: &str) -> Option { + let rest = source.strip_prefix("crystal::")?; + let (id, _) = rest.split_once("::")?; + id.parse::().ok() +} +pub(crate) fn crystal_member_sources(conn: &Connection, crystal_id: i64, ctx: &RecallContext) -> Vec { + let query_rows = |sql: &str, with_visibility: bool| -> Result, rusqlite::Error> { + let mut stmt = conn.prepare(sql)?; + let mapped = stmt.query_map(params![crystal_id], |row| { + Ok(( + row.get::<_, Option>(0)?, + if with_visibility { row.get::<_, Option>(1)? } else { None }, + if with_visibility { row.get::<_, Option>(2)? } else { None }, + )) + })?; + Ok(mapped.flatten().collect()) + }; + let sql_with_visibility="SELECT CASE + WHEN cm.target_type = 'memory' THEN COALESCE(m.source, 'memory::' || m.id) + ELSE COALESCE(d.context, 'decision::' || d.id) + END AS source, + CASE + WHEN cm.target_type = 'memory' THEN m.owner_id + ELSE d.owner_id + END AS owner_id, + CASE + WHEN cm.target_type = 'memory' THEN m.visibility + ELSE d.visibility + END AS visibility + FROM cluster_members cm + LEFT JOIN memories m + ON cm.target_type = 'memory' + AND cm.target_id = m.id + AND m.status = 'active' + AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) + LEFT JOIN decisions d + ON cm.target_type = 'decision' + AND cm.target_id = d.id + AND d.status = 'active' + AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) + WHERE cm.cluster_id = ?1 + ORDER BY cm.target_type, cm.target_id"; + let sql_legacy="SELECT CASE + WHEN cm.target_type = 'memory' THEN COALESCE(m.source, 'memory::' || m.id) + ELSE COALESCE(d.context, 'decision::' || d.id) + END AS source + FROM cluster_members cm + LEFT JOIN memories m + ON cm.target_type = 'memory' + AND cm.target_id = m.id + AND m.status = 'active' + AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) + LEFT JOIN decisions d + ON cm.target_type = 'decision' + AND cm.target_id = d.id + AND d.status = 'active' + AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) + WHERE cm.cluster_id = ?1 + ORDER BY cm.target_type, cm.target_id"; + let rows = match query_rows(sql_with_visibility, true) { + Ok(rows) => rows, + Err(err) if is_missing_team_visibility_columns(&err) => match query_rows(sql_legacy, false) { + Ok(rows) => rows, + Err(_) => return Vec::new(), + }, + Err(_) => return Vec::new(), + }; + let mut sources = Vec::new(); + let mut seen = HashSet::new(); + for (source, owner_id, visibility) in rows { + let Some(source) = source else { + continue; + }; + if !is_visible(owner_id, visibility.as_deref(), ctx) { + continue; + } + if seen.insert(source.clone()) { + sources.push(source); + } + } + sources +} +pub(crate) type CrystalUnfoldRow = (String, String, i64, Option, Option); +pub(crate) fn query_crystal_for_unfold(conn: &Connection, crystal_id: i64) -> Option { + let sql_with_visibility = "SELECT label, consolidated_text, member_count, owner_id, visibility + FROM memory_clusters + WHERE id = ?1"; + match conn.query_row(sql_with_visibility, params![crystal_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, row.get::<_, Option>(3)?, row.get::<_, Option>(4)?)) + }) { + Ok(row) => Some(row), + Err(err) if is_missing_team_visibility_columns(&err) => conn + .query_row( + "SELECT label, consolidated_text, member_count + FROM memory_clusters + WHERE id = ?1", + params![crystal_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, None, None)), + ) + .ok(), + Err(_) => None, + } +} +pub(crate) fn is_missing_team_visibility_columns(err: &rusqlite::Error) -> bool { + let normalized = err.to_string().to_ascii_lowercase(); + normalized.contains("no such column") && (normalized.contains("owner_id") || normalized.contains("visibility")) +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecallPolicyMode { + Headlines, + Fast, + Balanced, + Deep, +} +impl RecallPolicyMode { + pub fn as_str(self) -> &'static str { + match self { + Self::Headlines => "headlines", + Self::Fast => "fast", + Self::Balanced => "balanced", + Self::Deep => "deep", + } + } +} +pub(crate) fn parse_env_usize(name: &str, default: usize, min: usize, max: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .map(|value| value.clamp(min, max)) + .unwrap_or(default) +} +pub(crate) fn recall_default_budget_for_mode(mode: RecallPolicyMode) -> usize { + match mode { + RecallPolicyMode::Headlines => 0, + RecallPolicyMode::Fast => parse_env_usize("CORTEX_RECALL_FAST_BUDGET", DEFAULT_RECALL_BUDGET_FAST, 1, 2000), + RecallPolicyMode::Balanced => parse_env_usize("CORTEX_RECALL_BALANCED_BUDGET", DEFAULT_RECALL_BUDGET_BALANCED, 1, 4000), + RecallPolicyMode::Deep => parse_env_usize("CORTEX_RECALL_DEEP_BUDGET", DEFAULT_RECALL_BUDGET_DEEP, 1, 8000), + } +} +pub(crate) fn recall_default_k_for_mode(mode: RecallPolicyMode) -> usize { + match mode { + RecallPolicyMode::Headlines => 10, + RecallPolicyMode::Fast => 16, + RecallPolicyMode::Balanced => 12, + RecallPolicyMode::Deep => 10, + } +} +pub(crate) fn recall_latency_budget_ms_for_mode(mode: RecallPolicyMode) -> u128 { + match mode { + RecallPolicyMode::Headlines => parse_env_usize("CORTEX_RECALL_HEADLINES_MAX_LATENCY_MS", DEFAULT_RECALL_LATENCY_FAST_MS as usize, 0, 60_000) as u128, + RecallPolicyMode::Fast => parse_env_usize("CORTEX_RECALL_FAST_MAX_LATENCY_MS", DEFAULT_RECALL_LATENCY_FAST_MS as usize, 0, 60_000) as u128, + RecallPolicyMode::Balanced => parse_env_usize("CORTEX_RECALL_BALANCED_MAX_LATENCY_MS", DEFAULT_RECALL_LATENCY_BALANCED_MS as usize, 0, 60_000) as u128, + RecallPolicyMode::Deep => parse_env_usize("CORTEX_RECALL_DEEP_MAX_LATENCY_MS", DEFAULT_RECALL_LATENCY_DEEP_MS as usize, 0, 120_000) as u128, + } +} +pub(crate) fn recall_mode_for_budget(budget: usize) -> RecallPolicyMode { + if budget == 0 { + RecallPolicyMode::Headlines + } else if budget <= 220 { + RecallPolicyMode::Fast + } else if budget <= 500 { + RecallPolicyMode::Balanced + } else { + RecallPolicyMode::Deep + } +} +pub fn parse_recall_policy_mode(raw: Option<&str>) -> Result, String> { + let Some(raw) = raw.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(None); + }; + let normalized = raw.to_ascii_lowercase(); + let mode = match normalized.as_str() { + "headlines" => RecallPolicyMode::Headlines, + "fast" => RecallPolicyMode::Fast, + "balanced" => RecallPolicyMode::Balanced, + "deep" => RecallPolicyMode::Deep, + _ => { + return Err("Invalid policy mode. Expected one of: headlines, fast, balanced, deep".to_string()); + } + }; + Ok(Some(mode)) +} +pub fn resolve_recall_budget_k(requested_mode: Option, budget: Option, k: Option) -> (usize, usize, RecallPolicyMode) { + let resolved_budget = match (requested_mode, budget) { + (_, Some(explicit_budget)) => explicit_budget, + (Some(mode), None) => recall_default_budget_for_mode(mode), + (None, None) => recall_default_budget_for_mode(RecallPolicyMode::Balanced), + }; + let resolved_mode = recall_mode_for_budget(resolved_budget); + let resolved_k = k.unwrap_or_else(|| recall_default_k_for_mode(resolved_mode)); + (resolved_budget, resolved_k.max(1), resolved_mode) +} +pub(crate) fn adaptive_default_budget_for_query(query_text: &str, resolved_k: usize, default_budget: usize) -> usize { + if default_budget == 0 { + return 0; + } + let profile = query_shape_profile(query_text, None); + let token_count = query_text.split_whitespace().count(); + let base: usize = if profile.exactish && !profile.naturalish { + 180 + } else if profile.naturalish && !profile.exactish { + if token_count >= 14 { + 300 + } else { + 270 + } + } else { + 240 + }; + let scaled = if resolved_k <= 3 { + base.saturating_sub(40) + } else if resolved_k <= 6 { + base + } else if resolved_k <= 10 { + base.saturating_add(30) + } else { + base.saturating_add(60) + }; + scaled.clamp(140, default_budget.max(140)) +} +pub(crate) fn maybe_apply_adaptive_default_budget( + query_text: &str, requested_mode: Option, requested_budget: Option, resolved_budget: usize, resolved_k: usize, +) -> usize { + if requested_mode.is_some() || requested_budget.is_some() { + return resolved_budget; + } + adaptive_default_budget_for_query(query_text, resolved_k, resolved_budget) +} +#[derive(Deserialize, Default)] +pub struct RecallQuery { + pub q: Option, + pub k: Option, + pub budget: Option, + pub agent: Option, + pub source_prefix: Option, + pub pool_k: Option, + #[serde(alias = "policyMode")] + pub policy_mode: Option, +} +pub(crate) fn apply_recall_ranking_boosts(items: &mut [RecallItem], query_text: &str, entropy_mult: f64, entropy_cap: f64) { + let query_entities = query_entity_terms(query_text); + let alignment_profile = QueryAlignmentProfile::from_query(query_text); + let query_focus_term_count = alignment_profile.term_count; + for item in items { + let h = shannon_entropy(&item.excerpt); + item.entropy = Some(round4(h)); + let boost = ((h - 3.5).max(0.0) * entropy_mult).min(entropy_cap); + item.relevance = round4(item.relevance * (1.0 + boost)); + if !query_entities.is_empty() { + let haystack = format!("{} {}", item.source, item.excerpt); + let (entity_matches, entity_overlap) = entity_alignment_metrics_with_terms(&haystack, &query_entities); + let entity_boost = entity_signal_boost(entity_matches, entity_overlap); + if entity_boost > 0.0 { + item.relevance = round4(item.relevance * (1.0 + entity_boost)); + } + } + let alignment_boost = query_alignment_boost_with_profile(&item.source, &item.excerpt, &alignment_profile, query_focus_term_count); + if alignment_boost > 0.0 { + item.relevance = round4(item.relevance * (1.0 + alignment_boost)); + } + } +} +pub(crate) fn normalize_text(input: &str) -> String { + input + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() || ch == '-' || ch.is_ascii_whitespace() { ch.to_ascii_lowercase() } else { ' ' }) + .collect() +} +pub(crate) fn extract_keywords(text: &str) -> Vec { + let stop_words: HashSet<&'static str> = [ + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", + "may", "might", "shall", "can", "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into", "about", "that", "this", "it", "its", "not", + "but", "and", "or", "if", "then", "so", "what", "which", "who", "how", "when", "where", "why", "all", "each", "every", "both", "few", "more", "most", + "some", "any", "no", "my", "your", "his", "her", "our", "their", "i", "me", + ] + .into_iter() + .collect(); + normalize_text(text) + .split_whitespace() + .filter(|word| word.len() > 2 && !stop_words.contains(*word)) + .map(str::to_string) + .collect() +} +pub(crate) fn extract_search_keywords(text: &str) -> Vec { + normalize_text(text).split_whitespace().filter(|word| word.len() > 1).map(str::to_string).collect() +} +pub(crate) fn coding_synonyms(word: &str) -> Option<&'static str> { + match word { + "func" => Some("function"), + "fn" => Some("function"), + "err" => Some("error"), + "db" => Some("database"), + "auth" => Some("authentication"), + "authn" => Some("authentication"), + "authz" => Some("authorization"), + "cfg" => Some("config"), + "config" => Some("configuration"), + "msg" => Some("message"), + "req" => Some("request"), + "res" => Some("response"), + "resp" => Some("response"), + "impl" => Some("implementation"), + "repo" => Some("repository"), + "env" => Some("environment"), + "var" => Some("variable"), + "arg" => Some("argument"), + "args" => Some("arguments"), + "param" => Some("parameter"), + "params" => Some("parameters"), + "dir" => Some("directory"), + "tmp" => Some("temporary"), + "async" => Some("asynchronous"), + "sync" => Some("synchronous"), + "tx" => Some("transaction"), + "rx" => Some("receive"), + "conn" => Some("connection"), + "stmt" => Some("statement"), + "idx" => Some("index"), + "str" => Some("string"), + "int" => Some("integer"), + "bool" => Some("boolean"), + "vec" => Some("vector"), + "dict" => Some("dictionary"), + "obj" => Some("object"), + "num" => Some("number"), + "char" => Some("character"), + "lastname" => Some("surname"), + "surname" => Some("lastname"), + "attend" => Some("attended"), + "attended" => Some("attend"), + "abroad" => Some("overseas"), + "overseas" => Some("abroad"), + "coupon" => Some("voucher"), + "voucher" => Some("coupon"), + "gift" => Some("present"), + "present" => Some("gift"), + "buy" => Some("bought"), + "bought" => Some("buy"), + "repaint" => Some("paint"), + "repainted" => Some("paint"), + "painted" => Some("paint"), + "walls" => Some("wall"), + "wall" => Some("walls"), + "colour" => Some("color"), + "color" => Some("colour"), + "gray" => Some("grey"), + "grey" => Some("gray"), + _ => None, + } +} +pub(crate) fn is_low_signal_query_token(token: &str) -> bool { + matches!( + token, + "the" + | "a" + | "an" + | "is" + | "are" + | "was" + | "were" + | "be" + | "been" + | "being" + | "do" + | "does" + | "did" + | "to" + | "of" + | "in" + | "for" + | "on" + | "with" + | "at" + | "by" + | "from" + | "as" + | "into" + | "about" + | "that" + | "this" + | "it" + | "its" + | "my" + | "your" + | "our" + | "their" + | "i" + | "me" + | "we" + | "you" + | "what" + | "which" + | "who" + | "how" + | "when" + | "where" + | "why" + ) +} +pub(crate) fn query_intent_alias_terms(text: &str) -> Vec { + let lower = normalize_text(text); + let mut aliases = Vec::new(); + if lower.contains("study abroad") { + aliases.extend(["attend", "attended", "exchange", "semester"].into_iter().map(str::to_string)); + } + if lower.contains("coupon") && lower.contains("creamer") { + aliases.extend(["redeem", "redeemed", "store", "grocery"].into_iter().map(str::to_string)); + } + if lower.contains("birthday") && (lower.contains("gift") || lower.contains("present")) { + aliases.extend(["buy", "bought", "item", "present"].into_iter().map(str::to_string)); + } + aliases +} +pub(crate) fn build_search_term_groups(text: &str) -> Vec> { + let mut base = extract_search_keywords(text); + let profile = query_shape_profile(text, None); + if profile.naturalish && base.len() >= 6 { + let filtered = base.iter().filter(|token| !is_low_signal_query_token(token.as_str())).cloned().collect::>(); + if !filtered.is_empty() { + base = filtered; + } + } + let mut seen_base = HashSet::new(); + for alias in query_intent_alias_terms(text) { + if seen_base.insert(alias.clone()) && !base.iter().any(|token| token == &alias) { + base.push(alias); + } + } + let mut groups = Vec::with_capacity(base.len()); + for word in base { + let mut group = Vec::with_capacity(2); + let mut seen = HashSet::new(); + if let Some(expanded) = coding_synonyms(&word) { + let expanded = expanded.to_string(); + if seen.insert(expanded.clone()) { + group.push(expanded); + } + } + if seen.insert(word.clone()) { + group.push(word); + } + if !group.is_empty() { + groups.push(group); + } + } + groups +} +pub(crate) fn count_matching_term_groups(haystacks: &[String], term_groups: &[Vec]) -> i64 { + term_groups + .iter() + .filter(|group| group.iter().any(|term| haystacks.iter().any(|haystack| haystack.contains(term)))) + .count() as i64 +} +pub(crate) fn query_focus_terms(query_text: &str) -> Vec { + let mut terms = extract_keywords(query_text); + let mut seen: HashSet = terms.iter().cloned().collect(); + for group in build_search_term_groups(query_text) { + for term in group { + if seen.insert(term.clone()) { + terms.push(term); + } + } + } + if terms.is_empty() { + terms = extract_search_keywords(query_text); + } + terms +} +pub(crate) fn build_fts_query(groups: &[Vec]) -> String { + groups + .iter() + .map(|group| { + let alternates = group.iter().map(|t| format!("\"{}\"", t.replace('"', "\"\""))).collect::>().join(" OR "); + if group.len() > 1 { + format!("({alternates})") + } else { + alternates + } + }) + .collect::>() + .join(" AND ") +} +pub(crate) fn query_focus_terms_for_excerpt(query_text: &str) -> Vec { + let mut seen = HashSet::new(); + let mut terms = query_focus_terms(query_text) + .into_iter() + .filter_map(|term| { + let normalized = term.trim().to_ascii_lowercase(); + if normalized.is_empty() || !seen.insert(normalized.clone()) { + None + } else { + Some(normalized) + } + }) + .collect::>(); + terms.sort_by_key(|t| std::cmp::Reverse(t.len())); + terms +} +pub(crate) fn excerpt_signature_terms(source: &str, excerpt: &str) -> HashSet { + let mut terms = HashSet::new(); + for token in extract_search_keywords(source).into_iter().chain(extract_search_keywords(excerpt)) { + if token.len() > 2 { + terms.insert(token); + } + } + terms +} +pub(crate) fn term_set_jaccard(a: &HashSet, b: &HashSet) -> f64 { + if a.is_empty() && b.is_empty() { + return 1.0; + } + let intersection = a.intersection(b).count(); + let union = a.union(b).count(); + if union == 0 { + return 0.0; + } + intersection as f64 / union as f64 +} +pub(crate) fn query_term_coverage_gain(signature_terms: &HashSet, query_terms: &HashSet, covered_terms: &HashSet) -> usize { + query_terms.iter().filter(|term| signature_terms.contains(*term) && !covered_terms.contains(*term)).count() +} +pub(crate) fn should_skip_redundant_budget_candidate( + signature_terms: &HashSet, selected_signatures: &[HashSet], query_terms: &HashSet, covered_terms: &HashSet, +) -> bool { + if selected_signatures.is_empty() || signature_terms.is_empty() { + return false; + } + if query_term_coverage_gain(signature_terms, query_terms, covered_terms) > 0 { + return false; + } + let max_similarity = selected_signatures.iter().map(|existing| term_set_jaccard(existing, signature_terms)).fold(0.0_f64, f64::max); + max_similarity >= BUDGET_REDUNDANCY_SIMILARITY_THRESHOLD +} +pub(crate) fn update_query_term_coverage(signature_terms: &HashSet, query_terms: &HashSet, covered_terms: &mut HashSet) { + for term in query_terms { + if signature_terms.contains(term) { + covered_terms.insert(term.clone()); + } + } +} +pub(crate) fn should_early_stop_budget_selection( + token_budget: usize, spent_tokens: usize, selected_count: usize, query_terms: &HashSet, covered_terms: &HashSet, +) -> bool { + if token_budget == 0 || selected_count < 2 || query_terms.is_empty() { + return false; + } + if covered_terms.len() < query_terms.len() { + return false; + } + let pressure = spent_tokens as f64 / token_budget as f64; + pressure >= BUDGET_PRESSURE_EARLY_STOP_THRESHOLD +} +pub(crate) fn query_focused_excerpt_with_terms(text: &str, sorted_focus_terms: &[String], max_chars: usize) -> String { + if max_chars == 0 || text.is_empty() { + return String::new(); + } + let total_chars = text.chars().count(); + if total_chars <= max_chars { + return text.to_string(); + } + let lower_text = text.to_ascii_lowercase(); + if lower_text.contains("[assistant-question]") { + if let Some(answer_byte_idx) = lower_text.find("[user-answer]") { + let answer_char_idx = text[..answer_byte_idx].chars().count(); + let answer_end_char = (answer_char_idx + max_chars).min(total_chars); + let mut answer_excerpt = text.chars().skip(answer_char_idx).take(answer_end_char.saturating_sub(answer_char_idx)).collect::(); + if !answer_excerpt.trim().is_empty() { + if answer_char_idx > 0 { + answer_excerpt = format!("...{answer_excerpt}"); + } + if answer_end_char < total_chars { + answer_excerpt.push_str("..."); + } + return answer_excerpt; + } + } + } + if sorted_focus_terms.is_empty() { + return truncate_chars(text, max_chars); + } + let mut hit_byte_idx = None; + for term in sorted_focus_terms { + if let Some(idx) = lower_text.find(term.as_str()) { + hit_byte_idx = Some(idx); + break; + } + } + let Some(byte_idx) = hit_byte_idx else { + return truncate_chars(text, max_chars); + }; + let hit_char_idx = text[..byte_idx].chars().count(); + let left_window = max_chars / 3; + let mut start_char = hit_char_idx.saturating_sub(left_window); + let end_char = (start_char + max_chars).min(total_chars); + if end_char - start_char < max_chars { + start_char = end_char.saturating_sub(max_chars); + } + let mut excerpt = text.chars().skip(start_char).take(end_char - start_char).collect::(); + if start_char > 0 { + excerpt = format!("...{excerpt}"); + } + if end_char < total_chars { + excerpt.push_str("..."); + } + excerpt +} +pub(crate) fn query_focused_excerpt(text: &str, query_text: &str, max_chars: usize) -> String { + let terms = query_focus_terms_for_excerpt(query_text); + query_focused_excerpt_with_terms(text, &terms, max_chars) +} +pub(crate) fn recency_days(value: Option<&str>) -> i64 { + let ts = value.map(parse_timestamp_ms).unwrap_or(0); + if ts == 0 { + return 3650; + } + (Utc::now().timestamp_millis() - ts).max(0) / (24 * 60 * 60 * 1000) +} +pub(crate) fn blend_importance(score: Option, trust_score: Option) -> f64 { + let score = match score { + Some(value) if value.is_finite() => value.clamp(0.0, 1.0), + Some(_) => 0.0, + None => 1.0, + }; + let trust = match trust_score { + Some(value) if value.is_finite() => value.clamp(0.0, 1.0), + _ => score, + }; + round4((score * 0.65) + (trust * 0.35)) +} +pub(crate) fn compare_relevance_desc_source_asc(a_relevance: f64, a_source: &str, b_relevance: f64, b_source: &str) -> std::cmp::Ordering { + let a = if a_relevance.is_finite() { a_relevance } else { f64::NEG_INFINITY }; + let b = if b_relevance.is_finite() { b_relevance } else { f64::NEG_INFINITY }; + b.total_cmp(&a).then_with(|| a_source.cmp(b_source)) +} +#[derive(Clone)] +pub(crate) struct QueryAlignmentProfile { + pub(crate) lower_query: String, + pub(crate) terms: Vec, + pub(crate) term_count: usize, +} +impl QueryAlignmentProfile { + pub(crate) fn from_query(query_text: &str) -> Self { + let lower_query = query_text.trim().to_ascii_lowercase(); + let mut seen = HashSet::new(); + let mut terms = Vec::new(); + for term in query_focus_terms(query_text) { + let normalized = term.trim().to_ascii_lowercase(); + if normalized.is_empty() { + continue; + } + if seen.insert(normalized.clone()) { + terms.push(normalized); + } + } + let term_count = terms.len().max(1); + Self { lower_query, terms, term_count } + } + pub(crate) fn alignment_score(&self, text: &str) -> (usize, usize) { + if text.is_empty() || self.lower_query.is_empty() { + return (0, 0); + } + let lower_text = text.to_ascii_lowercase(); + let exact_phrase = usize::from(lower_text.contains(&self.lower_query)); + let keyword_hits = self.terms.iter().filter(|term| lower_text.contains(term.as_str())).count(); + (exact_phrase, keyword_hits) + } +} +pub(crate) fn prefer_query_focused_excerpt_with_profile(current: &str, candidate: &str, profile: &QueryAlignmentProfile) -> bool { + let current_score = profile.alignment_score(current); + let candidate_score = profile.alignment_score(candidate); + candidate_score > current_score || (candidate_score == current_score && candidate.len() < current.len()) +} +#[allow(dead_code)] +pub(crate) fn prefer_query_focused_excerpt(current: &str, candidate: &str, query_text: &str) -> bool { + let profile = QueryAlignmentProfile::from_query(query_text); + prefer_query_focused_excerpt_with_profile(current, candidate, &profile) +} +pub(crate) fn query_prefers_recency(query_text: &str) -> bool { + let lower = query_text.to_ascii_lowercase(); + ["latest", "most recent", "recent", "newest", "current", "today", "now", "up to date", "up-to-date"] + .iter() + .any(|needle| lower.contains(needle)) +} +pub(crate) fn temporal_intent_multiplier(ts_ms: i64) -> f64 { + if ts_ms <= 0 { + return 1.0 - (TEMPORAL_INTENT_MULTIPLIER_RANGE * 0.25); + } + let age_days = ((Utc::now().timestamp_millis() - ts_ms).max(0) as f64) / (1000.0 * 60.0 * 60.0 * 24.0); + let freshness = (1.0 / (1.0 + age_days / 14.0)).clamp(0.0, 1.0); + 1.0 + ((freshness - 0.5) * TEMPORAL_INTENT_MULTIPLIER_RANGE) +} +pub(crate) fn query_alignment_boost_with_profile(source: &str, excerpt: &str, profile: &QueryAlignmentProfile, query_focus_term_count: usize) -> f64 { + if profile.lower_query.is_empty() { + return 0.0; + } + let lower_source = source.to_ascii_lowercase(); + let lower_excerpt = excerpt.to_ascii_lowercase(); + let exact_phrase = usize::from(lower_source.contains(&profile.lower_query) || lower_excerpt.contains(&profile.lower_query)); + let keyword_hits = profile + .terms + .iter() + .filter(|term| lower_source.contains(term.as_str()) || lower_excerpt.contains(term.as_str())) + .count(); + if exact_phrase == 0 && keyword_hits == 0 { + return 0.0; + } + let term_count = query_focus_term_count.max(1) as f64; + let coverage = (keyword_hits as f64 / term_count).clamp(0.0, 1.0); + let exact_bonus = if exact_phrase > 0 { ALIGNMENT_EXACT_BONUS_MAX } else { 0.0 }; + let coverage_bonus = (coverage * ALIGNMENT_COVERAGE_BONUS_MAX).min(ALIGNMENT_COVERAGE_BONUS_MAX); + (exact_bonus + coverage_bonus).min(ALIGNMENT_BOOST_MAX) +} +pub(crate) fn is_entity_stopword(token: &str) -> bool { + matches!( + token, + "the" + | "a" + | "an" + | "and" + | "or" + | "for" + | "with" + | "from" + | "into" + | "this" + | "that" + | "these" + | "those" + | "what" + | "which" + | "when" + | "where" + | "why" + | "how" + | "about" + | "around" + | "there" + | "their" + | "your" + | "our" + | "have" + | "has" + | "had" + | "will" + | "would" + | "could" + | "should" + ) +} +pub(crate) fn is_short_technical_term(token: &str) -> bool { + matches!( + token, + "ai" | "ml" + | "db" + | "sql" + | "api" + | "jwt" + | "uid" + | "uuid" + | "id" + | "ip" + | "dns" + | "tls" + | "ssh" + | "http" + | "https" + | "url" + | "ui" + | "ux" + | "cpu" + | "gpu" + | "ram" + | "ios" + | "sdk" + ) +} +pub(crate) fn extract_entity_like_terms(text: &str) -> HashSet { + let mut terms = HashSet::new(); + for raw in text.split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.' | '/' | ':'))) { + let token = raw.trim_matches(|c: char| !c.is_ascii_alphanumeric()); + if token.len() < 3 { + continue; + } + let lowered = token.to_ascii_lowercase(); + if is_entity_stopword(&lowered) { + continue; + } + let has_uppercase = token.chars().any(|c| c.is_ascii_uppercase()); + let has_digit = token.chars().any(|c| c.is_ascii_digit()); + let has_symbol = token.chars().any(|c| matches!(c, '_' | '-' | '.' | '/' | ':')); + let long_specific = lowered.len() >= 9; + if has_uppercase || has_digit || has_symbol || long_specific { + terms.insert(lowered); + } + } + terms +} +pub(crate) fn query_entity_terms(query_text: &str) -> HashSet { + let mut terms = extract_entity_like_terms(query_text); + if terms.is_empty() { + for term in query_focus_terms(query_text) { + if !is_entity_stopword(&term) && (term.len() >= 3 || is_short_technical_term(&term)) { + terms.insert(term); + } + } + } + terms +} +pub(crate) fn entity_alignment_metrics_with_terms(haystack: &str, query_entities: &HashSet) -> (usize, f64) { + if query_entities.is_empty() { + return (0, 0.0); + } + let mut haystack_terms = extract_entity_like_terms(haystack); + if haystack_terms.is_empty() { + for term in extract_search_keywords(haystack) { + if !is_entity_stopword(&term) && (term.len() >= 3 || is_short_technical_term(&term)) { + haystack_terms.insert(term); + } + } + } + if haystack_terms.is_empty() { + return (0, 0.0); + } + let matches = query_entities.iter().filter(|term| haystack_terms.contains(*term)).count(); + if matches == 0 { + return (0, 0.0); + } + let overlap = matches as f64 / query_entities.len().max(1) as f64; + (matches, overlap) +} +pub(crate) fn entity_signal_boost(matches: usize, overlap: f64) -> f64 { + if matches == 0 { + return 0.0; + } + let overlap_component = overlap.clamp(0.0, 1.0) * ENTITY_SIGNAL_OVERLAP_WEIGHT; + let match_component = matches.min(3) as f64 * ENTITY_SIGNAL_MATCH_WEIGHT; + (overlap_component + match_component).min(ENTITY_SIGNAL_MAX_BOOST) +} +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct FusionWeights { + pub(crate) keyword: f64, + pub(crate) semantic: f64, +} +pub(crate) fn adaptive_rrf_weights(query_text: &str, source_prefix: Option<&str>, semantic_available: bool) -> FusionWeights { + if !semantic_available { + return FusionWeights { keyword: 1.0, semantic: 0.0 }; + } + let profile = query_shape_profile(query_text, source_prefix); + let mut keyword = 1.0_f64; + let mut semantic = 1.0_f64; + if profile.exactish { + keyword += 0.35; + semantic -= 0.15; + } + if profile.naturalish { + semantic += 0.35; + keyword -= 0.15; + } + FusionWeights { keyword: keyword.clamp(0.35, 1.75), semantic: semantic.clamp(0.35, 1.75) } +} +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct FallbackRankingWeights { + pub(crate) keyword: f64, + pub(crate) score: f64, + pub(crate) recency: f64, + pub(crate) retrieval: f64, +} +pub(crate) fn adaptive_fallback_ranking_weights(query_text: &str, term_group_count: usize) -> FallbackRankingWeights { + let profile = query_shape_profile(query_text, None); + let mut keyword = 0.40_f64; + let mut score = 0.25_f64; + let mut recency = 0.20_f64; + let mut retrieval = 0.15_f64; + if profile.exactish && !profile.naturalish { + keyword += 0.12; + score -= 0.03; + recency -= 0.05; + retrieval -= 0.04; + } else if profile.naturalish && !profile.exactish { + keyword -= 0.08; + score += 0.05; + recency += 0.02; + retrieval += 0.01; + } + if term_group_count <= 1 { + keyword += 0.05; + score += 0.01; + recency -= 0.03; + retrieval -= 0.03; + } else if term_group_count >= 5 { + keyword -= 0.04; + score += 0.02; + recency += 0.01; + retrieval += 0.01; + } + keyword = keyword.max(0.05); + score = score.max(0.05); + recency = recency.max(0.05); + retrieval = retrieval.max(0.05); + let total = keyword + score + recency + retrieval; + FallbackRankingWeights { + keyword: keyword / total, + score: score / total, + recency: recency / total, + retrieval: retrieval / total, + } +} +pub(crate) fn fallback_ranking_score( + query_text: &str, term_group_count: usize, matched: i64, effective_score: f64, recency_days: i64, retrievals: Option, +) -> f64 { + let keyword_weight = if term_group_count == 0 { 0.0 } else { matched as f64 / term_group_count as f64 }; + let recency_weight = 1.0 / (1.0 + recency_days.max(0) as f64 / 7.0); + let retrieval_weight = (retrievals.unwrap_or(0).clamp(0, 20) as f64) / 20.0; + let score_weight = effective_score.clamp(0.0, 1.0); + let weights = adaptive_fallback_ranking_weights(query_text, term_group_count); + (keyword_weight * weights.keyword) + (score_weight * weights.score) + (recency_weight * weights.recency) + (retrieval_weight * weights.retrieval) +} +pub(crate) fn rrf_fuse_weighted(lists: &[Vec<(i64, f64)>], weights: &[f64], k: f64) -> Vec<(i64, f64)> { + let smooth_k = if k.is_finite() && k >= 0.0 { k } else { 60.0 }; + let mut fused: HashMap = HashMap::new(); + for (list_index, list) in lists.iter().enumerate() { + let weight = match weights.get(list_index).copied() { + Some(value) if value.is_finite() => value.max(0.0), + Some(_) => 0.0, + None => 1.0, + }; + if weight == 0.0 { + continue; + } + for (rank, &(id, _score)) in list.iter().enumerate() { + *fused.entry(id).or_insert(0.0) += weight / (smooth_k + rank as f64 + 1.0); + } + } + let mut result: Vec<(i64, f64)> = fused.into_iter().collect(); + result.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + result +} +pub(crate) fn days_since(created_at: &str) -> f64 { + match chrono::DateTime::parse_from_rfc3339(created_at) { + Ok(dt) => { + let now = chrono::Utc::now(); + let duration = now.signed_duration_since(dt); + duration.num_days() as f64 + (duration.num_seconds() as f64 % 86400.0) / 86400.0 + } + Err(_) => f64::MAX, + } +} +pub(crate) fn normalize(importance: f64) -> f64 { + if !importance.is_finite() { + return 0.0; + } + let clamped = importance.clamp(0.0, 100.0); + if clamped <= 1.0 { + clamped + } else { + clamped / 100.0 + } +} +pub(crate) fn compound_score(rrf: f64, importance: f64, created_at: &str) -> f64 { + let days = days_since(created_at); + let recency = (-days / 30.0).exp(); + let importance_normalized = normalize(importance); + rrf * 0.6 + importance_normalized * 0.2 + recency * 0.2 +} +fn sort_search_candidates(ranked: &mut [SearchCandidate], by_keywords: bool) { + ranked.sort_by(|a, b| { + let ord = b.relevance.partial_cmp(&a.relevance).unwrap_or(std::cmp::Ordering::Equal); + let ord = if by_keywords { ord.then(b.matched_keywords.cmp(&a.matched_keywords)) } else { ord }; + ord.then(b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)) + .then(b.ts.cmp(&a.ts)) + .then(b.alignment.cmp(&a.alignment)) + .then_with(|| a.source.cmp(&b.source)) + }); +} +#[derive(Clone, Copy)] +enum SearchTableKind { + Memories, + Decisions, +} +const ACTIVE_TEMPORAL:&str="status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) AND (valid_from IS NULL OR valid_from <= datetime('now')) AND (valid_until IS NULL OR valid_until > datetime('now'))"; +fn fts_keyword_sort(ranked: &mut [SearchCandidate]) { + ranked.sort_by(|a, b| { + b.relevance + .partial_cmp(&a.relevance) + .unwrap_or(std::cmp::Ordering::Equal) + .then(b.matched_keywords.cmp(&a.matched_keywords)) + .then(b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)) + .then(b.ts.cmp(&a.ts)) + .then_with(|| a.source.cmp(&b.source)) + }); +} +fn search_source_key(kind: SearchTableKind, id: i64, alt: Option<&str>) -> String { + match (kind, alt) { + (SearchTableKind::Memories, Some(s)) => s.to_string(), + (SearchTableKind::Memories, None) => format!("memory::{id}"), + (SearchTableKind::Decisions, Some(c)) => c.to_string(), + (SearchTableKind::Decisions, None) => format!("decision::{id}"), + } +} +fn search_table_recency( + conn: &Connection, limit: usize, source_prefix: Option<&str>, source_like: Option<&str>, kind: SearchTableKind, excerpt_focus_terms: &[String], +) -> Result, String> { + let(sql,use_aging)=match kind{SearchTableKind::Memories=>(format!("SELECT id, text, source, tags, score, trust_score, retrievals, last_accessed, created_at, compressed_text, age_tier FROM memories WHERE {ACTIVE_TEMPORAL} AND (?2 IS NULL OR COALESCE(source, 'memory::' || id) LIKE ?2) ORDER BY COALESCE(last_accessed, created_at) DESC LIMIT ?1"),true,),SearchTableKind::Decisions=>(format!("SELECT id, decision, context, score, trust_score, retrievals, last_accessed, created_at FROM decisions WHERE {ACTIVE_TEMPORAL} AND (?2 IS NULL OR COALESCE(context, 'decision::' || id) LIKE ?2) ORDER BY COALESCE(last_accessed, created_at) DESC LIMIT ?1"),false,),}; + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params![limit as i64, source_like], |row| { + let effective_score = + blend_importance(row.get::<_, Option>(if use_aging { 4 } else { 3 })?, row.get::<_, Option>(if use_aging { 5 } else { 4 })?); + let (display, source) = if use_aging { + let text: String = row.get(1)?; + let compressed: Option = row.get(9)?; + let age_tier: String = row.get::<_, Option>(10)?.unwrap_or_else(|| "fresh".to_string()); + let display = crate::aging::get_display_text(&text, &compressed, &age_tier); + let source = row.get::<_, Option>(2)?.unwrap_or_else(|| format!("memory::{}", row.get::<_, i64>(0).unwrap_or(0))); + (display, source) + } else { + let decision: String = row.get(1)?; + let source = row.get::<_, Option>(2)?.unwrap_or_else(|| format!("decision::{}", row.get::<_, i64>(0).unwrap_or(0))); + (decision, source) + }; + Ok(SearchCandidate { + source, + excerpt: query_focused_excerpt_with_terms(&display, excerpt_focus_terms, 220), + alignment: (0, 0), + relevance: round4(0.5 * effective_score), + matched_keywords: 0, + score: effective_score, + ts: parse_timestamp_ms( + &row.get::<_, Option>(if use_aging { 7 } else { 6 })? + .or(row.get::<_, Option>(if use_aging { 8 } else { 7 })?) + .unwrap_or_default(), + ), + owner_id: None, + visibility: None, + }) + }) + .map_err(|e| e.to_string())?; + Ok(rows.flatten().filter(|row| source_matches_prefix(&row.source, source_prefix)).collect()) +} +fn search_table_fts( + conn: &Connection, fts_query: &str, limit: usize, source_like: Option<&str>, source_prefix: Option<&str>, kind: SearchTableKind, + term_groups: &[Vec], excerpt_focus_terms: &[String], query_text: &str, bm25: &Bm25Weights, +) -> Result, String> { + let sql=match kind{SearchTableKind::Memories=>format!("SELECT m.id, m.text, m.source, m.tags, m.score, m.trust_score, m.retrievals, m.last_accessed, m.created_at, m.compressed_text, m.age_tier, m.owner_id, m.visibility FROM memories_fts fts JOIN memories m ON m.id = fts.rowid WHERE memories_fts MATCH ?1 AND m.{ACTIVE_TEMPORAL} AND (?6 IS NULL OR COALESCE(m.source, 'memory::' || m.id) LIKE ?6) ORDER BY bm25(memories_fts, ?3, ?4, ?5) LIMIT ?2"),SearchTableKind::Decisions=>format!("SELECT d.id, d.decision, d.context, d.score, d.trust_score, d.retrievals, d.last_accessed, d.created_at, d.compressed_text, d.age_tier, d.owner_id, d.visibility FROM decisions_fts fts JOIN decisions d ON d.id = fts.rowid WHERE decisions_fts MATCH ?1 AND d.{ACTIVE_TEMPORAL} AND (?5 IS NULL OR COALESCE(d.context, 'decision::' || d.id) LIKE ?5) ORDER BY bm25(decisions_fts, ?3, ?4) LIMIT ?2"),}; + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let mut ranked = Vec::new(); + let mut push_fts_row = |id: i64, + primary: String, + alt: Option, + tags: Option, + score: Option, + trust_score: Option, + retrievals: Option, + last_accessed: Option, + created_at: Option, + compressed_text: Option, + age_tier: Option, + row_owner_id: Option, + row_visibility: Option| { + let source_key = search_source_key(kind, id, alt.as_deref()); + if !source_matches_prefix(&source_key, source_prefix) { + return; + } + let effective_score = blend_importance(score, trust_score); + let ts = parse_timestamp_ms(last_accessed.as_deref().or(created_at.as_deref()).unwrap_or("")); + let display = crate::aging::get_display_text(&primary, &compressed_text, age_tier.as_deref().unwrap_or("fresh")); + let haystacks: Vec = match kind { + SearchTableKind::Memories => { + vec![primary.to_lowercase(), alt.as_deref().unwrap_or("").to_lowercase(), tags.as_deref().unwrap_or("").to_lowercase()] + } + SearchTableKind::Decisions => vec![primary.to_lowercase(), alt.as_deref().unwrap_or("").to_lowercase()], + }; + let matched = count_matching_term_groups(&haystacks, term_groups); + let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); + let ranking = fallback_ranking_score(query_text, term_groups.len(), matched, effective_score, recency_d, retrievals); + ranked.push(SearchCandidate { + source: source_key, + excerpt: query_focused_excerpt_with_terms(&display, excerpt_focus_terms, 280), + alignment: (0, 0), + relevance: round4(ranking), + matched_keywords: matched, + score: effective_score, + ts, + owner_id: row_owner_id, + visibility: row_visibility, + }); + }; + if matches!(kind, SearchTableKind::Memories) { + let rows = stmt + .query_map(params![fts_query, limit as i64, bm25.memories_text, bm25.memories_source, bm25.memories_tags, source_like], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, Option>(11)?, + row.get::<_, Option>(12)?, + )) + }) + .map_err(|e| e.to_string())?; + for row in rows.flatten() { + let (id, primary, alt, tags, score, trust_score, retrievals, last_accessed, created_at, compressed_text, age_tier, row_owner_id, row_visibility) = + row; + push_fts_row( + id, + primary, + alt, + tags, + score, + trust_score, + retrievals, + last_accessed, + created_at, + compressed_text, + age_tier, + row_owner_id, + row_visibility, + ); + } + } else { + let rows = stmt + .query_map(params![fts_query, limit as i64, bm25.decisions_text, bm25.decisions_context, source_like], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, Option>(11)?, + )) + }) + .map_err(|e| e.to_string())?; + for row in rows.flatten() { + let (id, primary, alt, score, trust_score, retrievals, last_accessed, created_at, compressed_text, age_tier, row_owner_id, row_visibility) = row; + push_fts_row( + id, + primary, + alt, + None, + score, + trust_score, + retrievals, + last_accessed, + created_at, + compressed_text, + age_tier, + row_owner_id, + row_visibility, + ); + } + } + fts_keyword_sort(&mut ranked); + ranked.truncate(limit); + Ok(ranked) +} +fn search_table_scan_fallback( + conn: &Connection, query_text: &str, limit: usize, source_prefix: Option<&str>, kind: SearchTableKind, term_groups: &[Vec], + excerpt_focus_terms: &[String], alignment_profile: &QueryAlignmentProfile, +) -> Result, String> { + let source_like = source_prefix.map(|prefix| format!("{prefix}%")); + let mut ranked = Vec::new(); + let sql=match kind{SearchTableKind::Memories=>format!("SELECT id, text, source, tags, score, trust_score, retrievals, last_accessed, created_at FROM memories WHERE {ACTIVE_TEMPORAL} AND (?1 IS NULL OR COALESCE(source, 'memory::' || id) LIKE ?1)"),SearchTableKind::Decisions=>format!("SELECT id, decision, context, score, trust_score, retrievals, last_accessed, created_at FROM decisions WHERE {ACTIVE_TEMPORAL} AND (?1 IS NULL OR COALESCE(context, 'decision::' || id) LIKE ?1)"),}; + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params![source_like.as_deref()], |row| { + Ok(match kind { + SearchTableKind::Memories => ( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + ), + SearchTableKind::Decisions => ( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + None, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + ), + }) + }) + .map_err(|e| e.to_string())?; + for row in rows.flatten() { + let (id, primary, alt, tags, score, trust_score, retrievals, last_accessed, created_at) = row; + let source_key = search_source_key(kind, id, alt.as_deref()); + if !source_matches_prefix(&source_key, source_prefix) { + continue; + } + let effective_score = blend_importance(score, trust_score); + let ts = parse_timestamp_ms(last_accessed.as_deref().or(created_at.as_deref()).unwrap_or("")); + if term_groups.is_empty() { + let excerpt = query_focused_excerpt_with_terms(&primary, excerpt_focus_terms, 220); + ranked.push(SearchCandidate { + source: source_key, + alignment: alignment_profile.alignment_score(&excerpt), + excerpt, + relevance: round4(0.5 * effective_score), + matched_keywords: 0, + score: effective_score, + ts, + owner_id: None, + visibility: None, + }); + continue; + } + let haystacks: Vec = match kind { + SearchTableKind::Memories => { + vec![primary.to_lowercase(), alt.as_deref().unwrap_or("").to_lowercase(), tags.as_deref().unwrap_or("").to_lowercase()] + } + SearchTableKind::Decisions => vec![primary.to_lowercase(), alt.as_deref().unwrap_or("").to_lowercase()], + }; + let matched = count_matching_term_groups(&haystacks, term_groups); + if matched == 0 { + continue; + } + let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); + let ranking = fallback_ranking_score(query_text, term_groups.len(), matched, effective_score, recency_d, retrievals); + let excerpt = query_focused_excerpt_with_terms(&primary, excerpt_focus_terms, 260); + ranked.push(SearchCandidate { + source: source_key, + alignment: alignment_profile.alignment_score(&excerpt), + excerpt, + relevance: round4(ranking), + matched_keywords: matched, + score: effective_score, + ts, + owner_id: None, + visibility: None, + }); + } + sort_search_candidates(&mut ranked, !term_groups.is_empty()); + ranked.truncate(limit); + Ok(ranked) +} +fn search_table(conn: &Connection, query_text: &str, limit: usize, source_prefix: Option<&str>, kind: SearchTableKind) -> Result, String> { + let term_groups = build_search_term_groups(query_text); + let excerpt_focus_terms = query_focus_terms_for_excerpt(query_text); + let source_like = source_prefix.map(|prefix| format!("{prefix}%")); + if term_groups.is_empty() { + return search_table_recency(conn, limit, source_prefix, source_like.as_deref(), kind, &excerpt_focus_terms); + } + let fts_result = search_table_fts( + conn, + &build_fts_query(&term_groups), + limit, + source_like.as_deref(), + source_prefix, + kind, + &term_groups, + &excerpt_focus_terms, + query_text, + bm25_weights(), + ); + match fts_result { + Ok(results) if !results.is_empty() => Ok(results), + _ => search_table_scan_fallback( + conn, + query_text, + limit, + source_prefix, + kind, + &term_groups, + &excerpt_focus_terms, + &QueryAlignmentProfile::from_query(query_text), + ), + } +} +pub(crate) fn search_memories(conn: &Connection, query_text: &str, limit: usize, source_prefix: Option<&str>) -> Result, String> { + search_table(conn, query_text, limit, source_prefix, SearchTableKind::Memories) +} +pub(crate) fn search_decisions(conn: &Connection, query_text: &str, limit: usize, source_prefix: Option<&str>) -> Result, String> { + search_table(conn, query_text, limit, source_prefix, SearchTableKind::Decisions) +} +include!("engine_semantic.rs"); +include!("engine_support.rs"); +include!("engine_execution.rs"); diff --git a/daemon-rs/src/handlers/recall/engine_execution.rs b/daemon-rs/src/handlers/recall/engine_execution.rs new file mode 100644 index 00000000..30dd049e --- /dev/null +++ b/daemon-rs/src/handlers/recall/engine_execution.rs @@ -0,0 +1,1069 @@ +pub(crate) async fn emit_recall_query_event(state: &RuntimeState, agent: &str, source_prefix: Option<&str>, payload: Value) { + if is_benchmark_recall_scope(agent, source_prefix) { + return; + } + let conn = state.db.lock().await; + if crate::handlers::log_event(&conn, "recall_query", payload, agent).is_ok() { + checkpoint_wal_best_effort(&conn); + } +} +pub(crate) fn build_method_breakdown(results: &[RecallItem]) -> Value { + let mut counts: BTreeMap = BTreeMap::new(); + for item in results { + *counts.entry(item.method.clone()).or_insert(0) += 1; + } + json!(counts) +} +pub(crate) fn method_count(methods: &Value, method: &str) -> i64 { + methods.get(method).and_then(|v| v.as_i64()).unwrap_or(0) +} +pub(crate) fn classify_recall_tier(cached: bool, mode: &str, methods: &Value) -> &'static str { + if cached { + return "cache_hit"; + } + if mode == "headlines" { + return "headlines"; + } + if mode == "semantic" { + return "semantic_only"; + } + let keyword = method_count(methods, "keyword"); + let semantic = method_count(methods, "semantic"); + let hybrid = method_count(methods, "hybrid"); + let crystal = method_count(methods, "crystal"); + let associative = method_count(methods, "associative"); + if hybrid > 0 || (keyword > 0 && semantic > 0) { + if crystal > 0 { + return "hybrid_crystal"; + } + return "hybrid_fusion"; + } + if associative > 0 && (keyword > 0 || semantic > 0 || crystal > 0) { + return "associative_blend"; + } + if keyword > 0 { + if crystal > 0 { + return "keyword_crystal"; + } + return "keyword_only"; + } + if semantic > 0 { + if crystal > 0 { + return "semantic_crystal"; + } + return "semantic_only"; + } + if crystal > 0 { + return "crystal_only"; + } + if associative > 0 { + return "associative_only"; + } + "unknown" +} +pub(crate) fn run_budget_recall( + conn: &mut Connection, query_text: &str, token_budget: usize, k: usize, ctx: &RecallContext, source_prefix: Option<&str>, +) -> Result, String> { + run_budget_recall_with_engine(conn, query_text, token_budget, k, None, ctx, source_prefix, None) +} +pub(crate) fn run_semantic_recall_with_query_vector( + conn: &Connection, query_text: &str, k: usize, query_vector: Option<&[f32]>, ctx: &RecallContext, source_prefix: Option<&str>, + _canary: Option<&SqliteVecCanaryConfig>, _sqlite_vec_shadow_enabled: bool, +) -> (Vec, Value) { + let prefers_recency = query_prefers_recency(query_text); + let semantic_candidates = query_vector + .map(|query_vec| collect_semantic_candidates(conn, query_vec, query_text, ctx, source_prefix)) + .unwrap_or_default(); + let semantic_route = json!({"mode":"baseline","reason":"sqlite_vec_shadow_removed","sampled":false,"trialPercent":0,"routeMode":"baseline"}); + let mut ranked: Vec = semantic_candidates + .into_iter() + .map(|candidate| { + let mut relevance = round4(candidate.relevance); + if prefers_recency { + relevance = round4(relevance * temporal_intent_multiplier(candidate.ts)); + } + RecallItem { + source: candidate.source, + relevance, + excerpt: candidate.excerpt, + method: "semantic".to_string(), + tokens: None, + entropy: None, + family_members: Vec::new(), + collapsed_sources: Vec::new(), + collapsed_source_scores: Vec::new(), + } + }) + .collect(); + apply_recall_ranking_boosts(&mut ranked, query_text, 0.05, 0.08); + ranked.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + ranked.truncate(k); + (ranked, semantic_route) +} +pub(crate) fn budget_rank_char_cap(token_budget: usize, rank_idx: usize, query_text: &str) -> usize { + let base = if token_budget <= 220 { + match rank_idx { + 0 => 180, + 1 => 120, + 2 => 90, + _ => 70, + } + } else if token_budget <= 400 { + match rank_idx { + 0 => 260, + 1 => 170, + 2 => 130, + _ => 95, + } + } else if token_budget <= 800 { + match rank_idx { + 0 => 320, + 1 => 210, + 2 => 160, + _ => 120, + } + } else { + match rank_idx { + 0 => 420, + 1 => 260, + 2 => 200, + _ => 150, + } + }; + let profile = query_shape_profile(query_text, None); + let adjusted = if profile.exactish && !profile.naturalish { + ((base as f64) * 1.12).round() as usize + } else if profile.naturalish && !profile.exactish { + ((base as f64) * 0.86).round() as usize + } else { + base + }; + adjusted.max(MIN_EXCERPT_CHARS) +} +pub(crate) fn semantic_budget_min_relevance(top_relevance: f64, query_text: &str) -> f64 { + if top_relevance < 0.25 { + return 0.0; + } + let profile = query_shape_profile(query_text, None); + let (scale, floor) = if profile.naturalish && !profile.exactish { + (0.64, 0.14) + } else if profile.exactish && !profile.naturalish { + (0.78, 0.20) + } else { + (0.72, 0.18) + }; + (top_relevance * scale).max(floor) +} +pub(crate) fn semantic_budget_max_items(token_budget: usize, query_text: &str, hard_cap: usize) -> usize { + let base: usize = if token_budget <= 220 { + 4 + } else if token_budget <= 400 { + 6 + } else if token_budget <= 800 { + 8 + } else { + 10 + }; + let profile = query_shape_profile(query_text, None); + let adjusted = if profile.naturalish && !profile.exactish { + base.saturating_add(1) + } else if profile.exactish && !profile.naturalish { + base.saturating_sub(1).max(3) + } else { + base + }; + adjusted.clamp(3, 12).min(hard_cap.max(1)) +} +pub(crate) fn fit_excerpt_to_remaining_budget( + source: &str, excerpt: &str, query_text: &str, char_cap: usize, remaining_tokens: usize, +) -> Option<(String, usize)> { + if remaining_tokens <= MIN_BUDGET_HEADROOM_TOKENS { + return None; + } + let source_only_tokens = estimate_tokens(source); + if source_only_tokens > remaining_tokens { + return None; + } + if excerpt.is_empty() { + return Some((String::new(), source_only_tokens)); + } + let total_chars = excerpt.chars().count(); + let min_chars = MIN_EXCERPT_CHARS.min(total_chars.max(1)); + let mut chars = char_cap.min(total_chars).max(min_chars); + loop { + let clipped = query_focused_excerpt(excerpt, query_text, chars); + let tokens = estimate_tokens(&format!("{source}{clipped}")); + if tokens <= remaining_tokens { + return Some((clipped, tokens)); + } + if chars <= min_chars { + break; + } + let next = ((chars as f64) * 0.72) as usize; + chars = next.max(min_chars).min(chars.saturating_sub(1)); + } + Some((String::new(), source_only_tokens)) +} +pub(crate) fn prefer_family_candidate(candidate: &RecallItem, current: &RecallItem, alignment_profile: &QueryAlignmentProfile) -> bool { + let relevance_delta = candidate.relevance - current.relevance; + if relevance_delta > 0.03 { + return true; + } + if relevance_delta < -0.03 { + return false; + } + let candidate_alignment = alignment_profile.alignment_score(&candidate.excerpt); + let current_alignment = alignment_profile.alignment_score(¤t.excerpt); + if candidate_alignment != current_alignment { + return candidate_alignment > current_alignment; + } + if candidate.method == "crystal" && current.method != "crystal" { + return true; + } + if candidate.method != "crystal" && current.method == "crystal" { + return false; + } + if candidate.excerpt.len() != current.excerpt.len() { + return candidate.excerpt.len() < current.excerpt.len(); + } + candidate.source < current.source +} +pub(crate) fn compact_budget_family_candidates_with_trace( + candidates: Vec, query_text: &str, token_budget: usize, +) -> (Vec, Vec, Vec) { + if token_budget > 400 || candidates.len() <= 1 { + return (candidates, Vec::new(), Vec::new()); + } + let mut family_lookup = HashMap::new(); + for item in &candidates { + if item.family_members.is_empty() { + continue; + } + for member in &item.family_members { + family_lookup.entry(member.clone()).or_insert_with(|| item.source.clone()); + } + } + if family_lookup.is_empty() { + return (candidates, Vec::new(), Vec::new()); + } + let mut compacted: HashMap = HashMap::new(); + let mut dropped = Vec::new(); + let mut dropped_by_family: HashMap> = HashMap::new(); + let alignment_profile = QueryAlignmentProfile::from_query(query_text); + for item in candidates { + let family_key = + if !item.family_members.is_empty() { item.source.clone() } else { family_lookup.get(&item.source).cloned().unwrap_or_else(|| item.source.clone()) }; + match compacted.entry(family_key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + if prefer_family_candidate(&item, entry.get(), &alignment_profile) { + let replaced = entry.insert(item); + dropped_by_family.entry(entry.key().clone()).or_default().push(replaced.source.clone()); + dropped.push(replaced); + } else { + dropped_by_family.entry(entry.key().clone()).or_default().push(item.source.clone()); + dropped.push(item); + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(item); + } + } + } + dropped.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + let mut family_compactions = Vec::new(); + for (family_key, mut dropped_sources) in dropped_by_family { + if dropped_sources.is_empty() { + continue; + } + dedup_preserve_order(&mut dropped_sources); + let Some(kept_source) = compacted.get(&family_key).map(|item| item.source.clone()) else { + continue; + }; + family_compactions.push(RecallFamilyCompaction { family_key, kept_source, dropped_sources }); + } + family_compactions.sort_by(|a, b| a.family_key.cmp(&b.family_key)); + let mut compacted_items: Vec = compacted.into_values().collect(); + compacted_items.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + (compacted_items, dropped, family_compactions) +} +pub(crate) fn compact_budget_family_candidates(candidates: Vec, query_text: &str, token_budget: usize) -> Vec { + compact_budget_family_candidates_with_trace(candidates, query_text, token_budget).0 +} +pub(crate) fn apply_semantic_budget(raw: Vec, token_budget: usize, query_text: &str) -> Vec { + if token_budget == 0 { + return raw + .into_iter() + .map(|mut item| { + item.excerpt.clear(); + item.tokens = Some(estimate_tokens(&item.source)); + item + }) + .collect(); + } + let raw = compact_budget_family_candidates(raw, query_text, token_budget); + let top_relevance = raw.first().map(|item| item.relevance).unwrap_or(0.0); + let min_relevance = semantic_budget_min_relevance(top_relevance, query_text); + let max_items = semantic_budget_max_items(token_budget, query_text, raw.len()); + let mut candidates: Vec = raw.iter().filter(|item| item.relevance >= min_relevance).take(max_items).cloned().collect(); + if candidates.is_empty() { + candidates = raw.iter().take(max_items.max(1)).cloned().collect(); + } + let query_terms: HashSet = query_focus_terms_for_excerpt(query_text).into_iter().collect(); + let mut covered_terms: HashSet = HashSet::new(); + let mut selected_signatures: Vec> = Vec::new(); + let mut spent = 0usize; + let mut budgeted = Vec::new(); + for (idx, mut item) in candidates.into_iter().enumerate() { + let remaining = token_budget.saturating_sub(spent); + if remaining <= 10 { + break; + } + let cap = budget_rank_char_cap(token_budget, idx, query_text).min((remaining as f64 * 3.6) as usize).max(MIN_EXCERPT_CHARS); + if let Some((excerpt, tokens)) = fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) { + let signature_terms = excerpt_signature_terms(&item.source, &excerpt); + if should_skip_redundant_budget_candidate(&signature_terms, &selected_signatures, &query_terms, &covered_terms) { + continue; + } + item.excerpt = excerpt; + item.tokens = Some(tokens); + spent += tokens; + update_query_term_coverage(&signature_terms, &query_terms, &mut covered_terms); + selected_signatures.push(signature_terms); + budgeted.push(item); + if should_early_stop_budget_selection(token_budget, spent, budgeted.len(), &query_terms, &covered_terms) { + break; + } + } + } + budgeted +} +pub(crate) struct RecallBudgetTrace { + pub(crate) budgeted: Vec, + pub(crate) candidate_pool: Vec, + pub(crate) pre_compaction_candidate_count: usize, + pub(crate) family_compactions: Vec, + pub(crate) retrieval_depth: usize, + pub(crate) top_relevance: f64, + pub(crate) min_relevance: f64, + pub(crate) max_items: usize, + pub(crate) semantic_route: Value, +} +#[derive(Clone)] +pub(crate) struct RecallFamilyCompaction { + pub(crate) family_key: String, + pub(crate) kept_source: String, + pub(crate) dropped_sources: Vec, +} +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_budget_recall_trace_with_query_vector( + conn: &Connection, query_text: &str, token_budget: usize, k: usize, query_vector: Option<&[f32]>, ctx: &RecallContext, source_prefix: Option<&str>, + canary: Option<&SqliteVecCanaryConfig>, sqlite_vec_shadow_enabled: bool, +) -> Result { + let retrieval_depth = if token_budget <= 220 { + (k.max(10) * 3).min(30) + } else if token_budget <= 400 { + (k.max(10) * 2).min(28) + } else { + k.max(12) + }; + let recall_trace = + run_recall_with_query_vector_trace(conn, query_text, retrieval_depth, query_vector, ctx, source_prefix, canary, sqlite_vec_shadow_enabled)?; + let raw = recall_trace.ranked; + let semantic_route = recall_trace.semantic_route; + if raw.is_empty() { + return Ok(RecallBudgetTrace { + budgeted: vec![], + candidate_pool: vec![], + pre_compaction_candidate_count: 0, + family_compactions: vec![], + retrieval_depth, + top_relevance: 0.0, + min_relevance: 0.0, + max_items: 0, + semantic_route, + }); + } + let pre_compaction_pool = raw; + let pre_compaction_candidate_count = pre_compaction_pool.len(); + let (raw, _family_compaction_dropped, family_compactions) = compact_budget_family_candidates_with_trace(pre_compaction_pool, query_text, token_budget); + let top_relevance = raw.first().map(|item| item.relevance).unwrap_or(0.0); + let min_relevance = semantic_budget_min_relevance(top_relevance, query_text); + let max_items = semantic_budget_max_items(token_budget, query_text, k.max(1)); + let mut candidates: Vec = raw.iter().filter(|item| item.relevance >= min_relevance).take(max_items).cloned().collect(); + if candidates.is_empty() { + candidates = raw.iter().take(max_items).cloned().collect(); + } + let query_terms: HashSet = query_focus_terms_for_excerpt(query_text).into_iter().collect(); + let mut covered_terms: HashSet = HashSet::new(); + let mut selected_signatures: Vec> = Vec::new(); + let mut spent = 0usize; + let mut budgeted = Vec::new(); + for (idx, item) in candidates.into_iter().enumerate() { + let remaining = token_budget.saturating_sub(spent); + if remaining <= 10 { + break; + } + let cap = budget_rank_char_cap(token_budget, idx, query_text).min((remaining as f64 * 3.6) as usize).max(MIN_EXCERPT_CHARS); + if let Some((excerpt, tokens)) = fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) { + let signature_terms = excerpt_signature_terms(&item.source, &excerpt); + if should_skip_redundant_budget_candidate(&signature_terms, &selected_signatures, &query_terms, &covered_terms) { + continue; + } + spent += tokens; + update_query_term_coverage(&signature_terms, &query_terms, &mut covered_terms); + selected_signatures.push(signature_terms); + budgeted.push(RecallItem { + source: item.source, + relevance: item.relevance, + excerpt, + method: item.method, + tokens: Some(tokens), + entropy: item.entropy, + family_members: item.family_members, + collapsed_sources: item.collapsed_sources, + collapsed_source_scores: item.collapsed_source_scores, + }); + if should_early_stop_budget_selection(token_budget, spent, budgeted.len(), &query_terms, &covered_terms) { + break; + } + } + } + Ok(RecallBudgetTrace { + budgeted, + candidate_pool: raw, + pre_compaction_candidate_count, + family_compactions, + retrieval_depth, + top_relevance, + min_relevance, + max_items, + semantic_route, + }) +} +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_budget_recall_with_engine( + conn: &mut Connection, query_text: &str, token_budget: usize, k: usize, engine: Option<&crate::embeddings::EmbeddingEngine>, ctx: &RecallContext, + source_prefix: Option<&str>, degraded_flag: Option<&std::sync::Arc>, +) -> Result, String> { + let trace = run_budget_recall_trace_with_engine(conn, query_text, token_budget, k, engine, ctx, source_prefix, degraded_flag)?; + bump_retrievals_batch(conn, &trace.budgeted); + Ok(trace.budgeted) +} +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_budget_recall_trace_with_engine( + conn: &Connection, query_text: &str, token_budget: usize, k: usize, engine: Option<&crate::embeddings::EmbeddingEngine>, ctx: &RecallContext, + source_prefix: Option<&str>, degraded_flag: Option<&std::sync::Arc>, +) -> Result { + let query_vector = engine.and_then(|engine| engine.embed_query(query_text)); + if engine.is_some() { + update_semantic_search_health(degraded_flag, query_vector.is_some(), true); + } + run_budget_recall_trace_with_query_vector(conn, query_text, token_budget, k, query_vector.as_deref(), ctx, source_prefix, None, true) +} +pub(crate) fn update_semantic_search_health( + degraded_flag: Option<&std::sync::Arc>, semantic_available: bool, log_unavailable: bool, +) { + if let Some(flag) = degraded_flag { + if semantic_available { + flag.store(false, std::sync::atomic::Ordering::Relaxed); + return; + } + let transitioned = flag.compare_exchange(false, true, std::sync::atomic::Ordering::Relaxed, std::sync::atomic::Ordering::Relaxed).is_ok(); + if log_unavailable && transitioned { + eprintln!("[recall] Semantic search unavailable, using keyword fallback"); + } + } +} +pub(crate) fn run_recall( + conn: &mut Connection, query_text: &str, k: usize, ctx: &RecallContext, source_prefix: Option<&str>, +) -> Result, String> { + run_recall_with_engine(conn, query_text, k, None, ctx, source_prefix, None) +} +#[allow(clippy::type_complexity)] +pub(crate) fn run_recall_with_engine( + conn: &mut Connection, query_text: &str, k: usize, engine: Option<&crate::embeddings::EmbeddingEngine>, ctx: &RecallContext, source_prefix: Option<&str>, + degraded_flag: Option<&std::sync::Arc>, +) -> Result, String> { + let query_vector = engine.and_then(|engine| engine.embed_query(query_text)); + if engine.is_some() { + update_semantic_search_health(degraded_flag, query_vector.is_some(), true); + } + let trace = run_recall_with_query_vector_trace(conn, query_text, k, query_vector.as_deref(), ctx, source_prefix, None, true)?; + bump_retrievals_batch(conn, &trace.ranked); + Ok(trace.ranked) +} +pub async fn execute_unified_recall( + state: &RuntimeState, query_text: &str, budget: usize, k: usize, agent: &str, ctx: &RecallContext, source_prefix: Option<&str>, +) -> Result { + let started_at = Instant::now(); + let policy_mode = recall_mode_for_budget(budget); + let latency_budget_ms = recall_latency_budget_ms_for_mode(policy_mode); + + let engine = state.embedding_engine.clone(); + let dflag = Some(&state.degraded_mode); + let query_vector = match engine { + Some(runtime_engine) => runtime_engine.embed_query_async(query_text.to_string()).await, + None => None, + }; + if state.embedding_engine.is_some() { + update_semantic_search_health(dflag, query_vector.is_some(), true); + } + + let (mut results, semantic_route, fail_closed) = { + let conn = state.db_read.lock().await; + let (mut results, mut semantic_route) = if budget == 0 { + let trace = + run_recall_with_query_vector_trace(&conn, query_text, k, query_vector.as_deref(), ctx, source_prefix, Some(&state.sqlite_vec_canary), false)?; + (trace.ranked, trace.semantic_route) + } else { + let trace = run_budget_recall_trace_with_query_vector( + &conn, + query_text, + budget, + k, + query_vector.as_deref(), + ctx, + source_prefix, + Some(&state.sqlite_vec_canary), + false, + )?; + (trace.budgeted, trace.semantic_route) + }; + + let mut fail_closed = Value::Null; + if budget > 0 { + let elapsed_before_fallback = started_at.elapsed().as_millis(); + if elapsed_before_fallback >= latency_budget_ms { + let fallback_trace = + run_budget_recall_trace_with_query_vector(&conn, query_text, budget, k, None, ctx, source_prefix, Some(&state.sqlite_vec_canary), false)?; + results = fallback_trace.budgeted; + semantic_route = json!({ + "mode": "baseline", + "reason": "latency_budget_fail_closed", + "fallback": "deterministic_keyword_rrf", + "elapsedMsBeforeFallback": elapsed_before_fallback, + "latencyBudgetMs": latency_budget_ms, + "routeMode": state.sqlite_vec_canary.effective_route_mode().as_str() + }); + fail_closed = json!({ + "triggered": true, + "elapsedMsBeforeFallback": elapsed_before_fallback, + "latencyBudgetMs": latency_budget_ms, + "fallback": "deterministic_keyword_rrf" + }); + } + } + (results, semantic_route, fail_closed) + }; + + let shadow_semantic = json!({"status": "skipped", "reason": "shadow_removed"}); + let (reranked_results, rerank_route) = maybe_apply_rerank(state, results, budget); + results = reranked_results; + + { + let conn = state.db.lock().await; + bump_retrievals_batch(&conn, &results); + } + + if budget == 0 { + let method_breakdown = build_method_breakdown(&results); + let tier = classify_recall_tier(false, "headlines", &method_breakdown); + let latency_ms = started_at.elapsed().as_millis() as i64; + let headlines = results + .iter() + .map(|item| json!({"source": item.source, "relevance": item.relevance, "method": item.method})) + .collect::>(); + let usage = compute_headlines_token_usage(&results); + emit_recall_query_event( + state, + agent, + source_prefix, + json!({ + "agent": agent, + "query": truncate_chars(query_text, 120), + "budget": 0, + "spent": usage.spent, + "saved": usage.saved, + "hits": headlines.len(), + "mode": "headlines", + "cached": false, + "method_breakdown": method_breakdown, + "tier": tier, + "latency_ms": latency_ms, + "latency_budget_ms": latency_budget_ms, + "semantic_route": semantic_route.clone(), + "shadow_semantic": shadow_semantic, + "fail_closed": fail_closed, + "rerank": rerank_route.clone() + }), + ) + .await; + return Ok(json!({ + "count": headlines.len(), + "results": headlines, + "budget": 0, + "spent": usage.spent, + "saved": usage.saved, + "overBudget": usage.over_budget, + "tokenUsageLine": format_recall_token_usage_line(0, usage), + "mode": "headlines", + "policyMode": RecallPolicyMode::Headlines.as_str(), + "tier": tier, + "latencyMs": latency_ms, + "latencyBudgetMs": latency_budget_ms, + "failClosed": fail_closed, + "semanticRoute": semantic_route.clone(), + "rerankRoute": rerank_route + })); + } + + let results = dedup_and_mark_served(state, agent, query_text, ctx, results).await; + let results = enforce_budget_token_invariant(results, budget, query_text); + let usage = compute_recall_budget_usage(&results, budget); + let mode = recall_mode_for_budget(budget); + let method_breakdown = build_method_breakdown(&results); + let tier = classify_recall_tier(false, mode.as_str(), &method_breakdown); + let latency_ms = started_at.elapsed().as_millis() as i64; + emit_recall_query_event( + state, + agent, + source_prefix, + json!({ + "agent": agent, + "query": truncate_chars(query_text, 120), + "budget": budget, + "spent": usage.spent, + "saved": usage.saved, + "over_budget": usage.over_budget, + "hits": results.len(), + "mode": mode.as_str(), + "cached": false, + "method_breakdown": method_breakdown, + "tier": tier, + "latency_ms": latency_ms, + "latency_budget_ms": latency_budget_ms, + "semantic_route": semantic_route.clone(), + "shadow_semantic": shadow_semantic, + "fail_closed": fail_closed, + "rerank": rerank_route.clone() + }), + ) + .await; + Ok(json!({ + "results": results.into_iter().map(recall_to_json).collect::>(), + "budget": budget, + "spent": usage.spent, + "saved": usage.saved, + "overBudget": usage.over_budget, + "tokenUsageLine": format_recall_token_usage_line(budget, usage), + "mode": mode.as_str(), + "policyMode": mode.as_str(), + "tier": tier, + "latencyMs": latency_ms, + "latencyBudgetMs": latency_budget_ms, + "failClosed": fail_closed, + "semanticRoute": semantic_route, + "rerankRoute": rerank_route + })) +} +#[allow(clippy::too_many_arguments)] +pub async fn execute_recall_policy_explain( + state: &RuntimeState, query_text: &str, budget: usize, k: usize, agent: &str, ctx: &RecallContext, source_prefix: Option<&str>, pool_k: usize, + query_vector_override: Option<&[f32]>, +) -> Result { + let requested_k = k.max(1); + let pool_k = pool_k.max(requested_k).min(128); + let engine = state.embedding_engine.clone(); + let dflag = Some(&state.degraded_mode); + let query_vector = match query_vector_override { + Some(vector) => Some(vector.to_vec()), + None => match engine { + Some(runtime_engine) => runtime_engine.embed_query_async(query_text.to_string()).await, + None => None, + }, + }; + if query_vector_override.is_none() && state.embedding_engine.is_some() { + update_semantic_search_health(dflag, query_vector.is_some(), true); + } + let conn = state.db_read.lock().await; + let ( + budgeted, + candidate_pool, + pre_compaction_candidate_count, + family_compactions, + retrieval_depth, + min_relevance, + top_relevance, + max_items, + semantic_route, + ) = if budget == 0 { + let trace = + run_recall_with_query_vector_trace(&conn, query_text, pool_k, query_vector.as_deref(), ctx, source_prefix, Some(&state.sqlite_vec_canary), true)?; + let raw_pool = trace.ranked; + let budgeted = raw_pool + .iter() + .take(requested_k) + .cloned() + .map(|mut item| { + item.excerpt.clear(); + item.tokens = Some(estimate_tokens(&item.source)); + item + }) + .collect::>(); + let raw_pool_len = raw_pool.len(); + (budgeted, raw_pool, raw_pool_len, Vec::new(), pool_k, 0.0_f64, 0.0_f64, requested_k, trace.semantic_route) + } else { + let trace = run_budget_recall_trace_with_query_vector( + &conn, + query_text, + budget, + requested_k, + query_vector.as_deref(), + ctx, + source_prefix, + Some(&state.sqlite_vec_canary), + true, + )?; + ( + trace.budgeted, + trace.candidate_pool, + trace.pre_compaction_candidate_count, + trace.family_compactions, + trace.retrieval_depth, + trace.min_relevance, + trace.top_relevance, + trace.max_items, + trace.semantic_route, + ) + }; + let shadow_semantic = json!({"enabled":false,"status":"skipped","reason":"shadow_removed","topK":pool_k}); + drop(conn); + let (budgeted, rerank_route) = maybe_apply_rerank(state, budgeted, budget); + let final_results = dedup_and_mark_served(state, agent, query_text, ctx, budgeted).await; + let final_results = enforce_budget_token_invariant(final_results, budget, query_text); + let usage = compute_recall_budget_usage(&final_results, budget); + let mode = recall_mode_for_budget(budget); + let family_compacted_count: usize = family_compactions.iter().map(|entry| entry.dropped_sources.len()).sum(); + let family_compactions_json: Vec = family_compactions + .iter() + .map(|entry| json!({"familyKey":entry.family_key,"keptSource":entry.kept_source,"droppedSources":entry.dropped_sources,})) + .collect(); + let returned_sources: HashSet<&str> = final_results.iter().map(|item| item.source.as_str()).collect(); + let dropped_candidates:Vec=candidate_pool.iter().filter(|item|!returned_sources.contains(item.source.as_str())).take(24).map(|item|{let estimated_tokens=estimate_tokens(&format!("{}{}",item.source,item.excerpt));json!({"source":item.source,"relevance":item.relevance,"method":item.method,"estimatedTokens":estimated_tokens,"reason":"not_selected_under_current_budget_or_rank_cutoff"})}).collect(); + let query_entities = query_entity_terms(query_text); + let mut entity_metrics_by_source: HashMap = HashMap::new(); + for candidate in &candidate_pool { + let haystack = format!("{} {}", candidate.source, candidate.excerpt); + let (entity_matches, entity_overlap) = entity_alignment_metrics_with_terms(&haystack, &query_entities); + let entity_boost = entity_signal_boost(entity_matches, entity_overlap); + entity_metrics_by_source.insert(candidate.source.clone(), (entity_matches, round4(entity_overlap), round4(entity_boost))); + } + let final_with_factors:Vec=final_results.clone().into_iter().enumerate().map(|(idx,item)|{let tokens=item.tokens.unwrap_or_else(||estimate_tokens(&format!("{}{}",item.source,item.excerpt)));let budget_ratio=if budget==0{0.0}else{((tokens as f64)/(budget as f64)).min(1.0)};let(entity_matches,entity_overlap,entity_boost)=entity_metrics_by_source.get(&item.source).copied().unwrap_or_else(||{let haystack=format!("{} {}",item.source,item.excerpt);let(matches,overlap)=entity_alignment_metrics_with_terms(&haystack,&query_entities);(matches,round4(overlap),round4(entity_signal_boost(matches,overlap)),)});json!({"rank":idx+1,"source":item.source,"relevance":item.relevance,"method":item.method,"tokens":tokens,"rankingFactors":{"relevance":item.relevance,"method":item.method,"tokenCost":tokens,"budgetCostRatio":round4(budget_ratio),"entropy":item.entropy,"entityMatches":entity_matches,"entityOverlap":entity_overlap,"entityBoost":entity_boost}})}).collect(); + let post_compaction_dropped_count = candidate_pool.len().saturating_sub(final_with_factors.len()); + Ok( + json!({"query":query_text,"results":final_results.into_iter().map(recall_to_json).collect::>(),"budget":budget,"spent":usage.spent,"saved":usage.saved,"overBudget":usage.over_budget,"tokenUsageLine":format_recall_token_usage_line(budget,usage),"mode":mode.as_str(),"policyMode":mode.as_str(),"policy":{"name":"adaptive-recall-policy","mode":mode.as_str(),"budget":budget,"requestedK":requested_k,"poolK":pool_k,"retrievalDepth":retrieval_depth,"candidateCutoff":{"topRelevance":round4(top_relevance),"minRelevance":round4(min_relevance),"maxItemsBeforeBudget":max_items},"budgetReasoning":{"requestedBudget":budget,"spent":usage.spent,"saved":usage.saved,"budgetPressure":if budget==0{0.0}else{round4((usage.spent as f64)/(budget as f64))},"candidateCountBeforeFamilyCompaction":pre_compaction_candidate_count,"candidateCount":candidate_pool.len(),"candidateCountAfterFamilyCompaction":candidate_pool.len(),"familyCompactedCount":family_compacted_count,"returnedCount":final_with_factors.len(),"droppedCount":post_compaction_dropped_count,"totalPreBudgetDrops":family_compacted_count+post_compaction_dropped_count},"semanticRoute":semantic_route,"rerankRoute":rerank_route.clone()},"explain":{"returned":final_with_factors,"familyCompactions":family_compactions_json,"droppedCandidates":dropped_candidates,"shadowSemantic":shadow_semantic,"rerank":rerank_route}}), + ) +} +pub async fn execute_semantic_recall( + state: &RuntimeState, query_text: &str, budget: usize, k: usize, agent: &str, ctx: &RecallContext, source_prefix: Option<&str>, +) -> Result { + let started_at = Instant::now(); + let query_vector = match state.embedding_engine.clone() { + Some(engine) => engine.embed_query_async(query_text.to_string()).await, + None => None, + }; + let semantic_available = query_vector.is_some(); + let (budgeted, semantic_route) = { + let conn = state.db_read.lock().await; + let (results, semantic_route) = + run_semantic_recall_with_query_vector(&conn, query_text, k, query_vector.as_deref(), ctx, source_prefix, Some(&state.sqlite_vec_canary), false); + (apply_semantic_budget(results, budget, query_text), semantic_route) + }; + { + let conn = state.db.lock().await; + bump_retrievals_batch(&conn, &budgeted); + } + let budgeted = enforce_budget_token_invariant(budgeted, budget, query_text); + let usage = compute_recall_budget_usage(&budgeted, budget); + let mode = "semantic"; + let method_breakdown = build_method_breakdown(&budgeted); + let tier = classify_recall_tier(false, mode, &method_breakdown); + let latency_ms = started_at.elapsed().as_millis() as i64; + emit_recall_query_event(state,agent,source_prefix,json!({"agent":agent,"query":truncate_chars(query_text,120),"mode":mode,"k":k,"budget":budget,"spent":usage.spent,"saved":usage.saved,"over_budget":usage.over_budget,"hits":budgeted.len(),"results":budgeted.len(),"semantic_available":semantic_available,"cached":false,"method_breakdown":method_breakdown,"tier":tier,"latency_ms":latency_ms,"semantic_route":semantic_route.clone(),}),).await; + Ok( + json!({"results":budgeted.into_iter().map(recall_to_json).collect::>(),"mode":"semantic","budget":budget,"spent":usage.spent,"saved":usage.saved,"overBudget":usage.over_budget,"tokenUsageLine":format_recall_token_usage_line(budget,usage),"semanticAvailable":semantic_available,"semanticRoute":semantic_route,"tier":tier,"latencyMs":latency_ms,}), + ) +} +#[allow(clippy::type_complexity)] +pub(crate) fn run_recall_with_query_vector_trace( + conn: &Connection, query_text: &str, k: usize, query_vector: Option<&[f32]>, ctx: &RecallContext, source_prefix: Option<&str>, + canary: Option<&SqliteVecCanaryConfig>, sqlite_vec_shadow_enabled: bool, +) -> Result { + let extracted = extract_search_keywords(query_text); + let prefers_recency = query_prefers_recency(query_text); + let alignment_profile = QueryAlignmentProfile::from_query(query_text); + let keyword_query = if extracted.is_empty() { query_text.to_string() } else { extracted.join(" ") }; + let mut crystal_items: HashMap = HashMap::new(); + let mut crystal_family_lookup: HashMap = HashMap::new(); + if let Some(query_vec) = query_vector { + for (crystal_id, label, text, relevance) in crate::crystallize::search_crystals_filtered(conn, query_vec, 3, ctx.caller_id, ctx.team_mode) { + let source = crystal_source(crystal_id, &label); + if !source_matches_prefix(&source, source_prefix) { + continue; + } + let family_members = crystal_member_sources(conn, crystal_id, ctx); + for member_source in &family_members { + crystal_family_lookup.entry(member_source.clone()).or_insert_with(|| source.clone()); + } + crystal_items.insert( + source.clone(), + RecallItem { + source, + relevance: scale_semantic_similarity(relevance as f32), + excerpt: query_focused_excerpt(&text, query_text, 300), + method: "crystal".to_string(), + tokens: None, + entropy: None, + family_members, + collapsed_sources: Vec::new(), + collapsed_source_scores: Vec::new(), + }, + ); + } + } + const TIER2_CONFIDENCE: f64 = 0.78; + const TIER2_GAP: f64 = 0.10; + let raw_k = if ctx.team_mode { k.max(10) * 5 } else { 20 }; + let mut fts_limit = raw_k.max(20); + let kw_candidates: Vec = { + let mut retry = 0; + let mut all: Vec = Vec::new(); + loop { + all.clear(); + for row in search_memories(conn, &keyword_query, fts_limit, source_prefix)? + .into_iter() + .filter(|r| is_visible(r.owner_id, r.visibility.as_deref(), ctx)) + { + all.push(row); + } + for row in search_decisions(conn, &keyword_query, fts_limit, source_prefix)? + .into_iter() + .filter(|r| is_visible(r.owner_id, r.visibility.as_deref(), ctx)) + { + all.push(row); + } + all.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + if ctx.team_mode && all.len() < k && retry < 2 { + fts_limit *= 2; + retry += 1; + continue; + } + break; + } + all + }; + let required_keyword_hits = if extracted.is_empty() { 1_i64 } else { ((extracted.len() as f64) * 0.6).ceil() as i64 }; + let tier2_resolved = if let Some(top) = kw_candidates.first() { + let gap = kw_candidates.get(1).map(|next| top.relevance - next.relevance).unwrap_or(top.relevance); + top.relevance >= TIER2_CONFIDENCE && top.matched_keywords >= required_keyword_hits && gap >= TIER2_GAP + } else { + false + }; + let (semantic_candidates, semantic_route) = if tier2_resolved { + ( + Vec::new(), + json!({"mode":"baseline","reason":"tier2_keyword_resolved","sampled":false,"trialPercent":canary.map(|config|config.trial_percent).unwrap_or(0),"routeMode":canary.map(|config|config.effective_route_mode().as_str()).unwrap_or("baseline")}), + ) + } else { + let baseline_semantic = query_vector + .map(|query_vec| collect_semantic_candidates(conn, query_vec, query_text, ctx, source_prefix)) + .unwrap_or_default(); + let semantic_route = json!({"mode":"baseline","reason":if sqlite_vec_shadow_enabled{"shadow_removed"}else{"hot_path_shadow_skipped"},"sampled":false,"trialPercent":canary.map(|config|config.trial_percent).unwrap_or(0),"routeMode":canary.map(|config|config.effective_route_mode().as_str()).unwrap_or("baseline")}); + (baseline_semantic, semantic_route) + }; + let mut source_index: HashMap = HashMap::new(); + let mut index_source: Vec = Vec::new(); + let mut get_idx = |source: &str| -> i64 { + if let Some(&idx) = source_index.get(source) { + return idx; + } + let idx = index_source.len() as i64; + source_index.insert(source.to_string(), idx); + index_source.push(source.to_string()); + idx + }; + let kw_list: Vec<(i64, f64)> = kw_candidates.iter().map(|c| (get_idx(&c.source), c.relevance)).collect(); + let sem_list: Vec<(i64, f64)> = semantic_candidates.iter().map(|candidate| (get_idx(&candidate.source), candidate.relevance)).collect(); + let fusion_weights = adaptive_rrf_weights(query_text, source_prefix, !semantic_candidates.is_empty()); + let fused = rrf_fuse_weighted(&[kw_list, sem_list], &[fusion_weights.keyword, fusion_weights.semantic], 60.0); + let kw_by_source: HashMap<&str, &SearchCandidate> = kw_candidates.iter().map(|candidate| (candidate.source.as_str(), candidate)).collect(); + let sem_by_source: HashMap<&str, &SemanticCandidate> = semantic_candidates.iter().map(|candidate| (candidate.source.as_str(), candidate)).collect(); + let mut merged: HashMap = HashMap::new(); + for (idx, rrf_score) in &fused { + let source = match index_source.get(*idx as usize) { + Some(s) => s.clone(), + None => continue, + }; + let source_key = source.as_str(); + let (excerpt, importance, ts_ms, method) = if let Some(kw) = kw_by_source.get(source_key) { + let method = if sem_by_source.contains_key(source_key) { "hybrid" } else { "keyword" }; + (kw.excerpt.clone(), kw.score, kw.ts, method) + } else if let Some(sem) = sem_by_source.get(source_key) { + (sem.excerpt.clone(), sem.importance, sem.ts, "semantic") + } else { + continue; + }; + let created_at_str = if ts_ms > 0 { Utc.timestamp_millis_opt(ts_ms).single().map(|dt| dt.to_rfc3339()).unwrap_or_default() } else { String::new() }; + let mut relevance = round4(compound_score(*rrf_score, importance * 100.0, &created_at_str)); + if prefers_recency { + relevance = round4(relevance * temporal_intent_multiplier(ts_ms)); + } + if let Some(crystal_source) = crystal_family_lookup.get(&source) { + if let Some(crystal_item) = crystal_items.get_mut(crystal_source) { + crystal_item.relevance = round4(crystal_item.relevance.max(relevance)); + if !crystal_item.collapsed_sources.iter().any(|collapsed| collapsed == &source) { + crystal_item.collapsed_sources.push(source.clone()); + } + crystal_item.collapsed_source_scores.push((source.clone(), relevance)); + if prefer_query_focused_excerpt_with_profile(&crystal_item.excerpt, &excerpt, &alignment_profile) { + crystal_item.excerpt = excerpt.clone(); + } + } + continue; + } + merged.insert( + source.clone(), + RecallItem { + source, + relevance, + excerpt, + method: method.to_string(), + tokens: None, + entropy: None, + family_members: Vec::new(), + collapsed_sources: Vec::new(), + collapsed_source_scores: Vec::new(), + }, + ); + } + for (src, mut item) in crystal_items { + dedup_preserve_order(&mut item.family_members); + normalize_collapsed_source_rank(&mut item); + merged.entry(src).or_insert(item); + } + let mut ranked: Vec = merged.into_values().collect(); + apply_recall_ranking_boosts(&mut ranked, query_text, 0.08, 0.12); + let sources: Vec = ranked.iter().map(|r| r.source.clone()).collect(); + let boosts = crate::handlers::feedback::compute_boosts(conn, &sources, query_vector); + if !boosts.is_empty() { + for item in &mut ranked { + if let Some(&boost) = boosts.get(&item.source) { + item.relevance = round4(item.relevance * (1.0 + boost)); + } + } + } + ranked.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + ranked.truncate(k); + Ok(RecallWithVectorTrace { ranked, semantic_route }) +} +pub fn unfold_source(conn: &Connection, source: &str, ctx: &RecallContext) -> Option { + if let Some(crystal_id) = parse_crystal_source_id(source) { + if let Some((label, consolidated_text, member_count, owner_id, visibility)) = query_crystal_for_unfold(conn, crystal_id) { + if is_visible(owner_id, visibility.as_deref(), ctx) { + let members = crystal_member_sources(conn, crystal_id, ctx); + let mut full_text = consolidated_text.clone(); + if !members.is_empty() { + full_text.push_str("\n\nFamily members:\n"); + for member in members.iter().take(16) { + full_text.push_str("- "); + full_text.push_str(member); + full_text.push('\n'); + } + if member_count as usize > members.len() { + full_text.push_str(&format!("... plus {} more hidden or archived member(s)", (member_count as usize).saturating_sub(members.len()))); + } + } + return Some( + json!({"source":crystal_source(crystal_id,&label),"text":full_text.trim_end().to_string(),"type":"crystal","label":label,"clusterId":crystal_id,"members":members,"memberCount":member_count,}), + ); + } + } + } + if let Some((text, ty, owner_id, visibility)) = query_memory_for_unfold(conn, source) { + if is_visible(owner_id, visibility.as_deref(), ctx) { + return Some(json!({"text":text,"type":ty})); + } + } + if let Some(id_str) = source.strip_prefix("decision::") { + if let Ok(id) = id_str.parse::() { + if let Some((decision, context, owner_id, visibility)) = query_decision_by_id_for_unfold(conn, id) { + if is_visible(owner_id, visibility.as_deref(), ctx) { + let full = match context { + Some(c) => format!("{decision}\n\nContext: {c}"), + None => decision, + }; + return Some(json!({"text":full,"type":"decision"})); + } + } + } + } + if let Some((decision, context, owner_id, visibility)) = query_decision_by_context_for_unfold(conn, source) { + if is_visible(owner_id, visibility.as_deref(), ctx) { + let full = match context { + Some(c) => format!("{decision}\n\nContext: {c}"), + None => decision, + }; + return Some(json!({"text":full,"type":"decision"})); + } + } + let stripped = source.strip_prefix("memory::").unwrap_or(source); + if stripped != source { + if let Some((text, ty, owner_id, visibility)) = query_memory_for_unfold(conn, stripped) { + if is_visible(owner_id, visibility.as_deref(), ctx) { + return Some(json!({"text":text,"type":ty})); + } + } + } + None +} +const UNFOLD_ACTIVE:&str="status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) AND (valid_from IS NULL OR valid_from <= datetime('now')) AND (valid_until IS NULL OR valid_until > datetime('now'))"; +pub(crate) type MemoryUnfoldRow = (String, String, Option, Option); +pub(crate) type DecisionUnfoldRow = (String, Option, Option, Option); +fn query_acl_row(conn: &Connection, with_sql: &str, without_sql: &str, bind: &[&dyn rusqlite::types::ToSql], map_with: F, map_without: G) -> Option +where + F: FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result, + G: FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result, +{ + match conn.query_row(with_sql, bind, map_with) { + Ok(row) => Some(row), + Err(err) if is_missing_team_visibility_columns(&err) => conn.query_row(without_sql, bind, map_without).ok(), + Err(_) => None, + } +} +pub(crate) fn query_memory_for_unfold(conn: &Connection, source: &str) -> Option { + let bind: Vec<&dyn rusqlite::types::ToSql> = vec![&source]; + query_acl_row( + conn, + &format!("SELECT text, type, owner_id, visibility FROM memories WHERE source = ?1 AND {UNFOLD_ACTIVE} ORDER BY score DESC LIMIT 1"), + &format!("SELECT text, type FROM memories WHERE source = ?1 AND {UNFOLD_ACTIVE} ORDER BY score DESC LIMIT 1"), + &bind, + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| Ok((row.get(0)?, row.get(1)?, None, None)), + ) +} +fn query_decision_for_unfold(conn: &Connection, predicate: &str, bind: &[&dyn rusqlite::types::ToSql], order_limit: &str) -> Option { + query_acl_row( + conn, + &format!("SELECT decision, context, owner_id, visibility FROM decisions WHERE {predicate} AND {UNFOLD_ACTIVE}{order_limit}"), + &format!("SELECT decision, context FROM decisions WHERE {predicate} AND {UNFOLD_ACTIVE}{order_limit}"), + bind, + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| Ok((row.get(0)?, row.get(1)?, None, None)), + ) +} +pub(crate) fn query_decision_by_id_for_unfold(conn: &Connection, id: i64) -> Option { + query_decision_for_unfold(conn, "id = ?1", &[&id], "") +} +pub(crate) fn query_decision_by_context_for_unfold(conn: &Connection, source: &str) -> Option { + query_decision_for_unfold(conn, "context = ?1", &[&source], " ORDER BY score DESC LIMIT 1") +} diff --git a/daemon-rs/src/handlers/recall/engine_semantic.rs b/daemon-rs/src/handlers/recall/engine_semantic.rs new file mode 100644 index 00000000..8bf0927a --- /dev/null +++ b/daemon-rs/src/handlers/recall/engine_semantic.rs @@ -0,0 +1,206 @@ +pub(crate) fn scale_semantic_similarity(sim: f32) -> f64 { + SEMANTIC_SCALE_BASE + (sim as f64 - SEMANTIC_SIM_FLOOR) * ((1.0 - SEMANTIC_SCALE_BASE) / (1.0 - SEMANTIC_SIM_FLOOR)) +} +pub(crate) fn scale_semantic_similarity_with_keyword_overlap(sim: f32, text: &str, keyword_terms: &[String]) -> f64 { + let mut scaled = scale_semantic_similarity(sim); + if !keyword_terms.is_empty() { + let haystack = text.to_lowercase(); + let overlap = keyword_terms.iter().filter(|term| haystack.contains(term.as_str())).count(); + scaled *= if overlap == 0 { 0.82 } else { 1.0 + (overlap as f64 / keyword_terms.len().max(1) as f64) * 0.08 }; + } + scaled +} +fn upsert_best_semantic_candidate( + candidates: &mut HashMap, source: String, excerpt: String, relevance: f64, importance: f64, ts: i64, +) { + let entry = candidates + .entry(source.clone()) + .or_insert(SemanticCandidate { source, excerpt: excerpt.clone(), relevance, importance, ts }); + if relevance > entry.relevance { + *entry = SemanticCandidate { source: entry.source.clone(), excerpt, relevance, importance, ts }; + } +} +#[derive(Clone, Copy)] +enum SemanticEmbedKind { + Memory, + Decision, +} +impl SemanticEmbedKind { + fn target_type(self) -> &'static str { + match self { + Self::Memory => "memory", + Self::Decision => "decision", + } + } + fn join_table(self) -> &'static str { + match self { + Self::Memory => "memories", + Self::Decision => "decisions", + } + } + fn source_filter(self) -> &'static str { + match self { + Self::Memory => "(?5 IS NULL OR m.source LIKE ?5)", + Self::Decision => "(?5 IS NULL OR d.context LIKE ?5)", + } + } + fn query_with_acl(self) -> String { + let table = self.join_table(); + let alias = if matches!(self, Self::Memory) { "m" } else { "d" }; + let cols = match self { + Self::Memory => { + "e.vector, m.text, m.source, m.owner_id, m.visibility, m.score, \ + m.trust_score, m.last_accessed, m.created_at" + } + Self::Decision => { + "e.vector, d.decision, d.context, d.owner_id, d.visibility, d.score, \ + d.trust_score, d.last_accessed, d.created_at" + } + }; + self.query(cols, alias, table) + } + fn query_without_acl(self) -> String { + let table = self.join_table(); + let alias = if matches!(self, Self::Memory) { "m" } else { "d" }; + let cols = match self { + Self::Memory => { + "e.vector, m.text, m.source, NULL AS owner_id, NULL AS visibility, m.score, \ + m.trust_score, m.last_accessed, m.created_at" + } + Self::Decision => { + "e.vector, d.decision, d.context, NULL AS owner_id, NULL AS visibility, d.score, \ + d.trust_score, d.last_accessed, d.created_at" + } + }; + self.query(cols, alias, table) + } + fn shadow_query_with_acl(self) -> String { + let table = self.join_table(); + let alias = if matches!(self, Self::Memory) { "m" } else { "d" }; + let cols = match self { + Self::Memory => "e.vector, m.source, m.owner_id, m.visibility", + Self::Decision => "e.vector, d.decision, d.context, d.owner_id, d.visibility", + }; + self.query(cols, alias, table) + } + fn shadow_query_without_acl(self) -> String { + let table = self.join_table(); + let alias = if matches!(self, Self::Memory) { "m" } else { "d" }; + let cols = match self { + Self::Memory => "e.vector, m.source, NULL AS owner_id, NULL AS visibility", + Self::Decision => "e.vector, d.decision, d.context, NULL AS owner_id, NULL AS visibility", + }; + self.query(cols, alias, table) + } + fn query(self, cols: &str, alias: &str, table: &str) -> String { + let order_expr = match self { + Self::Memory => { + "COALESCE(m.score, 1.0) * COALESCE(m.trust_score, 0.8) DESC, \ + COALESCE(m.last_accessed, m.created_at) DESC, m.id DESC" + } + Self::Decision => { + "COALESCE(d.score, 1.0) * COALESCE(d.trust_score, 0.8) DESC, \ + COALESCE(d.last_accessed, d.created_at) DESC, d.id DESC" + } + }; + format!( + "SELECT {cols} + FROM embeddings e + JOIN {table} {alias} + ON e.target_type = '{target_type}' + AND e.target_id = {alias}.id + WHERE {alias}.status = 'active' + AND ({alias}.expires_at IS NULL OR {alias}.expires_at > datetime('now')) + AND ({alias}.valid_from IS NULL OR {alias}.valid_from <= datetime('now')) + AND ({alias}.valid_until IS NULL OR {alias}.valid_until > datetime('now')) + AND (e.model IS NULL OR LOWER(e.model) = ?1) + AND ( + length(e.vector) = ?2 + OR (length(e.vector) = ?3 AND substr(e.vector, 1, 2) = ?4) + ) + AND {source_filter} + ORDER BY {order_expr} + LIMIT ?6", + target_type = self.target_type(), + source_filter = self.source_filter(), + ) + } +} +fn prepare_acl_stmt<'a>(conn: &'a Connection, with_acl: &str, without_acl: &str) -> Option> { + match conn.prepare(with_acl) { + Ok(stmt) => Some(stmt), + Err(err) if is_missing_team_visibility_columns(&err) => conn.prepare(without_acl).ok(), + Err(_) => None, + } +} +fn decision_source_key(context: Option, decision: &str) -> String { + context.unwrap_or_else(|| format!("decision::{}", decision.chars().take(40).collect::())) +} +pub(crate) fn collect_semantic_candidates( + conn: &Connection, query_vector: &[f32], query_text: &str, ctx: &RecallContext, source_prefix: Option<&str>, +) -> Vec { + let selected_model = crate::embeddings::selected_model_key(); + let expected_legacy_vector_bytes = std::mem::size_of_val(query_vector) as i64; + let expected_pq8_vector_bytes = (query_vector.len() + crate::embeddings::PQ8_HEADER_BYTES) as i64; + let pq8_prefix = [crate::embeddings::PQ8_MAGIC_BYTE, crate::embeddings::PQ8_FORMAT_VERSION]; + let candidate_limit = MAX_SEMANTIC_SQL_ROWS_PER_KIND as i64; + let source_like = source_prefix.map(|prefix| format!("{prefix}%")); + let keyword_terms = extract_search_keywords(query_text); + let semantic_floor = if keyword_terms.len() >= 3 { SEMANTIC_SIM_FLOOR + 0.12 } else { SEMANTIC_SIM_FLOOR }; + let mut candidates: HashMap = HashMap::new(); + for kind in [SemanticEmbedKind::Memory, SemanticEmbedKind::Decision] { + let Some(mut stmt) = prepare_acl_stmt(conn, &kind.query_with_acl(), &kind.query_without_acl()) else { + continue; + }; + let Ok(rows) = stmt.query_map( + params![ + selected_model, + expected_legacy_vector_bytes, + expected_pq8_vector_bytes, + pq8_prefix.as_slice(), + source_like.as_deref(), + candidate_limit + ], + |row| { + Ok(( + row.get::<_, Vec>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + )) + }, + ) else { + continue; + }; + for (blob, primary, alt, owner_id, visibility, score, trust_score, last_accessed, created_at) in rows.flatten() { + if !is_visible(owner_id, visibility.as_deref(), ctx) { + continue; + } + let source = match kind { + SemanticEmbedKind::Memory => primary.clone(), + SemanticEmbedKind::Decision => decision_source_key(alt, &primary), + }; + if !source_matches_prefix(&source, source_prefix) { + continue; + } + let sim = crate::embeddings::cosine_similarity(query_vector, &crate::embeddings::blob_to_vector(&blob)); + if sim <= semantic_floor as f32 { + continue; + } + let scaled = scale_semantic_similarity_with_keyword_overlap(sim, &primary, &keyword_terms); + let excerpt = query_focused_excerpt(&primary, query_text, 280); + let importance = blend_importance(score, trust_score); + let ts = parse_timestamp_ms(last_accessed.as_deref().or(created_at.as_deref()).unwrap_or_default()); + upsert_best_semantic_candidate(&mut candidates, source, excerpt, scaled, importance, ts); + } + } + let mut sorted: Vec = candidates.into_values().collect(); + sorted.sort_by(|a, b| compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source)); + sorted.truncate(MAX_SEMANTIC_RRF_CANDIDATES); + sorted +} diff --git a/daemon-rs/src/handlers/recall/engine_support.rs b/daemon-rs/src/handlers/recall/engine_support.rs new file mode 100644 index 00000000..cf88f53b --- /dev/null +++ b/daemon-rs/src/handlers/recall/engine_support.rs @@ -0,0 +1,287 @@ +pub(crate) fn round4(value: f64) -> f64 { + if !value.is_finite() { + return 0.0; + } + (value * 10000.0).round() / 10000.0 +} +fn numbered_placeholders(start: usize, len: usize) -> String { + let mut placeholders = String::new(); + for idx in 0..len { + if idx > 0 { + placeholders.push(','); + } + let _ = write!(placeholders, "?{}", start + idx); + } + placeholders +} +fn bump_retrievals_str_keys(conn: &Connection, table: &str, key_col: &str, now: &str, keys: &[&str]) { + if keys.is_empty() { + return; + } + let placeholders = numbered_placeholders(2, keys.len()); + let sql=format!("UPDATE {table} SET retrievals = retrievals + 1, last_accessed = ?1, score = MIN(1.0, score + 0.15 / (1.0 + 0.1 * retrievals)) WHERE {key_col} IN ({placeholders})"); + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(keys.len() + 1); + params.push(&now); + for key in keys { + params.push(key); + } + let _ = conn.execute(&sql, params.as_slice()); +} +fn bump_retrievals_i64_keys(conn: &Connection, table: &str, key_col: &str, now: &str, keys: &[i64]) { + if keys.is_empty() { + return; + } + let placeholders = numbered_placeholders(2, keys.len()); + let sql=format!("UPDATE {table} SET retrievals = retrievals + 1, last_accessed = ?1, score = MIN(1.0, score + 0.15 / (1.0 + 0.1 * retrievals)) WHERE {key_col} IN ({placeholders})"); + let mut params: Vec<&dyn rusqlite::types::ToSql> = Vec::with_capacity(keys.len() + 1); + params.push(&now); + for key in keys { + params.push(key); + } + let _ = conn.execute(&sql, params.as_slice()); +} +pub(crate) fn bump_retrievals_batch(conn: &Connection, items: &[RecallItem]) { + if items.is_empty() { + return; + } + let now = now_iso(); + let sources: Vec<&str> = items.iter().map(|item| item.source.as_str()).collect(); + bump_retrievals_str_keys(conn, "memories", "source", &now, &sources); + let decision_ids: Vec = sources.iter().filter_map(|s| s.strip_prefix("decision::").and_then(|id| id.parse::().ok())).collect(); + bump_retrievals_i64_keys(conn, "decisions", "id", &now, &decision_ids); + let context_sources: Vec<&str> = sources.iter().copied().filter(|source| !source.starts_with("decision::")).collect(); + bump_retrievals_str_keys(conn, "decisions", "context", &now, &context_sources); +} +pub(crate) fn recall_to_json(item: RecallItem) -> Value { + let mut payload = json!({"source":item.source,"relevance":item.relevance,"excerpt":item.excerpt,"method":item.method}); + if let Value::Object(ref mut map) = payload { + if let Some(tokens) = item.tokens { + map.insert("tokens".to_string(), Value::Number((tokens as u64).into())); + } + if !item.family_members.is_empty() { + let family_size = item.family_members.len() as u64; + map.insert("familyMembers".to_string(), Value::Array(item.family_members.into_iter().map(Value::String).collect())); + map.insert("familySize".to_string(), Value::Number(family_size.into())); + } + if !item.collapsed_sources.is_empty() { + map.insert("collapsedSources".to_string(), Value::Array(item.collapsed_sources.into_iter().map(Value::String).collect())); + } + if !item.collapsed_source_scores.is_empty() { + map.insert( + "collapsedSourceScores".to_string(), + Value::Array( + item.collapsed_source_scores + .into_iter() + .map(|(source, relevance)| json!({"source":source,"relevance":relevance,})) + .collect(), + ), + ); + } + } + payload +} +#[derive(Clone, Copy, Debug)] +pub(crate) struct RecallBudgetUsage { + pub(crate) spent: usize, + pub(crate) saved: i64, + pub(crate) over_budget: bool, +} +pub(crate) fn recall_item_token_cost(item: &RecallItem) -> usize { + item.tokens.unwrap_or_else(|| estimate_tokens(&format!("{}{}", item.source, item.excerpt))) +} +pub(crate) fn compute_recall_budget_usage(items: &[RecallItem], budget: usize) -> RecallBudgetUsage { + let spent: usize = items.iter().map(recall_item_token_cost).sum(); + let saved = budget as i64 - spent as i64; + RecallBudgetUsage { spent, saved, over_budget: budget > 0 && spent > budget } +} +pub(crate) fn compute_headlines_token_usage(items: &[RecallItem]) -> RecallBudgetUsage { + let spent = items.iter().map(|item| estimate_tokens(&item.source)).sum::(); + let full_recall_tokens = items.iter().map(recall_item_token_cost).sum::(); + RecallBudgetUsage { spent, saved: full_recall_tokens as i64 - spent as i64, over_budget: false } +} +pub(crate) fn format_recall_token_usage_line(budget: usize, usage: RecallBudgetUsage) -> String { + if budget == 0 { + if usage.saved > 0 { + format!("Cortex recall used {} tokens in headlines mode and saved {} vs full excerpts.", usage.spent, usage.saved) + } else { + format!("Cortex recall used {} tokens (headlines mode).", usage.spent) + } + } else if usage.saved >= 0 { + format!("Cortex recall used {} tokens and saved {} of {} budget.", usage.spent, usage.saved, budget) + } else { + format!("Cortex recall used {} tokens ({} over budget {}).", usage.spent, usage.saved.abs(), budget) + } +} +pub(crate) fn enforce_budget_token_invariant(results: Vec, token_budget: usize, query_text: &str) -> Vec { + if token_budget == 0 || results.is_empty() { + return results; + } + let usage = compute_recall_budget_usage(&results, token_budget); + if !usage.over_budget { + return results; + } + let mut kept = Vec::new(); + let mut spent = 0usize; + for (idx, mut item) in results.into_iter().enumerate() { + let remaining = token_budget.saturating_sub(spent); + if remaining <= MIN_BUDGET_HEADROOM_TOKENS { + break; + } + let direct_tokens = recall_item_token_cost(&item); + if direct_tokens <= remaining { + item.tokens = Some(direct_tokens); + spent += direct_tokens; + kept.push(item); + continue; + } + let cap = budget_rank_char_cap(token_budget, idx, query_text).min((remaining as f64 * 3.6) as usize).max(MIN_EXCERPT_CHARS); + if let Some((excerpt, tokens)) = fit_excerpt_to_remaining_budget(&item.source, &item.excerpt, query_text, cap, remaining) { + if tokens <= remaining { + item.excerpt = excerpt; + item.tokens = Some(tokens); + spent += tokens; + kept.push(item); + } + } + } + kept +} +pub(crate) fn hash_content(content: &str) -> u32 { + let mut hash: u32 = 2_166_136_261; + for ch in content.chars().take(100) { + hash ^= ch as u32; + hash = hash.wrapping_mul(16_777_619); + } + hash +} +pub(crate) fn source_dedup_hash(source: &str) -> u32 { + hash_content(&format!("source::{source}")) +} +pub(crate) fn collapse_score_is_better(candidate_score: f64, candidate_order: usize, best_score: f64, best_order: usize) -> bool { + match candidate_score.total_cmp(&best_score) { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Less => false, + std::cmp::Ordering::Equal => candidate_order < best_order, + } +} +pub(crate) async fn load_collapsed_source_fallback(state: &RuntimeState, source: &str, query: &str, ctx: &RecallContext, relevance: f64) -> Option { + let conn = state.db_read.lock().await; + let payload = unfold_source(&conn, source, ctx)?; + let canonical_source = payload.get("source").and_then(|value| value.as_str()).unwrap_or(source).to_string(); + let text = payload.get("text").and_then(|value| value.as_str())?; + Some(RecallItem { + source: canonical_source, + relevance, + excerpt: query_focused_excerpt(text, query, 260), + method: "crystal".to_string(), + tokens: None, + entropy: None, + family_members: Vec::new(), + collapsed_sources: Vec::new(), + collapsed_source_scores: Vec::new(), + }) +} +pub(crate) const SERVED_TTL_MS: i64 = 60_000; +pub(crate) async fn dedup_and_mark_served(state: &RuntimeState, agent: &str, query: &str, ctx: &RecallContext, results: Vec) -> Vec { + if results.is_empty() { + return results; + } + let now = Utc::now().timestamp_millis(); + let scope_key = served_content_scope(agent, query, ctx); + let mut seen_hashes: HashSet = { + let mut served = state.served_content.lock().await; + let map = served.entry(scope_key.clone()).or_insert_with(HashMap::::new); + map.retain(|_, ts| now - *ts < SERVED_TTL_MS); + map.keys().copied().collect() + }; + let mut staged_hashes: Vec = Vec::with_capacity(results.len() * 2); + let mut filtered = Vec::new(); + for result in results { + let excerpt_hash = hash_content(&result.excerpt); + let source_hash = source_dedup_hash(&result.source); + let already_served = seen_hashes.contains(&excerpt_hash) || seen_hashes.contains(&source_hash); + if already_served { + if result.method == "crystal" && !result.collapsed_sources.is_empty() { + let fallback_candidates: Vec<(usize, String, f64)> = if result.collapsed_source_scores.is_empty() { + result.collapsed_sources.iter().enumerate().map(|(idx, source)| (idx, source.clone(), 0.0)).collect() + } else { + result.collapsed_source_scores.iter().enumerate().map(|(idx, (source, score))| (idx, source.clone(), *score)).collect() + }; + let mut best_candidate: Option<(usize, f64, RecallItem)> = None; + for (order, collapsed_source, collapsed_score) in fallback_candidates { + let collapsed_source_hash = source_dedup_hash(&collapsed_source); + if seen_hashes.contains(&collapsed_source_hash) { + continue; + } + let candidate_relevance = round4(collapsed_score.max(0.0)); + let Some(candidate) = load_collapsed_source_fallback(state, &collapsed_source, query, ctx, candidate_relevance).await else { + continue; + }; + let candidate_excerpt_hash = hash_content(&candidate.excerpt); + let candidate_source_hash = source_dedup_hash(&candidate.source); + if seen_hashes.contains(&candidate_excerpt_hash) || seen_hashes.contains(&candidate_source_hash) { + continue; + } + let replace = match &best_candidate { + None => true, + Some((best_order, best_score, _)) => collapse_score_is_better(candidate_relevance, order, *best_score, *best_order), + }; + if replace { + best_candidate = Some((order, candidate_relevance, candidate)); + } + } + if let Some((_, _, candidate)) = best_candidate { + let candidate_excerpt_hash = hash_content(&candidate.excerpt); + let candidate_source_hash = source_dedup_hash(&candidate.source); + seen_hashes.insert(candidate_excerpt_hash); + seen_hashes.insert(candidate_source_hash); + staged_hashes.push(candidate_excerpt_hash); + staged_hashes.push(candidate_source_hash); + filtered.push(candidate); + } + } + continue; + } + seen_hashes.insert(excerpt_hash); + seen_hashes.insert(source_hash); + staged_hashes.push(excerpt_hash); + staged_hashes.push(source_hash); + filtered.push(result); + } + if !staged_hashes.is_empty() { + let mut served = state.served_content.lock().await; + let map = served.entry(scope_key).or_insert_with(HashMap::::new); + map.retain(|_, ts| now - *ts < SERVED_TTL_MS); + for hash in staged_hashes { + map.insert(hash, now); + } + } + filtered +} +pub(crate) fn recall_owner_scope(ctx: &RecallContext) -> String { + if !ctx.team_mode { + return "solo".to_string(); + } + match ctx.caller_id { + Some(owner_id) => format!("team:{owner_id}"), + None => "team:none".to_string(), + } +} +pub(crate) fn recall_scope_key(agent: &str, ctx: &RecallContext) -> String { + format!("{}::{agent}", recall_owner_scope(ctx)) +} +pub(crate) fn served_content_scope(agent: &str, query: &str, ctx: &RecallContext) -> String { + let normalized_query = query.split_whitespace().map(|segment| segment.to_ascii_lowercase()).collect::>().join(" "); + format!("{}::{agent}::{normalized_query}", recall_owner_scope(ctx)) +} +pub(crate) fn maybe_apply_rerank(state: &RuntimeState, results: Vec, budget: usize) -> (Vec, Value) { + let config = &state.rerank_config; + let reason = if budget == 0 { "headlines_mode" } else { "mode_off" }; + (results, json!({"status":"skipped","reason":reason,"mode":config.mode.as_str()})) +} +pub(crate) fn is_benchmark_recall_scope(agent: &str, source_prefix: Option<&str>) -> bool { + if agent.trim().to_ascii_lowercase().starts_with(BENCHMARK_SOURCE_AGENT_PREFIX) { + return true; + } + source_prefix.map(str::trim).unwrap_or_default().to_ascii_lowercase().starts_with(BENCHMARK_SOURCE_SCOPE_PREFIX) +} diff --git a/daemon-rs/src/handlers/recall/handlers.rs b/daemon-rs/src/handlers/recall/handlers.rs index f7c369a0..4c4d0608 100644 --- a/daemon-rs/src/handlers/recall/handlers.rs +++ b/daemon-rs/src/handlers/recall/handlers.rs @@ -1,517 +1,294 @@ -// SPDX-License-Identifier: MIT +use super::*; +use crate::budgets::BudgetEndpoint; +use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, estimate_tokens, json_response, resolve_source_identity}; +use crate::rate_limit::RequestClass; +use crate::state::RuntimeState; use axum::extract::{Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; use serde::Deserialize; use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── GET /recall ───────────────────────────────────────────────────────────── - -pub async fn handle_recall( - State(state): State, - Query(query): Query, - headers: HeaderMap, +pub(crate) const MAX_UNFOLD_SOURCES: usize = 50; +#[derive(Deserialize, Default)] +pub struct UnfoldQuery { + pub sources: Option, +} +async fn auth_recall_caller(headers: &HeaderMap, state: &RuntimeState) -> Result, Response> { + let caller_id = ensure_auth_with_caller_rated_for_class(headers, state, RequestClass::Recall).await?; + if state.team_mode && caller_id.is_none() { + return Err(json_response(StatusCode::FORBIDDEN, json!({"error":"Team mode requires a caller-scoped ctx_ API key"}))); + } + Ok(caller_id) +} +fn trim_source_prefix(source_prefix: Option<&str>) -> Option<&str> { + source_prefix.map(str::trim).filter(|s| !s.is_empty()) +} +fn attach_policy_modes(payload: &mut Value, resolved: RecallPolicyMode, requested: Option) { + if let Value::Object(map) = payload { + map.insert("policyMode".to_string(), Value::String(resolved.as_str().to_string())); + if let Some(mode) = requested { + map.insert("requestedPolicyMode".to_string(), Value::String(mode.as_str().to_string())); + } + } +} +fn fire_recall_brain_event(state: &RuntimeState, payload: &Value, agent: &str) { + let node_ids = extract_recall_node_ids(payload); + let _ = state.brain_firing.send(crate::state::BrainFiringEvent { + kind: crate::state::BrainKind::Recall, + payload: json!({"node_ids":node_ids,"agent":agent}), + owner_id: state.default_owner_id, + }); +} +async fn run_unified_recall_handler( + state: &RuntimeState, headers: &HeaderMap, q: String, requested_policy_mode: Option, budget: Option, k: Option, + source_prefix: Option, agent_default: Option<&str>, missing_q_error: &str, failure_prefix: &str, ) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, + let caller_id = match auth_recall_caller(headers, state).await { + Ok(id) => id, Err(resp) => return resp, }; - let q = query.q.unwrap_or_default(); - let requested_policy_mode = match parse_recall_policy_mode(query.policy_mode.as_deref()) { - Ok(mode) => mode, - Err(err) => { - return json_response(StatusCode::BAD_REQUEST, json!({ "error": err })); - } - }; - let (mut budget, k, _resolved_policy_mode) = - resolve_recall_budget_k(requested_policy_mode, query.budget, query.k); - let source_prefix = query - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - let agent = resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("http")).agent; - + let source_prefix = trim_source_prefix(source_prefix.as_deref()); + let agent = resolve_source_identity(headers, agent_default.unwrap_or("http")).agent; if q.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing query parameter: q" }), - ); + return json_response(StatusCode::BAD_REQUEST, json!({"error":missing_q_error})); } - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(headers, state, BudgetEndpoint::Recall, &agent).await { return resp; } - budget = maybe_apply_adaptive_default_budget( - q.trim(), - requested_policy_mode, - query.budget, - budget, - k, - ); + let requested_budget = budget; + let (mut budget, k, _) = resolve_recall_budget_k(requested_policy_mode, requested_budget, k); + budget = maybe_apply_adaptive_default_budget(q.trim(), requested_policy_mode, requested_budget, budget, k); let resolved_policy_mode = recall_mode_for_budget(budget); - - let ctx = RecallContext::from_caller(caller_id, &state); - match execute_unified_recall(&state, q.trim(), budget, k, &agent, &ctx, source_prefix).await { + let ctx = RecallContext::from_caller(caller_id, state); + match execute_unified_recall(state, q.trim(), budget, k, &agent, &ctx, source_prefix).await { Ok(mut payload) => { - if let Value::Object(map) = &mut payload { - map.insert( - "policyMode".to_string(), - Value::String(resolved_policy_mode.as_str().to_string()), - ); - if let Some(mode) = requested_policy_mode { - map.insert( - "requestedPolicyMode".to_string(), - Value::String(mode.as_str().to_string()), - ); - } - } - let node_ids = extract_recall_node_ids(&payload); - let _ = state.brain_firing.send(crate::state::BrainFiringEvent { - kind: crate::state::BrainKind::Recall, - payload: json!({ "node_ids": node_ids, "agent": agent }), - owner_id: state.default_owner_id, - }); + attach_policy_modes(&mut payload, resolved_policy_mode, requested_policy_mode); + fire_recall_brain_event(state, &payload, &agent); json_response(StatusCode::OK, payload) } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Recall failed: {err}") }), - ), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("{failure_prefix}: {err}")})), } } - -/// Extract a flat list of node ID strings from a recall payload for Brain -/// telemetry. Returns up to 16 IDs to keep the SSE event small. Looks at -/// "memories", "decisions", "crystals", and "results" arrays under any nesting. -pub(crate) fn extract_recall_node_ids(payload: &Value) -> Vec { - fn walk(v: &Value, out: &mut Vec, limit: usize) { - if out.len() >= limit { - return; - } - match v { - Value::Object(map) => { - if let (Some(target_type), Some(target_id)) = ( - map.get("type").and_then(|t| t.as_str()), - map.get("id").and_then(|t| t.as_i64()), - ) { - if matches!(target_type, "memory" | "decision" | "crystal") { - out.push(format!("{target_type}-{target_id}")); - if out.len() >= limit { - return; - } - } - } - for (_, child) in map.iter() { - walk(child, out, limit); - } - } - Value::Array(arr) => { - for child in arr.iter() { - walk(child, out, limit); - } - } - _ => {} - } - } - let mut out = Vec::new(); - walk(payload, &mut out, 16); - out -} - -// ─── POST /recall ──────────────────────────────────────────────────────────── - -pub async fn handle_recall_post( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, - Err(resp) => return resp, +pub async fn handle_recall(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { + let requested_policy_mode = match parse_recall_policy_mode(query.policy_mode.as_deref()) { + Ok(mode) => mode, + Err(err) => return json_response(StatusCode::BAD_REQUEST, json!({"error":err})), }; - let q = body.q.unwrap_or_default(); + run_unified_recall_handler( + &state, + &headers, + query.q.unwrap_or_default(), + requested_policy_mode, + query.budget, + query.k, + query.source_prefix, + query.agent.as_deref(), + "Missing query parameter: q", + "Recall failed", + ) + .await +} +pub async fn handle_recall_post(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { let requested_policy_mode = match parse_recall_policy_mode(body.policy_mode.as_deref()) { Ok(mode) => mode, - Err(err) => { - return json_response(StatusCode::BAD_REQUEST, json!({ "error": err })); - } + Err(err) => return json_response(StatusCode::BAD_REQUEST, json!({"error":err})), }; - let (mut budget, k, _resolved_policy_mode) = - resolve_recall_budget_k(requested_policy_mode, body.budget, body.k); - let source_prefix = body - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - let agent = resolve_source_identity(&headers, body.agent.as_deref().unwrap_or("http")).agent; - - if q.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing recall payload field: q" }), - ); - } - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { - return resp; - } - budget = maybe_apply_adaptive_default_budget( - q.trim(), + run_unified_recall_handler( + &state, + &headers, + body.q.unwrap_or_default(), requested_policy_mode, body.budget, - budget, - k, - ); - let resolved_policy_mode = recall_mode_for_budget(budget); - - let ctx = RecallContext::from_caller(caller_id, &state); - match execute_unified_recall(&state, q.trim(), budget, k, &agent, &ctx, source_prefix).await { - Ok(mut payload) => { - if let Value::Object(map) = &mut payload { - map.insert( - "policyMode".to_string(), - Value::String(resolved_policy_mode.as_str().to_string()), - ); - if let Some(mode) = requested_policy_mode { - map.insert( - "requestedPolicyMode".to_string(), - Value::String(mode.as_str().to_string()), - ); - } - } - let node_ids = extract_recall_node_ids(&payload); - let _ = state.brain_firing.send(crate::state::BrainFiringEvent { - kind: crate::state::BrainKind::Recall, - payload: json!({ "node_ids": node_ids, "agent": agent }), - owner_id: state.default_owner_id, - }); - json_response(StatusCode::OK, payload) - } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Recall failed: {err}") }), - ), - } + body.k, + body.source_prefix, + body.agent.as_deref(), + "Missing recall payload field: q", + "Recall failed", + ) + .await } - -pub async fn handle_semantic_recall( - State(state): State, - Query(query): Query, - headers: HeaderMap, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, +pub async fn handle_semantic_recall(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { + let caller_id = match auth_recall_caller(&headers, &state).await { + Ok(id) => id, Err(resp) => return resp, }; let q = query.q.unwrap_or_default(); let k = query.k.unwrap_or(10); let budget = query.budget.unwrap_or(200); - let source_prefix = query - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); + let source_prefix = trim_source_prefix(query.source_prefix.as_deref()); let agent = resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("http")).agent; - if q.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing query parameter: q" }), - ); + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing query parameter: q"})); } - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await { return resp; } - let ctx = RecallContext::from_caller(caller_id, &state); match execute_semantic_recall(&state, q.trim(), budget, k, &agent, &ctx, source_prefix).await { Ok(payload) => json_response(StatusCode::OK, payload), - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Semantic recall failed: {err}") }), - ), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Semantic recall failed: {err}")})), } } - -// ─── GET /recall/budget ────────────────────────────────────────────────────── - -pub async fn handle_budget_recall( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, +pub async fn handle_budget_recall(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { + let caller_id = match auth_recall_caller(&headers, &state).await { + Ok(id) => id, Err(resp) => return resp, }; let q = match query.q.as_deref() { Some(s) if !s.trim().is_empty() => s.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing query parameter: q" }), - ); - } + _ => return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing query parameter: q"})), }; - let agent = resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("http")).agent; - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await { return resp; } let budget = query.budget.unwrap_or(300); let k = query.k.unwrap_or(10); - let source_prefix = query - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - + let source_prefix = trim_source_prefix(query.source_prefix.as_deref()); let ctx = RecallContext::from_caller(caller_id, &state); let mut conn = state.db.lock().await; let engine = state.embedding_engine.as_deref(); - match run_budget_recall_with_engine( - &mut conn, - &q, - budget, - k, - engine, - &ctx, - source_prefix, - Some(&state.degraded_mode), - ) { + match run_budget_recall_with_engine(&mut conn, &q, budget, k, engine, &ctx, source_prefix, Some(&state.degraded_mode)) { Ok(results) => { let usage = compute_recall_budget_usage(&results, budget); json_response( StatusCode::OK, - json!({ - "results": results.into_iter().map(recall_to_json).collect::>(), - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "tokenUsageLine": format_recall_token_usage_line(budget, usage), - }), + json!({"results":results.into_iter().map(recall_to_json).collect::>(),"budget":budget,"spent":usage.spent,"saved":usage.saved,"tokenUsageLine":format_recall_token_usage_line(budget,usage),}), ) } - Err(e) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Budget recall failed: {e}") }), - ), + Err(e) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Budget recall failed: {e}")})), } } - -pub async fn handle_recall_explain( - State(state): State, - Query(query): Query, - headers: HeaderMap, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, +pub async fn handle_recall_explain(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { + let caller_id = match auth_recall_caller(&headers, &state).await { + Ok(id) => id, Err(resp) => return resp, }; let q = query.q.unwrap_or_default(); if q.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing query parameter: q" }), - ); + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing query parameter: q"})); } - let requested_policy_mode = match parse_recall_policy_mode(query.policy_mode.as_deref()) { Ok(mode) => mode, - Err(err) => { - return json_response(StatusCode::BAD_REQUEST, json!({ "error": err })); - } + Err(err) => return json_response(StatusCode::BAD_REQUEST, json!({"error":err})), }; - let (mut budget, k, _resolved_policy_mode) = - resolve_recall_budget_k(requested_policy_mode, query.budget, query.k); + let (mut budget, k, _) = resolve_recall_budget_k(requested_policy_mode, query.budget, query.k); let pool_k = query.pool_k.unwrap_or((k.max(8) * 3).min(64)); - let source_prefix = query - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); + let source_prefix = trim_source_prefix(query.source_prefix.as_deref()); let agent = resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("http")).agent; - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await { return resp; } - let ctx = RecallContext::from_caller(caller_id, &state); - budget = maybe_apply_adaptive_default_budget( - q.trim(), - requested_policy_mode, - query.budget, - budget, - k, - ); + budget = maybe_apply_adaptive_default_budget(q.trim(), requested_policy_mode, query.budget, budget, k); let resolved_policy_mode = recall_mode_for_budget(budget); - - match execute_recall_policy_explain( - &state, - q.trim(), - budget, - k, - &agent, - &ctx, - source_prefix, - pool_k, - ) - .await - { + let ctx = RecallContext::from_caller(caller_id, &state); + match execute_recall_policy_explain(&state, q.trim(), budget, k, &agent, &ctx, source_prefix, pool_k, None).await { Ok(mut payload) => { - if let Value::Object(map) = &mut payload { - map.insert( - "policyMode".to_string(), - Value::String(resolved_policy_mode.as_str().to_string()), - ); - if let Some(mode) = requested_policy_mode { - map.insert( - "requestedPolicyMode".to_string(), - Value::String(mode.as_str().to_string()), - ); - } - } + attach_policy_modes(&mut payload, resolved_policy_mode, requested_policy_mode); json_response(StatusCode::OK, payload) } - Err(err) => json_response( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Recall explain failed: {err}") }), - ), + Err(err) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":format!("Recall explain failed: {err}")})), } } - -// ─── GET /peek ─────────────────────────────────────────────────────────────── - -pub async fn handle_peek( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, +pub async fn handle_peek(State(state): State, headers: HeaderMap, Query(query): Query) -> Response { + let caller_id = match auth_recall_caller(&headers, &state).await { + Ok(id) => id, Err(resp) => return resp, }; let q = match &query.q { Some(q) if !q.trim().is_empty() => q.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({"error": "Missing query parameter: q"}), - ); - } + _ => return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing query parameter: q"})), }; - let source_prefix = query - .source_prefix - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); + let source_prefix = trim_source_prefix(query.source_prefix.as_deref()); let k = query.k.unwrap_or(10); let agent = resolve_source_identity(&headers, query.agent.as_deref().unwrap_or("http")).agent; - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await { return resp; } let ctx = RecallContext::from_caller(caller_id, &state); let mut conn = state.db.lock().await; match run_recall(&mut conn, &q, k, &ctx, source_prefix) { Ok(results) => { - let matches: Vec = results - .iter() - .map(|r| { - json!({ - "source": r.source, - "relevance": r.relevance, - "method": r.method, - }) - }) - .collect(); + let matches: Vec = results.iter().map(|r| json!({"source":r.source,"relevance":r.relevance,"method":r.method})).collect(); let usage = compute_headlines_token_usage(&results); json_response( StatusCode::OK, - json!({ - "count": matches.len(), - "matches": matches, - "tokenUsage": { - "used": usage.spent, - "saved": usage.saved - }, - "tokenUsageLine": format!( - "Token usage: used {} tokens, saved {} vs full recall excerpts.", - usage.spent, usage.saved - ) - }), + json!({"count":matches.len(),"matches":matches,"tokenUsage":{"used":usage.spent,"saved":usage.saved},"tokenUsageLine":format!("Token usage: used {} tokens, saved {} vs full recall excerpts.",usage.spent,usage.saved)}), ) } - Err(e) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error": e})), + Err(e) => json_response(StatusCode::INTERNAL_SERVER_ERROR, json!({"error":e})), + } +} +pub async fn handle_unfold(State(state): State, Query(query): Query, headers: HeaderMap) -> Response { + let caller_id = match auth_recall_caller(&headers, &state).await { + Ok(id) => id, + Err(resp) => return resp, + }; + let ctx = RecallContext::from_caller(caller_id, &state); + let sources_str = match &query.sources { + Some(s) if !s.trim().is_empty() => s.trim().to_string(), + _ => return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing query parameter: sources (comma-separated)"})), + }; + let requested: Vec<&str> = sources_str.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()).collect(); + if requested.is_empty() { + return json_response(StatusCode::BAD_REQUEST, json!({"error":"No valid sources provided"})); + } + if requested.len() > MAX_UNFOLD_SOURCES { + return json_response(StatusCode::BAD_REQUEST, json!({"error":format!("Too many sources (max {MAX_UNFOLD_SOURCES})")})); + } + let agent = resolve_source_identity(&headers, "http").agent; + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await { + return resp; + } + let conn = state.db_read.lock().await; + let mut results: Vec = Vec::new(); + let mut total_tokens = 0usize; + for source in &requested { + if let Some(mut item) = unfold_source(&conn, source, &ctx) { + let tokens = estimate_tokens(item["text"].as_str().unwrap_or("")); + total_tokens += tokens; + if let Value::Object(ref mut map) = item { + if !map.contains_key("source") { + map.insert("source".to_string(), Value::String(source.to_string())); + } + map.insert("tokens".to_string(), Value::Number((tokens as u64).into())); + } + results.push(item); + } else { + results.push(json!({"source":source,"text":null,"type":"not_found","tokens":0})); + } } + json_response(StatusCode::OK, json!({"results":results,"totalTokens":total_tokens,"count":results.iter().filter(|r|r["type"]!="not_found").count(),})) +} +pub(crate) fn extract_recall_node_ids(payload: &Value) -> Vec { + fn walk(v: &Value, out: &mut Vec, limit: usize) { + if out.len() >= limit { + return; + } + match v { + Value::Object(map) => { + if let (Some(target_type), Some(target_id)) = (map.get("type").and_then(|t| t.as_str()), map.get("id").and_then(|t| t.as_i64())) { + if matches!(target_type, "memory" | "decision" | "crystal") { + out.push(format!("{target_type}-{target_id}")); + if out.len() >= limit { + return; + } + } + } + for (_, child) in map.iter() { + walk(child, out, limit); + } + } + Value::Array(arr) => { + for child in arr.iter() { + walk(child, out, limit); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(payload, &mut out, 16); + out } - diff --git a/daemon-rs/src/handlers/recall/mod.rs b/daemon-rs/src/handlers/recall/mod.rs index e81ef081..d7979a39 100644 --- a/daemon-rs/src/handlers/recall/mod.rs +++ b/daemon-rs/src/handlers/recall/mod.rs @@ -1,43 +1,8 @@ -// SPDX-License-Identifier: MIT - -mod budget; -mod cache; -mod core; +mod engine; mod handlers; -mod nlp; -mod pipeline; -mod rerank; -mod scoring; -mod search; -mod semantic; -mod telemetry; -mod types; -mod unfold; - #[cfg(test)] mod tests; - -pub(crate) use budget::*; -pub(crate) use cache::*; -pub(crate) use core::*; -pub(crate) use nlp::*; -pub(crate) use pipeline::*; -pub(crate) use rerank::*; -pub(crate) use scoring::*; -pub(crate) use search::*; -pub(crate) use semantic::*; -pub(crate) use telemetry::*; -pub(crate) use types::*; -pub(crate) use unfold::*; - -pub use handlers::{ - handle_budget_recall, handle_peek, handle_recall, handle_recall_explain, handle_recall_post, - handle_semantic_recall, -}; -pub use unfold::handle_unfold; -pub use pipeline::{ - execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, -}; -pub use types::{parse_recall_policy_mode, resolve_recall_budget_k, RecallContext, RecallPolicyMode}; -pub use types::shannon_entropy; -pub use unfold::unfold_source; +pub(crate) use engine::*; +pub use engine::{execute_recall_policy_explain, execute_semantic_recall, execute_unified_recall, unfold_source}; +pub use engine::{parse_recall_policy_mode, resolve_recall_budget_k, RecallContext, RecallPolicyMode}; +pub use handlers::{handle_budget_recall, handle_peek, handle_recall, handle_recall_explain, handle_recall_post, handle_semantic_recall, handle_unfold}; diff --git a/daemon-rs/src/handlers/recall/nlp.rs b/daemon-rs/src/handlers/recall/nlp.rs deleted file mode 100644 index a24040b5..00000000 --- a/daemon-rs/src/handlers/recall/nlp.rs +++ /dev/null @@ -1,506 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── Text / keyword utilities ──────────────────────────────────────────────── - -pub(crate) fn normalize_text(input: &str) -> String { - input - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || ch == '-' || ch.is_ascii_whitespace() { - ch.to_ascii_lowercase() - } else { - ' ' - } - }) - .collect() -} - -pub(crate) fn extract_keywords(text: &str) -> Vec { - let stop_words: HashSet<&'static str> = [ - "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", - "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can", - "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into", "about", "that", - "this", "it", "its", "not", "but", "and", "or", "if", "then", "so", "what", "which", "who", - "how", "when", "where", "why", "all", "each", "every", "both", "few", "more", "most", - "some", "any", "no", "my", "your", "his", "her", "our", "their", "i", "me", - ] - .into_iter() - .collect(); - - normalize_text(text) - .split_whitespace() - .filter(|word| word.len() > 2 && !stop_words.contains(*word)) - .map(str::to_string) - .collect() -} - -pub(crate) fn extract_search_keywords(text: &str) -> Vec { - normalize_text(text) - .split_whitespace() - .filter(|word| word.len() > 1) - .map(str::to_string) - .collect() -} - -/// Coding synonym map: maps abbreviated/shorthand terms to their full-form equivalents -/// and vice versa. Used during FTS query construction to expand search coverage. -/// -/// Strategy: every token in the query that has a synonym gets BOTH forms added to the -/// OR list. This is directional expansion (short → long, or long → short) -- the map -/// handles both directions as separate entries. -pub(crate) fn coding_synonyms(word: &str) -> Option<&'static str> { - match word { - "func" => Some("function"), - "fn" => Some("function"), - "err" => Some("error"), - "db" => Some("database"), - "auth" => Some("authentication"), - "authn" => Some("authentication"), - "authz" => Some("authorization"), - "cfg" => Some("config"), - "config" => Some("configuration"), - "msg" => Some("message"), - "req" => Some("request"), - "res" => Some("response"), - "resp" => Some("response"), - "impl" => Some("implementation"), - "repo" => Some("repository"), - "env" => Some("environment"), - "var" => Some("variable"), - "arg" => Some("argument"), - "args" => Some("arguments"), - "param" => Some("parameter"), - "params" => Some("parameters"), - "dir" => Some("directory"), - "tmp" => Some("temporary"), - "async" => Some("asynchronous"), - "sync" => Some("synchronous"), - "tx" => Some("transaction"), - "rx" => Some("receive"), - "conn" => Some("connection"), - "stmt" => Some("statement"), - "idx" => Some("index"), - "str" => Some("string"), - "int" => Some("integer"), - "bool" => Some("boolean"), - "vec" => Some("vector"), - "dict" => Some("dictionary"), - "obj" => Some("object"), - "num" => Some("number"), - "char" => Some("character"), - // Personal-memory recall aliases used by real user queries. - "lastname" => Some("surname"), - "surname" => Some("lastname"), - "attend" => Some("attended"), - "attended" => Some("attend"), - "abroad" => Some("overseas"), - "overseas" => Some("abroad"), - "coupon" => Some("voucher"), - "voucher" => Some("coupon"), - "gift" => Some("present"), - "present" => Some("gift"), - "buy" => Some("bought"), - "bought" => Some("buy"), - "repaint" => Some("paint"), - "repainted" => Some("paint"), - "painted" => Some("paint"), - "walls" => Some("wall"), - "wall" => Some("walls"), - "colour" => Some("color"), - "color" => Some("colour"), - "gray" => Some("grey"), - "grey" => Some("gray"), - _ => None, - } -} - -/// Like `extract_search_keywords` but also expands coding synonyms. -/// Each token that has a known synonym produces both the original and the expanded form. -/// Deduplicates the final list while preserving order. -#[cfg(test)] -pub(crate) fn extract_search_keywords_with_synonyms(text: &str) -> Vec { - build_search_term_groups(text) - .into_iter() - .flatten() - .collect() -} - -pub(crate) fn is_low_signal_query_token(token: &str) -> bool { - matches!( - token, - "the" - | "a" - | "an" - | "is" - | "are" - | "was" - | "were" - | "be" - | "been" - | "being" - | "do" - | "does" - | "did" - | "to" - | "of" - | "in" - | "for" - | "on" - | "with" - | "at" - | "by" - | "from" - | "as" - | "into" - | "about" - | "that" - | "this" - | "it" - | "its" - | "my" - | "your" - | "our" - | "their" - | "i" - | "me" - | "we" - | "you" - | "what" - | "which" - | "who" - | "how" - | "when" - | "where" - | "why" - ) -} - -pub(crate) fn query_intent_alias_terms(text: &str) -> Vec { - let lower = normalize_text(text); - let mut aliases = Vec::new(); - if lower.contains("study abroad") { - aliases.extend( - ["attend", "attended", "exchange", "semester"] - .into_iter() - .map(str::to_string), - ); - } - if lower.contains("coupon") && lower.contains("creamer") { - aliases.extend( - ["redeem", "redeemed", "store", "grocery"] - .into_iter() - .map(str::to_string), - ); - } - if lower.contains("birthday") && (lower.contains("gift") || lower.contains("present")) { - aliases.extend( - ["buy", "bought", "item", "present"] - .into_iter() - .map(str::to_string), - ); - } - aliases -} - -pub(crate) fn build_search_term_groups(text: &str) -> Vec> { - let mut base = extract_search_keywords(text); - let profile = query_shape_profile(text, None); - if profile.naturalish && base.len() >= 6 { - let filtered = base - .iter() - .filter(|token| !is_low_signal_query_token(token.as_str())) - .cloned() - .collect::>(); - if !filtered.is_empty() { - base = filtered; - } - } - let mut seen_base = HashSet::new(); - for alias in query_intent_alias_terms(text) { - if seen_base.insert(alias.clone()) && !base.iter().any(|token| token == &alias) { - base.push(alias); - } - } - let mut groups = Vec::with_capacity(base.len()); - for word in base { - let mut group = Vec::with_capacity(2); - let mut seen = HashSet::new(); - if let Some(expanded) = coding_synonyms(&word) { - let expanded = expanded.to_string(); - if seen.insert(expanded.clone()) { - group.push(expanded); - } - } - if seen.insert(word.clone()) { - group.push(word); - } - if !group.is_empty() { - groups.push(group); - } - } - groups -} - -pub(crate) fn count_matching_term_groups(haystacks: &[String], term_groups: &[Vec]) -> i64 { - term_groups - .iter() - .filter(|group| { - group - .iter() - .any(|term| haystacks.iter().any(|haystack| haystack.contains(term))) - }) - .count() as i64 -} - -pub(crate) fn query_focus_terms(query_text: &str) -> Vec { - let mut terms = extract_keywords(query_text); - let mut seen: HashSet = terms.iter().cloned().collect(); - for group in build_search_term_groups(query_text) { - for term in group { - if seen.insert(term.clone()) { - terms.push(term); - } - } - } - if terms.is_empty() { - terms = extract_search_keywords(query_text); - } - terms -} - -pub(crate) fn build_fts_query(groups: &[Vec]) -> String { - groups - .iter() - .map(|group| { - let alternates = group - .iter() - .map(|t| format!("\"{}\"", t.replace('"', "\"\""))) - .collect::>() - .join(" OR "); - if group.len() > 1 { - format!("({alternates})") - } else { - alternates - } - }) - .collect::>() - .join(" AND ") -} - -pub(crate) fn query_focus_terms_for_excerpt(query_text: &str) -> Vec { - let mut seen = HashSet::new(); - let mut terms = query_focus_terms(query_text) - .into_iter() - .filter_map(|term| { - let normalized = term.trim().to_ascii_lowercase(); - if normalized.is_empty() || !seen.insert(normalized.clone()) { - None - } else { - Some(normalized) - } - }) - .collect::>(); - terms.sort_by_key(|t| std::cmp::Reverse(t.len())); - terms -} - -pub(crate) fn excerpt_signature_terms(source: &str, excerpt: &str) -> HashSet { - let mut terms = HashSet::new(); - for token in extract_search_keywords(source) - .into_iter() - .chain(extract_search_keywords(excerpt)) - { - if token.len() > 2 { - terms.insert(token); - } - } - terms -} - -pub(crate) fn term_set_jaccard(a: &HashSet, b: &HashSet) -> f64 { - if a.is_empty() && b.is_empty() { - return 1.0; - } - let intersection = a.intersection(b).count(); - let union = a.union(b).count(); - if union == 0 { - return 0.0; - } - intersection as f64 / union as f64 -} - -pub(crate) fn query_term_coverage_gain( - signature_terms: &HashSet, - query_terms: &HashSet, - covered_terms: &HashSet, -) -> usize { - query_terms - .iter() - .filter(|term| signature_terms.contains(*term) && !covered_terms.contains(*term)) - .count() -} - -pub(crate) fn should_skip_redundant_budget_candidate( - signature_terms: &HashSet, - selected_signatures: &[HashSet], - query_terms: &HashSet, - covered_terms: &HashSet, -) -> bool { - if selected_signatures.is_empty() || signature_terms.is_empty() { - return false; - } - if query_term_coverage_gain(signature_terms, query_terms, covered_terms) > 0 { - return false; - } - let max_similarity = selected_signatures - .iter() - .map(|existing| term_set_jaccard(existing, signature_terms)) - .fold(0.0_f64, f64::max); - max_similarity >= BUDGET_REDUNDANCY_SIMILARITY_THRESHOLD -} - -pub(crate) fn update_query_term_coverage( - signature_terms: &HashSet, - query_terms: &HashSet, - covered_terms: &mut HashSet, -) { - for term in query_terms { - if signature_terms.contains(term) { - covered_terms.insert(term.clone()); - } - } -} - -pub(crate) fn should_early_stop_budget_selection( - token_budget: usize, - spent_tokens: usize, - selected_count: usize, - query_terms: &HashSet, - covered_terms: &HashSet, -) -> bool { - if token_budget == 0 || selected_count < 2 || query_terms.is_empty() { - return false; - } - if covered_terms.len() < query_terms.len() { - return false; - } - let pressure = spent_tokens as f64 / token_budget as f64; - pressure >= BUDGET_PRESSURE_EARLY_STOP_THRESHOLD -} - -pub(crate) fn query_focused_excerpt_with_terms( - text: &str, - sorted_focus_terms: &[String], - max_chars: usize, -) -> String { - if max_chars == 0 || text.is_empty() { - return String::new(); - } - - let total_chars = text.chars().count(); - if total_chars <= max_chars { - return text.to_string(); - } - - let lower_text = text.to_ascii_lowercase(); - if lower_text.contains("[assistant-question]") { - if let Some(answer_byte_idx) = lower_text.find("[user-answer]") { - let answer_char_idx = text[..answer_byte_idx].chars().count(); - let answer_end_char = (answer_char_idx + max_chars).min(total_chars); - let mut answer_excerpt = text - .chars() - .skip(answer_char_idx) - .take(answer_end_char.saturating_sub(answer_char_idx)) - .collect::(); - if !answer_excerpt.trim().is_empty() { - if answer_char_idx > 0 { - answer_excerpt = format!("...{answer_excerpt}"); - } - if answer_end_char < total_chars { - answer_excerpt.push_str("..."); - } - return answer_excerpt; - } - } - } - if sorted_focus_terms.is_empty() { - return truncate_chars(text, max_chars); - } - - let mut hit_byte_idx = None; - for term in sorted_focus_terms { - if let Some(idx) = lower_text.find(term.as_str()) { - hit_byte_idx = Some(idx); - break; - } - } - - let Some(byte_idx) = hit_byte_idx else { - return truncate_chars(text, max_chars); - }; - - let hit_char_idx = text[..byte_idx].chars().count(); - let left_window = max_chars / 3; - let mut start_char = hit_char_idx.saturating_sub(left_window); - let end_char = (start_char + max_chars).min(total_chars); - if end_char - start_char < max_chars { - start_char = end_char.saturating_sub(max_chars); - } - - let mut excerpt = text - .chars() - .skip(start_char) - .take(end_char - start_char) - .collect::(); - if start_char > 0 { - excerpt = format!("...{excerpt}"); - } - if end_char < total_chars { - excerpt.push_str("..."); - } - excerpt -} - -pub(crate) fn query_focused_excerpt(text: &str, query_text: &str, max_chars: usize) -> String { - let terms = query_focus_terms_for_excerpt(query_text); - query_focused_excerpt_with_terms(text, &terms, max_chars) -} - -pub(crate) fn recency_days(value: Option<&str>) -> i64 { - let ts = value.map(parse_timestamp_ms).unwrap_or(0); - if ts == 0 { - return 3650; - } - (Utc::now().timestamp_millis() - ts).max(0) / (24 * 60 * 60 * 1000) -} - diff --git a/daemon-rs/src/handlers/recall/pipeline.rs b/daemon-rs/src/handlers/recall/pipeline.rs deleted file mode 100644 index 97390c4e..00000000 --- a/daemon-rs/src/handlers/recall/pipeline.rs +++ /dev/null @@ -1,1100 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub async fn execute_unified_recall( - state: &RuntimeState, - query_text: &str, - budget: usize, - k: usize, - agent: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Result { - let started_at = Instant::now(); - let policy_mode = recall_mode_for_budget(budget); - let latency_budget_ms = recall_latency_budget_ms_for_mode(policy_mode); - let recall_scope = recall_scope_key(agent, ctx); - let scope_prefix = recall_owner_scope(ctx); - - // Check pre-cache - if budget > 0 && !state.rerank_config.is_active() { - if let Some(cached) = get_pre_cached(state, &recall_scope, &scope_prefix, query_text).await - { - let deduped_cached = dedup_and_mark_served(state, agent, query_text, ctx, cached).await; - let mode = recall_mode_for_budget(budget); - let method_breakdown = build_method_breakdown(&deduped_cached); - let tier = classify_recall_tier(true, mode.as_str(), &method_breakdown); - let latency_ms = started_at.elapsed().as_millis() as i64; - let semantic_route = json!({ - "mode": "baseline", - "reason": "cache_hit", - "sampled": false, - "trialPercent": if matches!( - state.sqlite_vec_canary.effective_route_mode(), - SqliteVecRouteMode::Primary - ) { - 100 - } else { - state.sqlite_vec_canary.trial_percent - }, - "routeMode": state.sqlite_vec_canary.effective_route_mode().as_str() - }); - emit_recall_query_event( - state, - agent, - source_prefix, - json!({ - "agent": agent, - "query": truncate_chars(query_text, 120), - "budget": budget, - "spent": 0, - "saved": budget as i64, - "hits": deduped_cached.len(), - "mode": mode.as_str(), - "cached": true, - "method_breakdown": method_breakdown, - "tier": tier, - "latency_ms": latency_ms, - "semantic_route": semantic_route.clone(), - "shadow_semantic": { - "status": "skipped", - "reason": "cache_hit" - } - }), - ) - .await; - let usage = RecallBudgetUsage { - spent: 0, - saved: budget as i64, - over_budget: false, - }; - return Ok(json!({ - "results": deduped_cached.into_iter().map(recall_to_json).collect::>(), - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "overBudget": usage.over_budget, - "tokenUsageLine": format_recall_token_usage_line(budget, usage), - "mode": mode.as_str(), - "policyMode": mode.as_str(), - "cached": true, - "tier": tier, - "latencyMs": latency_ms, - "semanticRoute": semantic_route - })); - } - } - - let engine = state.embedding_engine.clone(); - let dflag = Some(&state.degraded_mode); - let mut query_vector = match engine { - Some(runtime_engine) => { - runtime_engine - .embed_query_async(query_text.to_string()) - .await - } - None => None, - }; - if state.embedding_engine.is_some() { - update_semantic_search_health(dflag, query_vector.is_some(), true); - } - let mut conn = state.db.lock().await; - let (mut results, mut semantic_baseline, mut semantic_route) = if budget == 0 { - let trace = run_recall_with_query_vector_trace( - &mut conn, - query_text, - k, - query_vector.as_deref(), - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - )?; - (trace.ranked, trace.semantic_baseline, trace.semantic_route) - } else { - let trace = run_budget_recall_trace_with_query_vector( - &mut conn, - query_text, - budget, - k, - query_vector.as_deref(), - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - )?; - ( - trace.budgeted, - trace.semantic_baseline, - trace.semantic_route, - ) - }; - let mut fail_closed = Value::Null; - if budget > 0 { - let elapsed_before_fallback = started_at.elapsed().as_millis(); - if elapsed_before_fallback >= latency_budget_ms { - let fallback_trace = run_budget_recall_trace_with_query_vector( - &mut conn, - query_text, - budget, - k, - None, - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - )?; - results = fallback_trace.budgeted; - semantic_baseline = fallback_trace.semantic_baseline; - semantic_route = json!({ - "mode": "baseline", - "reason": "latency_budget_fail_closed", - "fallback": "deterministic_keyword_rrf", - "elapsedMsBeforeFallback": elapsed_before_fallback, - "latencyBudgetMs": latency_budget_ms, - "routeMode": state.sqlite_vec_canary.effective_route_mode().as_str() - }); - query_vector = None; - fail_closed = json!({ - "triggered": true, - "elapsedMsBeforeFallback": elapsed_before_fallback, - "latencyBudgetMs": latency_budget_ms, - "fallback": "deterministic_keyword_rrf" - }); - } - } - let shadow_semantic = { - let shadow_detail = build_shadow_semantic_explain( - &conn, - query_vector.as_deref(), - query_text, - ctx, - source_prefix, - k, - semantic_baseline.as_ref(), - ); - shadow_semantic_telemetry_summary(&shadow_detail) - }; - let (reranked_results, rerank_route) = maybe_apply_rerank(state, query_text, results, budget); - results = reranked_results; - - // Co-occurrence tracking (recording only -- predictions excluded from response) - let sources: Vec = results.iter().map(|item| item.source.clone()).collect(); - if sources.len() >= 2 { - if co_occurrence::record(&conn, &sources).is_ok() { - checkpoint_wal_best_effort(&conn); - } else { - let _ = co_occurrence::reset(&conn); - } - } - drop(conn); - - // Record recall pattern for prediction - record_recall_pattern(state, &recall_scope, query_text).await; - - // Fire-and-forget pre-cache warming - let state_clone = state.clone(); - let scope_owned = recall_scope.clone(); - let query_owned = query_text.to_string(); - let ctx_owned = *ctx; - tokio::spawn(async move { - let _ = predict_and_cache(state_clone, &scope_owned, &query_owned, ctx_owned).await; - }); - - // Headlines mode (budget == 0) - if budget == 0 { - let method_breakdown = build_method_breakdown(&results); - let tier = classify_recall_tier(false, "headlines", &method_breakdown); - let latency_ms = started_at.elapsed().as_millis() as i64; - let headlines = results - .iter() - .map(|item| { - json!({ - "source": item.source, - "relevance": item.relevance, - "method": item.method - }) - }) - .collect::>(); - let usage = compute_headlines_token_usage(&results); - emit_recall_query_event( - state, - agent, - source_prefix, - json!({ - "agent": agent, - "query": truncate_chars(query_text, 120), - "budget": 0, - "spent": usage.spent, - "saved": usage.saved, - "hits": headlines.len(), - "mode": "headlines", - "cached": false, - "method_breakdown": method_breakdown, - "tier": tier, - "latency_ms": latency_ms, - "latency_budget_ms": latency_budget_ms, - "semantic_route": semantic_route.clone(), - "shadow_semantic": shadow_semantic, - "fail_closed": fail_closed, - "rerank": rerank_route.clone() - }), - ) - .await; - return Ok(json!({ - "count": headlines.len(), - "results": headlines, - "budget": 0, - "spent": usage.spent, - "saved": usage.saved, - "overBudget": usage.over_budget, - "tokenUsageLine": format_recall_token_usage_line(0, usage), - "mode": "headlines", - "policyMode": RecallPolicyMode::Headlines.as_str(), - "tier": tier, - "latencyMs": latency_ms, - "latencyBudgetMs": latency_budget_ms, - "failClosed": fail_closed, - "semanticRoute": semantic_route.clone(), - "rerankRoute": rerank_route - })); - } - - // Dedup and budget accounting - let results = dedup_and_mark_served(state, agent, query_text, ctx, results).await; - let results = enforce_budget_token_invariant(results, budget, query_text); - let usage = compute_recall_budget_usage(&results, budget); - let mode = recall_mode_for_budget(budget); - let method_breakdown = build_method_breakdown(&results); - let tier = classify_recall_tier(false, mode.as_str(), &method_breakdown); - let latency_ms = started_at.elapsed().as_millis() as i64; - emit_recall_query_event( - state, - agent, - source_prefix, - json!({ - "agent": agent, - "query": truncate_chars(query_text, 120), - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "over_budget": usage.over_budget, - "hits": results.len(), - "mode": mode.as_str(), - "cached": false, - "method_breakdown": method_breakdown, - "tier": tier, - "latency_ms": latency_ms, - "latency_budget_ms": latency_budget_ms, - "semantic_route": semantic_route.clone(), - "shadow_semantic": shadow_semantic, - "fail_closed": fail_closed, - "rerank": rerank_route.clone() - }), - ) - .await; - - let payload = json!({ - "results": results.into_iter().map(recall_to_json).collect::>(), - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "overBudget": usage.over_budget, - "tokenUsageLine": format_recall_token_usage_line(budget, usage), - "mode": mode.as_str(), - "policyMode": mode.as_str(), - "tier": tier, - "latencyMs": latency_ms, - "latencyBudgetMs": latency_budget_ms, - "failClosed": fail_closed, - "semanticRoute": semantic_route, - "rerankRoute": rerank_route - }); - - Ok(payload) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn execute_recall_policy_explain_inner( - state: &RuntimeState, - query_text: &str, - budget: usize, - k: usize, - agent: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, - pool_k: usize, - query_vector_override: Option<&[f32]>, -) -> Result { - let requested_k = k.max(1); - let pool_k = pool_k.max(requested_k).min(128); - let engine = state.embedding_engine.clone(); - let dflag = Some(&state.degraded_mode); - let query_vector = match query_vector_override { - Some(vector) => Some(vector.to_vec()), - None => match engine { - Some(runtime_engine) => { - runtime_engine - .embed_query_async(query_text.to_string()) - .await - } - None => None, - }, - }; - if query_vector_override.is_none() && state.embedding_engine.is_some() { - update_semantic_search_health(dflag, query_vector.is_some(), true); - } - let mut conn = state.db.lock().await; - - let ( - budgeted, - candidate_pool, - pre_compaction_candidate_count, - family_compactions, - retrieval_depth, - min_relevance, - top_relevance, - max_items, - semantic_baseline, - semantic_route, - ) = if budget == 0 { - let trace = run_recall_with_query_vector_trace( - &mut conn, - query_text, - pool_k, - query_vector.as_deref(), - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - )?; - let raw_pool = trace.ranked; - let budgeted = raw_pool - .iter() - .take(requested_k) - .cloned() - .map(|mut item| { - item.excerpt.clear(); - item.tokens = Some(estimate_tokens(&item.source)); - item - }) - .collect::>(); - let raw_pool_len = raw_pool.len(); - ( - budgeted, - raw_pool, - raw_pool_len, - Vec::new(), - pool_k, - 0.0_f64, - 0.0_f64, - requested_k, - trace.semantic_baseline, - trace.semantic_route, - ) - } else { - let trace = run_budget_recall_trace_with_query_vector( - &mut conn, - query_text, - budget, - requested_k, - query_vector.as_deref(), - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - )?; - ( - trace.budgeted, - trace.candidate_pool, - trace.pre_compaction_candidate_count, - trace.family_compactions, - trace.retrieval_depth, - trace.min_relevance, - trace.top_relevance, - trace.max_items, - trace.semantic_baseline, - trace.semantic_route, - ) - }; - let shadow_semantic = build_shadow_semantic_explain( - &conn, - query_vector.as_deref(), - query_text, - ctx, - source_prefix, - pool_k, - semantic_baseline.as_ref(), - ); - drop(conn); - - let (budgeted, rerank_route) = maybe_apply_rerank(state, query_text, budgeted, budget); - let final_results = dedup_and_mark_served(state, agent, query_text, ctx, budgeted).await; - let final_results = enforce_budget_token_invariant(final_results, budget, query_text); - let usage = compute_recall_budget_usage(&final_results, budget); - let mode = recall_mode_for_budget(budget); - let family_compacted_count: usize = family_compactions - .iter() - .map(|entry| entry.dropped_sources.len()) - .sum(); - let family_compactions_json: Vec = family_compactions - .iter() - .map(|entry| { - json!({ - "familyKey": entry.family_key, - "keptSource": entry.kept_source, - "droppedSources": entry.dropped_sources, - }) - }) - .collect(); - let returned_sources: HashSet<&str> = final_results - .iter() - .map(|item| item.source.as_str()) - .collect(); - let dropped_candidates: Vec = candidate_pool - .iter() - .filter(|item| !returned_sources.contains(item.source.as_str())) - .take(24) - .map(|item| { - let estimated_tokens = estimate_tokens(&format!("{}{}", item.source, item.excerpt)); - json!({ - "source": item.source, - "relevance": item.relevance, - "method": item.method, - "estimatedTokens": estimated_tokens, - "reason": "not_selected_under_current_budget_or_rank_cutoff" - }) - }) - .collect(); - let query_entities = query_entity_terms(query_text); - let mut entity_metrics_by_source: HashMap = HashMap::new(); - for candidate in &candidate_pool { - let haystack = format!("{} {}", candidate.source, candidate.excerpt); - let (entity_matches, entity_overlap) = - entity_alignment_metrics_with_terms(&haystack, &query_entities); - let entity_boost = entity_signal_boost(entity_matches, entity_overlap); - entity_metrics_by_source.insert( - candidate.source.clone(), - (entity_matches, round4(entity_overlap), round4(entity_boost)), - ); - } - let final_with_factors: Vec = final_results - .clone() - .into_iter() - .enumerate() - .map(|(idx, item)| { - let tokens = item - .tokens - .unwrap_or_else(|| estimate_tokens(&format!("{}{}", item.source, item.excerpt))); - let budget_ratio = if budget == 0 { - 0.0 - } else { - ((tokens as f64) / (budget as f64)).min(1.0) - }; - let (entity_matches, entity_overlap, entity_boost) = entity_metrics_by_source - .get(&item.source) - .copied() - .unwrap_or_else(|| { - let haystack = format!("{} {}", item.source, item.excerpt); - let (matches, overlap) = - entity_alignment_metrics_with_terms(&haystack, &query_entities); - ( - matches, - round4(overlap), - round4(entity_signal_boost(matches, overlap)), - ) - }); - json!({ - "rank": idx + 1, - "source": item.source, - "relevance": item.relevance, - "method": item.method, - "tokens": tokens, - "rankingFactors": { - "relevance": item.relevance, - "method": item.method, - "tokenCost": tokens, - "budgetCostRatio": round4(budget_ratio), - "entropy": item.entropy, - "entityMatches": entity_matches, - "entityOverlap": entity_overlap, - "entityBoost": entity_boost - } - }) - }) - .collect(); - let post_compaction_dropped_count = candidate_pool - .len() - .saturating_sub(final_with_factors.len()); - - Ok(json!({ - "query": query_text, - "results": final_results.into_iter().map(recall_to_json).collect::>(), - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "overBudget": usage.over_budget, - "tokenUsageLine": format_recall_token_usage_line(budget, usage), - "mode": mode.as_str(), - "policyMode": mode.as_str(), - "policy": { - "name": "adaptive-recall-policy", - "mode": mode.as_str(), - "budget": budget, - "requestedK": requested_k, - "poolK": pool_k, - "retrievalDepth": retrieval_depth, - "candidateCutoff": { - "topRelevance": round4(top_relevance), - "minRelevance": round4(min_relevance), - "maxItemsBeforeBudget": max_items - }, - "budgetReasoning": { - "requestedBudget": budget, - "spent": usage.spent, - "saved": usage.saved, - "budgetPressure": if budget == 0 { 0.0 } else { round4((usage.spent as f64) / (budget as f64)) }, - "candidateCountBeforeFamilyCompaction": pre_compaction_candidate_count, - "candidateCount": candidate_pool.len(), - "candidateCountAfterFamilyCompaction": candidate_pool.len(), - "familyCompactedCount": family_compacted_count, - "returnedCount": final_with_factors.len(), - "droppedCount": post_compaction_dropped_count, - "totalPreBudgetDrops": family_compacted_count + post_compaction_dropped_count - }, - "semanticRoute": semantic_route, - "rerankRoute": rerank_route.clone() - }, - "explain": { - "returned": final_with_factors, - "familyCompactions": family_compactions_json, - "droppedCandidates": dropped_candidates, - "shadowSemantic": shadow_semantic, - "rerank": rerank_route - } - })) -} - -#[allow(clippy::too_many_arguments)] -pub async fn execute_recall_policy_explain( - state: &RuntimeState, - query_text: &str, - budget: usize, - k: usize, - agent: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, - pool_k: usize, -) -> Result { - execute_recall_policy_explain_inner( - state, - query_text, - budget, - k, - agent, - ctx, - source_prefix, - pool_k, - None, - ) - .await -} - -pub async fn execute_semantic_recall( - state: &RuntimeState, - query_text: &str, - budget: usize, - k: usize, - agent: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Result { - let started_at = Instant::now(); - let query_vector = match state.embedding_engine.clone() { - Some(engine) => engine.embed_query_async(query_text.to_string()).await, - None => None, - }; - let semantic_available = query_vector.is_some(); - let (budgeted, semantic_route) = { - let conn = state.db.lock().await; - let (results, semantic_route) = run_semantic_recall_with_query_vector( - &conn, - query_text, - k, - query_vector.as_deref(), - ctx, - source_prefix, - Some(&state.sqlite_vec_canary), - ); - ( - apply_semantic_budget(results, budget, query_text), - semantic_route, - ) - }; - let budgeted = enforce_budget_token_invariant(budgeted, budget, query_text); - let usage = compute_recall_budget_usage(&budgeted, budget); - let mode = "semantic"; - let method_breakdown = build_method_breakdown(&budgeted); - let tier = classify_recall_tier(false, mode, &method_breakdown); - let latency_ms = started_at.elapsed().as_millis() as i64; - - emit_recall_query_event( - state, - agent, - source_prefix, - json!({ - "agent": agent, - "query": truncate_chars(query_text, 120), - "mode": mode, - "k": k, - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "over_budget": usage.over_budget, - "hits": budgeted.len(), - "results": budgeted.len(), - "semantic_available": semantic_available, - "cached": false, - "method_breakdown": method_breakdown, - "tier": tier, - "latency_ms": latency_ms, - "semantic_route": semantic_route.clone(), - }), - ) - .await; - - Ok(json!({ - "results": budgeted.into_iter().map(recall_to_json).collect::>(), - "mode": "semantic", - "budget": budget, - "spent": usage.spent, - "saved": usage.saved, - "overBudget": usage.over_budget, - "tokenUsageLine": format_recall_token_usage_line(budget, usage), - "semanticAvailable": semantic_available, - "semanticRoute": semantic_route, - "tier": tier, - "latencyMs": latency_ms, - })) -} - -#[allow(clippy::type_complexity)] -pub(crate) fn run_recall_with_query_vector_trace( - conn: &mut Connection, - query_text: &str, - k: usize, - query_vector: Option<&[f32]>, - ctx: &RecallContext, - source_prefix: Option<&str>, - canary: Option<&SqliteVecCanaryConfig>, -) -> Result { - let extracted = extract_search_keywords(query_text); - let prefers_recency = query_prefers_recency(query_text); - let keyword_query = if extracted.is_empty() { - query_text.to_string() - } else { - extracted.join(" ") - }; - - // ── Tier 0/1: Cache check (handled upstream in execute_unified_recall) ──── - // This function is the retrieval engine; caching is the caller's responsibility. - - // ── Crystal search (highest priority, always runs when engine available) ── - // Crystals bypass Tier 2 early-exit: they represent consolidated knowledge - // and should always surface regardless of FTS confidence. - let scale_sim = |sim: f32| -> f64 { - SEMANTIC_SCALE_BASE - + (sim as f64 - SEMANTIC_SIM_FLOOR) - * ((1.0 - SEMANTIC_SCALE_BASE) / (1.0 - SEMANTIC_SIM_FLOOR)) - }; - - // Crystal results keyed by source. Their member sources are tracked so the - // final merge can collapse near-duplicate family members under the crystal. - let mut crystal_items: HashMap = HashMap::new(); - let mut crystal_family_lookup: HashMap = HashMap::new(); - - if let Some(query_vec) = query_vector { - for (crystal_id, label, text, relevance) in crate::crystallize::search_crystals_filtered( - conn, - query_vec, - 3, - ctx.caller_id, - ctx.team_mode, - ) { - let source = crystal_source(crystal_id, &label); - if !source_matches_prefix(&source, source_prefix) { - continue; - } - let family_members = crystal_member_sources(conn, crystal_id, ctx); - for member_source in &family_members { - crystal_family_lookup - .entry(member_source.clone()) - .or_insert_with(|| source.clone()); - } - crystal_items.insert( - source.clone(), - RecallItem { - source, - relevance: scale_sim(relevance as f32), - excerpt: query_focused_excerpt(&text, query_text, 300), - method: "crystal".to_string(), - tokens: None, - entropy: None, - family_members, - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - }, - ); - } - } - - // ── Tier 2: Keyword-only fast path (ByteRover-inspired) ────────────────── - // Run FTS5 first. If the top result is confident (score >= 0.93) with a - // meaningful gap from #2 (delta >= 0.08), return immediately without - // spending cycles on embedding inference. Target: 40%+ queries resolved here. - const TIER2_CONFIDENCE: f64 = 0.78; - const TIER2_GAP: f64 = 0.10; - - let raw_k = if ctx.team_mode { k.max(10) * 5 } else { 20 }; - let mut fts_limit = raw_k.max(20); - - // Collect keyword candidates for Tier 2 check and later RRF - let kw_candidates: Vec = { - let mut retry = 0; - let mut all: Vec = Vec::new(); - loop { - all.clear(); - for row in search_memories(conn, &keyword_query, fts_limit, source_prefix)? - .into_iter() - .filter(|r| is_visible(r.owner_id, r.visibility.as_deref(), ctx)) - { - all.push(row); - } - for row in search_decisions(conn, &keyword_query, fts_limit, source_prefix)? - .into_iter() - .filter(|r| is_visible(r.owner_id, r.visibility.as_deref(), ctx)) - { - all.push(row); - } - all.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - if ctx.team_mode && all.len() < k && retry < 2 { - fts_limit *= 2; - retry += 1; - continue; - } - break; - } - all - }; - - // Tier 2 early exit: high-confidence keyword result with no close competitor - let required_keyword_hits = if extracted.is_empty() { - 1_i64 - } else { - ((extracted.len() as f64) * 0.6).ceil() as i64 - }; - let tier2_resolved = if let Some(top) = kw_candidates.first() { - let gap = kw_candidates - .get(1) - .map(|next| top.relevance - next.relevance) - .unwrap_or(top.relevance); - top.relevance >= TIER2_CONFIDENCE - && top.matched_keywords >= required_keyword_hits - && gap >= TIER2_GAP - } else { - false - }; - - // ── Semantic search (skipped on Tier 2 early exit or no engine) ────────── - // Produces a ranked list of (source, score) pairs for RRF. - // Also accumulates per-source metadata (score, ts) for compound scoring. - let (semantic_candidates, semantic_route, semantic_baseline) = if tier2_resolved { - ( - Vec::new(), - json!({ - "mode": "baseline", - "reason": "tier2_keyword_resolved", - "sampled": false, - "trialPercent": canary - .map(|config| { - if matches!(config.effective_route_mode(), SqliteVecRouteMode::Primary) { - 100 - } else { - config.trial_percent - } - }) - .unwrap_or(0), - "routeMode": canary - .map(|config| config.effective_route_mode().as_str()) - .unwrap_or("baseline") - }), - None, - ) - } else { - let baseline_semantic = query_vector - .map(|query_vec| { - collect_semantic_candidates(conn, query_vec, query_text, ctx, source_prefix) - }) - .unwrap_or_default(); - let semantic_baseline = if baseline_semantic.is_empty() { - None - } else { - Some(ShadowSemanticBaseline { - candidate_count: baseline_semantic.len(), - ranked_sources: baseline_semantic - .iter() - .take(MAX_SEMANTIC_RRF_CANDIDATES) - .map(|candidate| candidate.source.clone()) - .collect(), - }) - }; - let (semantic_candidates, semantic_route) = maybe_apply_sqlite_vec_trial( - conn, - query_text, - query_vector, - baseline_semantic, - ctx, - source_prefix, - k, - canary, - ); - (semantic_candidates, semantic_route, semantic_baseline) - }; - - // ── RRF fusion ──────────────────────────────────────────────────────────── - // Assign stable integer indices to each unique source across both lists, - // then fuse ranks. rrf_fuse() works on (i64, f64) so we map source → index. - // - // On Tier 2 early exit: semantic list is empty, RRF degrades to keyword-only - // ranking (correct behavior -- no fusion penalty). - let mut source_index: HashMap = HashMap::new(); - let mut index_source: Vec = Vec::new(); - - let mut get_idx = |source: &str| -> i64 { - if let Some(&idx) = source_index.get(source) { - return idx; - } - let idx = index_source.len() as i64; - source_index.insert(source.to_string(), idx); - index_source.push(source.to_string()); - idx - }; - - // Build ranked list for keyword results (sorted by relevance desc) - let kw_list: Vec<(i64, f64)> = kw_candidates - .iter() - .map(|c| (get_idx(&c.source), c.relevance)) - .collect(); - - // Build ranked list for semantic results (sorted by relevance desc) - let sem_list: Vec<(i64, f64)> = semantic_candidates - .iter() - .map(|candidate| (get_idx(&candidate.source), candidate.relevance)) - .collect(); - - let fusion_weights = - adaptive_rrf_weights(query_text, source_prefix, !semantic_candidates.is_empty()); - let fused = rrf_fuse_weighted( - &[kw_list, sem_list], - &[fusion_weights.keyword, fusion_weights.semantic], - 60.0, - ); - - // ── Compound scoring + merge into RecallItem map ────────────────────────── - // For each fused entry: look up metadata from keyword or semantic candidates, - // determine method label, then apply compound_score(). - let mut merged: HashMap = HashMap::new(); - - for (idx, rrf_score) in &fused { - let source = match index_source.get(*idx as usize) { - Some(s) => s.clone(), - None => continue, - }; - - // Prefer keyword candidate metadata (has score + ts); fall back to sem - let (excerpt, importance, ts_ms, method) = - if let Some(kw) = kw_candidates.iter().find(|c| c.source == source) { - let in_sem = semantic_candidates.iter().any(|sem| sem.source == source); - let method = if in_sem { "hybrid" } else { "keyword" }; - (kw.excerpt.clone(), kw.score, kw.ts, method) - } else if let Some(sem) = semantic_candidates.iter().find(|sem| sem.source == source) { - (sem.excerpt.clone(), sem.importance, sem.ts, "semantic") - } else { - continue; - }; - - // Convert ts (Unix-ms) to ISO 8601 for compound_score() - let created_at_str = if ts_ms > 0 { - Utc.timestamp_millis_opt(ts_ms) - .single() - .map(|dt| dt.to_rfc3339()) - .unwrap_or_default() - } else { - String::new() - }; - - // importance is 0-1 in DB; normalize() expects 0-100 range - let mut relevance = round4(compound_score( - *rrf_score, - importance * 100.0, - &created_at_str, - )); - if prefers_recency { - relevance = round4(relevance * temporal_intent_multiplier(ts_ms)); - } - - if let Some(crystal_source) = crystal_family_lookup.get(&source) { - if let Some(crystal_item) = crystal_items.get_mut(crystal_source) { - crystal_item.relevance = round4(crystal_item.relevance.max(relevance)); - if !crystal_item - .collapsed_sources - .iter() - .any(|collapsed| collapsed == &source) - { - crystal_item.collapsed_sources.push(source.clone()); - } - crystal_item - .collapsed_source_scores - .push((source.clone(), relevance)); - if prefer_query_focused_excerpt(&crystal_item.excerpt, &excerpt, query_text) { - crystal_item.excerpt = excerpt.clone(); - } - } - continue; - } - - merged.insert( - source.clone(), - RecallItem { - source, - relevance, - excerpt, - method: method.to_string(), - tokens: None, - entropy: None, - family_members: Vec::new(), - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - }, - ); - } - - // Crystal items bypass RRF (they're already fused/consolidated knowledge); - // insert after -- they will not be overwritten since crystal:: keys don't appear in kw/sem - for (src, mut item) in crystal_items { - dedup_preserve_order(&mut item.family_members); - normalize_collapsed_source_rank(&mut item); - merged.entry(src).or_insert(item); - } - - // ── Entropy-weighted re-ranking ─────────────────────────────────────────── - // High-entropy (information-dense) excerpts get a relevance boost (+/-15% - // around midpoint H=3.5). Applied after compound scoring so entropy acts as - // a diversity signal on top of the RRF+compound base. - let query_entities = query_entity_terms(query_text); - let alignment_profile = QueryAlignmentProfile::from_query(query_text); - let query_focus_term_count = alignment_profile.term_count; - let mut ranked: Vec = merged - .into_values() - .map(|mut item| { - let h = shannon_entropy(&item.excerpt); - item.entropy = Some(round4(h)); - let boost = ((h - 3.5).max(0.0) * 0.08).min(0.12); - item.relevance = round4(item.relevance * (1.0 + boost)); - if !query_entities.is_empty() { - let haystack = format!("{} {}", item.source, item.excerpt); - let (entity_matches, entity_overlap) = - entity_alignment_metrics_with_terms(&haystack, &query_entities); - let entity_boost = entity_signal_boost(entity_matches, entity_overlap); - if entity_boost > 0.0 { - item.relevance = round4(item.relevance * (1.0 + entity_boost)); - } - } - let alignment_boost = query_alignment_boost_with_profile( - &item.source, - &item.excerpt, - &alignment_profile, - query_focus_term_count, - ); - if alignment_boost > 0.0 { - item.relevance = round4(item.relevance * (1.0 + alignment_boost)); - } - item - }) - .collect(); - - // ── Relevance feedback reranking ────────────────────────────────────────── - // Boost results that have been useful in past recalls (unfolded), - // penalize results that were consistently ignored. Graceful no-op when - // no feedback data exists (cold start). - let sources: Vec = ranked.iter().map(|r| r.source.clone()).collect(); - let boosts = crate::handlers::feedback::compute_boosts(conn, &sources, query_vector); - if !boosts.is_empty() { - for item in &mut ranked { - if let Some(&boost) = boosts.get(&item.source) { - item.relevance = round4(item.relevance * (1.0 + boost)); - } - } - } - - ranked.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - ranked.truncate(k); - - bump_retrievals_batch(conn, &ranked); - - Ok(RecallWithVectorTrace { - ranked, - semantic_baseline, - semantic_route, - }) -} - -#[allow(clippy::type_complexity)] -pub(crate) fn run_recall_with_query_vector( - conn: &mut Connection, - query_text: &str, - k: usize, - query_vector: Option<&[f32]>, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Result, String> { - Ok(run_recall_with_query_vector_trace( - conn, - query_text, - k, - query_vector, - ctx, - source_prefix, - None, - )? - .ranked) -} - diff --git a/daemon-rs/src/handlers/recall/rerank.rs b/daemon-rs/src/handlers/recall/rerank.rs deleted file mode 100644 index 25fba03f..00000000 --- a/daemon-rs/src/handlers/recall/rerank.rs +++ /dev/null @@ -1,619 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub(crate) fn rerank_candidate_text(item: &RecallItem) -> String { - let text = if item.excerpt.trim().is_empty() { - item.source.clone() - } else { - format!("{} {}", item.source, item.excerpt) - }; - truncate_chars(&text, 1800) -} - -pub(crate) fn build_rerank_candidates(results: &[RecallItem], top_n: usize) -> Vec { - results - .iter() - .take(top_n.max(1)) - .map(|item| RerankCandidate { - id: item.source.clone(), - text: rerank_candidate_text(item), - base_score: item.relevance, - }) - .collect() -} - -pub(crate) fn remap_fused_score_to_relevance(fused_score: f64, window: &[RecallItem]) -> f64 { - let mut min = f64::INFINITY; - let mut max = f64::NEG_INFINITY; - for relevance in window.iter().map(|item| item.relevance) { - if relevance.is_finite() { - min = min.min(relevance); - max = max.max(relevance); - } - } - if !min.is_finite() || !max.is_finite() { - return round4(fused_score.clamp(0.0, 1.0)); - } - let span = (max - min).max(0.01); - round4(min + (span * fused_score.clamp(0.0, 1.0))) -} - -pub(crate) fn apply_primary_rerank(results: Vec, reranked: &[RerankedScore]) -> Vec { - if reranked.is_empty() { - return results; - } - let window_len = reranked.len().min(results.len()); - let window = &results[..window_len]; - let mut by_source: HashMap = results - .iter() - .take(window_len) - .cloned() - .map(|item| (item.source.clone(), item)) - .collect(); - let mut output = Vec::with_capacity(results.len()); - for score in reranked { - if let Some(mut item) = by_source.remove(&score.id) { - item.relevance = remap_fused_score_to_relevance(score.fused_score, window); - if !item.method.contains("rerank") { - item.method = format!("{}+rerank", item.method); - } - output.push(item); - } - } - for item in results.iter().take(window_len) { - if let Some(item) = by_source.remove(&item.source) { - output.push(item); - } - } - output.extend(results.into_iter().skip(window_len)); - output -} - -pub(crate) fn rerank_scores_json(reranked: &[RerankedScore]) -> Vec { - reranked - .iter() - .take(12) - .enumerate() - .map(|(idx, score)| { - json!({ - "rank": idx + 1, - "source": score.id, - "baseScore": round4(score.base_score), - "rerankScore": round4(score.rerank_score), - "fusedScore": round4(score.fused_score), - }) - }) - .collect() -} - -pub(crate) fn maybe_apply_rerank( - state: &RuntimeState, - query_text: &str, - results: Vec, - budget: usize, -) -> (Vec, Value) { - let config = &state.rerank_config; - if budget == 0 { - return ( - results, - json!({ - "status": "skipped", - "reason": "headlines_mode", - "mode": config.mode.as_str(), - }), - ); - } - if !config.is_active() { - return ( - results, - json!({ - "status": "skipped", - "reason": "mode_off", - "mode": config.mode.as_str(), - }), - ); - } - if results.len() < 2 { - let candidate_count = results.len(); - return ( - results, - json!({ - "status": "skipped", - "reason": "not_enough_candidates", - "mode": config.mode.as_str(), - "candidateCount": candidate_count, - }), - ); - } - let Some(reranker) = state.reranker.as_ref() else { - return ( - results, - json!({ - "status": "unavailable", - "reason": "model_not_loaded", - "mode": config.mode.as_str(), - "configuredModel": crate::rerank::selected_reranker_selection().key, - }), - ); - }; - - let top_n = config.top_n.min(results.len()); - let candidates = build_rerank_candidates(&results, top_n); - let baseline_top_sources = candidates - .iter() - .map(|candidate| candidate.id.clone()) - .collect::>(); - match reranker.rerank(query_text, &candidates, config.fusion_alpha) { - Ok(reranked) => { - let reranked_top_sources = reranked - .iter() - .map(|score| score.id.clone()) - .collect::>(); - let telemetry = json!({ - "status": "ok", - "mode": config.mode.as_str(), - "applied": config.is_primary(), - "model": reranker.name(), - "modelSizeMb": reranker.model_size_mb(), - "topN": top_n, - "fusionAlpha": round4(config.fusion_alpha), - "baselineTopSources": baseline_top_sources, - "rerankedTopSources": reranked_top_sources, - "scores": rerank_scores_json(&reranked), - }); - let results = if config.is_primary() { - apply_primary_rerank(results, &reranked) - } else { - results - }; - (results, telemetry) - } - Err(error) => ( - results, - json!({ - "status": "error", - "mode": config.mode.as_str(), - "applied": false, - "model": reranker.name(), - "reason": truncate_chars(&error, 240), - }), - ), - } -} - -pub(crate) fn sqlite_vec_trial_sampled( - query_text: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, - trial_percent: u8, -) -> bool { - if trial_percent == 0 { - return false; - } - if trial_percent >= 100 { - return true; - } - - let mut hasher = DefaultHasher::new(); - query_text.hash(&mut hasher); - ctx.team_mode.hash(&mut hasher); - ctx.caller_id.hash(&mut hasher); - source_prefix.unwrap_or_default().hash(&mut hasher); - let bucket = (hasher.finish() % 100) as u8; - bucket < trial_percent -} - -pub(crate) fn parse_shadow_sources(shadow_semantic: &Value, field: &str) -> Vec { - shadow_semantic - .get(field) - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter_map(|value| value.as_str().map(ToString::to_string)) - .collect::>() - }) - .unwrap_or_default() -} - -pub(crate) fn shadow_guard_failure_reason(shadow_semantic: &Value) -> Option<&'static str> { - if shadow_semantic - .get("status") - .and_then(Value::as_str) - .unwrap_or("error") - != "ok" - { - return Some("shadow_not_ok"); - } - - let overlap_ratio = shadow_semantic - .get("overlapRatio") - .and_then(Value::as_f64) - .unwrap_or(0.0); - if overlap_ratio < SQLITE_VEC_TRIAL_MIN_OVERLAP_RATIO { - return Some("overlap_ratio_below_gate"); - } - - let jaccard = shadow_semantic - .get("jaccard") - .and_then(Value::as_f64) - .unwrap_or(0.0); - if jaccard < SQLITE_VEC_TRIAL_MIN_JACCARD { - return Some("jaccard_below_gate"); - } - - let mean_abs_rank_delta = shadow_semantic - .get("meanAbsRankDelta") - .and_then(Value::as_f64) - .unwrap_or(f64::INFINITY); - if mean_abs_rank_delta > SQLITE_VEC_TRIAL_MAX_MEAN_ABS_RANK_DELTA { - return Some("rank_delta_above_gate"); - } - - if SQLITE_VEC_TRIAL_TOP1_MATCH_REQUIRED { - let top1_match = shadow_semantic - .get("top1Match") - .and_then(Value::as_bool) - .unwrap_or(false); - if !top1_match { - return Some("top1_match_required"); - } - } - - None -} - -pub(crate) fn sqlite_vec_source_fallback_candidate( - conn: &Connection, - source: &str, - query_text: &str, - fallback_relevance: f64, -) -> Option { - let build_candidate = |excerpt_text: String, - score: Option, - trust_score: Option, - last_accessed: Option, - created_at: Option| { - let ts_source = last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or_default(); - SemanticCandidate { - source: source.to_string(), - excerpt: query_focused_excerpt(&excerpt_text, query_text, 280), - relevance: fallback_relevance, - importance: blend_importance(score, trust_score), - ts: parse_timestamp_ms(ts_source), - } - }; - - let memory_by_id = source - .strip_prefix("memory::") - .and_then(|raw| raw.parse::().ok()) - .and_then(|id| { - conn.query_row( - "SELECT text, score, trust_score, last_accessed, created_at - FROM memories - WHERE id = ?1 AND status = 'active' - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - LIMIT 1", - params![id], - |row| { - Ok(build_candidate( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - )) - }, - ) - .optional() - .ok() - .flatten() - }); - if let Some(candidate) = memory_by_id { - return Some(candidate); - } - - let memory_by_source = conn - .query_row( - "SELECT text, score, trust_score, last_accessed, created_at - FROM memories - WHERE source = ?1 AND status = 'active' - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(build_candidate( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - )) - }, - ) - .optional() - .ok() - .flatten(); - if let Some(candidate) = memory_by_source { - return Some(candidate); - } - - let decision_by_id = source - .strip_prefix("decision::") - .and_then(|raw| raw.parse::().ok()) - .and_then(|id| { - conn.query_row( - "SELECT decision, score, trust_score, last_accessed, created_at - FROM decisions - WHERE id = ?1 AND status = 'active' - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - LIMIT 1", - params![id], - |row| { - Ok(build_candidate( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - )) - }, - ) - .optional() - .ok() - .flatten() - }); - if let Some(candidate) = decision_by_id { - return Some(candidate); - } - - conn.query_row( - "SELECT decision, score, trust_score, last_accessed, created_at - FROM decisions - WHERE context = ?1 AND status = 'active' - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) - ORDER BY COALESCE(last_accessed, created_at) DESC - LIMIT 1", - params![source], - |row| { - Ok(build_candidate( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - )) - }, - ) - .optional() - .ok() - .flatten() -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn maybe_apply_sqlite_vec_trial( - conn: &Connection, - query_text: &str, - query_vector: Option<&[f32]>, - semantic_candidates: Vec, - ctx: &RecallContext, - source_prefix: Option<&str>, - top_k: usize, - canary: Option<&SqliteVecCanaryConfig>, -) -> (Vec, Value) { - let Some(canary) = canary else { - return ( - semantic_candidates, - json!({ - "mode": "baseline", - "reason": "trial_not_configured", - "sampled": false, - "trialPercent": 0, - "routeMode": "baseline" - }), - ); - }; - let effective_route_mode = canary.effective_route_mode(); - let route_mode = effective_route_mode.as_str(); - let active_trial_percent = if matches!(effective_route_mode, SqliteVecRouteMode::Primary) { - 100 - } else { - canary.trial_percent - }; - let baseline_route = |reason: &str, sampled: bool, trial_percent: u8| { - json!({ - "mode": "baseline", - "reason": reason, - "sampled": sampled, - "trialPercent": trial_percent, - "routeMode": route_mode - }) - }; - - if matches!(effective_route_mode, SqliteVecRouteMode::Baseline) { - let reason = if canary.force_off { - "trial_force_off" - } else { - "route_mode_baseline" - }; - return ( - semantic_candidates, - baseline_route(reason, false, active_trial_percent), - ); - } - let Some(query_vector) = query_vector else { - return ( - semantic_candidates, - baseline_route("query_embedding_unavailable", false, active_trial_percent), - ); - }; - if semantic_candidates.is_empty() { - return ( - semantic_candidates, - baseline_route("no_semantic_candidates", false, active_trial_percent), - ); - } - - let sampled = if matches!(effective_route_mode, SqliteVecRouteMode::Trial) { - if canary.trial_percent == 0 { - return ( - semantic_candidates, - baseline_route("trial_percent_zero", false, active_trial_percent), - ); - } - let sampled = - sqlite_vec_trial_sampled(query_text, ctx, source_prefix, canary.trial_percent); - if !sampled { - return ( - semantic_candidates, - baseline_route("not_sampled", false, active_trial_percent), - ); - } - true - } else { - true - }; - - let baseline = ShadowSemanticBaseline { - candidate_count: semantic_candidates.len(), - ranked_sources: semantic_candidates - .iter() - .take(MAX_SEMANTIC_RRF_CANDIDATES) - .map(|candidate| candidate.source.clone()) - .collect(), - }; - let shadow_semantic = build_shadow_semantic_explain( - conn, - Some(query_vector), - query_text, - ctx, - source_prefix, - top_k, - Some(&baseline), - ); - if let Some(reason) = shadow_guard_failure_reason(&shadow_semantic) { - return ( - semantic_candidates, - baseline_route(reason, sampled, active_trial_percent), - ); - } - - let shadow_sources = parse_shadow_sources(&shadow_semantic, "shadowTopSources"); - if shadow_sources.is_empty() { - return ( - semantic_candidates, - baseline_route("shadow_top_sources_empty", sampled, active_trial_percent), - ); - } - - let mut by_source: HashMap = semantic_candidates - .iter() - .cloned() - .map(|candidate| (candidate.source.clone(), candidate)) - .collect(); - let mut reordered: Vec = Vec::new(); - let baseline_max = semantic_candidates - .first() - .map(|candidate| candidate.relevance) - .unwrap_or(SEMANTIC_SCALE_BASE); - let baseline_min = semantic_candidates - .last() - .map(|candidate| candidate.relevance) - .unwrap_or(SEMANTIC_SIM_FLOOR); - let relevance_span = (baseline_max - baseline_min).abs().max(0.02); - let rank_denominator = shadow_sources.len().saturating_sub(1).max(1) as f64; - let fallback_relevance_for_rank = |rank_idx: usize| { - let rank_weight = 1.0 - (rank_idx as f64 / rank_denominator); - round4( - (baseline_min + (relevance_span * rank_weight)) - .clamp(SEMANTIC_SIM_FLOOR, baseline_max.max(SEMANTIC_SIM_FLOOR)), - ) - }; - for (rank_idx, source) in shadow_sources.iter().enumerate() { - if let Some(candidate) = by_source.remove(source) { - reordered.push(candidate); - continue; - } - let fallback_relevance = fallback_relevance_for_rank(rank_idx); - if let Some(candidate) = - sqlite_vec_source_fallback_candidate(conn, source, query_text, fallback_relevance) - { - reordered.push(candidate); - continue; - } - reordered.push(SemanticCandidate { - source: source.clone(), - excerpt: query_focused_excerpt(source, query_text, 160), - relevance: fallback_relevance, - importance: 0.5, - ts: 0, - }); - } - for candidate in &semantic_candidates { - if let Some(remaining) = by_source.remove(&candidate.source) { - reordered.push(remaining); - } - } - reordered.truncate(semantic_candidates.len()); - - ( - reordered, - json!({ - "mode": if matches!(effective_route_mode, SqliteVecRouteMode::Primary) { - "vec0_primary" - } else { - "vec0_trial" - }, - "reason": if matches!(effective_route_mode, SqliteVecRouteMode::Primary) { - "route_mode_primary" - } else { - "guard_passed" - }, - "sampled": sampled, - "trialPercent": active_trial_percent, - "routeMode": route_mode - }), - ) -} - diff --git a/daemon-rs/src/handlers/recall/scoring.rs b/daemon-rs/src/handlers/recall/scoring.rs deleted file mode 100644 index 26942780..00000000 --- a/daemon-rs/src/handlers/recall/scoring.rs +++ /dev/null @@ -1,587 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub(crate) fn blend_importance(score: Option, trust_score: Option) -> f64 { - let score = match score { - Some(value) if value.is_finite() => value.clamp(0.0, 1.0), - Some(_) => 0.0, - None => 1.0, - }; - let trust = match trust_score { - Some(value) if value.is_finite() => value.clamp(0.0, 1.0), - _ => score, - }; - round4((score * 0.65) + (trust * 0.35)) -} - -pub(crate) fn compare_relevance_desc_source_asc( - a_relevance: f64, - a_source: &str, - b_relevance: f64, - b_source: &str, -) -> std::cmp::Ordering { - // NaN/infinite values are treated as the lowest possible relevance so - // fallback ordering stays deterministic and finite scores always win. - let a = if a_relevance.is_finite() { - a_relevance - } else { - f64::NEG_INFINITY - }; - let b = if b_relevance.is_finite() { - b_relevance - } else { - f64::NEG_INFINITY - }; - b.total_cmp(&a).then_with(|| a_source.cmp(b_source)) -} - -#[derive(Clone)] -pub(crate) struct QueryAlignmentProfile { - pub(crate) lower_query: String, - pub(crate) terms: Vec, - pub(crate) term_count: usize, -} - -impl QueryAlignmentProfile { - pub(crate) fn from_query(query_text: &str) -> Self { - let lower_query = query_text.trim().to_ascii_lowercase(); - let mut seen = HashSet::new(); - let mut terms = Vec::new(); - for term in query_focus_terms(query_text) { - let normalized = term.trim().to_ascii_lowercase(); - if normalized.is_empty() { - continue; - } - if seen.insert(normalized.clone()) { - terms.push(normalized); - } - } - let term_count = terms.len().max(1); - Self { - lower_query, - terms, - term_count, - } - } - - pub(crate) fn alignment_score(&self, text: &str) -> (usize, usize) { - if text.is_empty() || self.lower_query.is_empty() { - return (0, 0); - } - let lower_text = text.to_ascii_lowercase(); - let exact_phrase = usize::from(lower_text.contains(&self.lower_query)); - let keyword_hits = self - .terms - .iter() - .filter(|term| lower_text.contains(term.as_str())) - .count(); - (exact_phrase, keyword_hits) - } -} - -pub(crate) fn prefer_query_focused_excerpt_with_profile( - current: &str, - candidate: &str, - profile: &QueryAlignmentProfile, -) -> bool { - let current_score = profile.alignment_score(current); - let candidate_score = profile.alignment_score(candidate); - candidate_score > current_score - || (candidate_score == current_score && candidate.len() < current.len()) -} - -pub(crate) fn prefer_query_focused_excerpt(current: &str, candidate: &str, query_text: &str) -> bool { - let profile = QueryAlignmentProfile::from_query(query_text); - prefer_query_focused_excerpt_with_profile(current, candidate, &profile) -} - -pub(crate) fn query_prefers_recency(query_text: &str) -> bool { - let lower = query_text.to_ascii_lowercase(); - [ - "latest", - "most recent", - "recent", - "newest", - "current", - "today", - "now", - "up to date", - "up-to-date", - ] - .iter() - .any(|needle| lower.contains(needle)) -} - -pub(crate) fn temporal_intent_multiplier(ts_ms: i64) -> f64 { - if ts_ms <= 0 { - return 1.0 - (TEMPORAL_INTENT_MULTIPLIER_RANGE * 0.25); - } - let age_days = - ((Utc::now().timestamp_millis() - ts_ms).max(0) as f64) / (1000.0 * 60.0 * 60.0 * 24.0); - let freshness = (1.0 / (1.0 + age_days / 14.0)).clamp(0.0, 1.0); - 1.0 + ((freshness - 0.5) * TEMPORAL_INTENT_MULTIPLIER_RANGE) -} - -pub(crate) fn query_alignment_boost_with_profile( - source: &str, - excerpt: &str, - profile: &QueryAlignmentProfile, - query_focus_term_count: usize, -) -> f64 { - if profile.lower_query.is_empty() { - return 0.0; - } - let lower_source = source.to_ascii_lowercase(); - let lower_excerpt = excerpt.to_ascii_lowercase(); - let exact_phrase = usize::from( - lower_source.contains(&profile.lower_query) || lower_excerpt.contains(&profile.lower_query), - ); - let keyword_hits = profile - .terms - .iter() - .filter(|term| { - lower_source.contains(term.as_str()) || lower_excerpt.contains(term.as_str()) - }) - .count(); - if exact_phrase == 0 && keyword_hits == 0 { - return 0.0; - } - let term_count = query_focus_term_count.max(1) as f64; - let coverage = (keyword_hits as f64 / term_count).clamp(0.0, 1.0); - let exact_bonus = if exact_phrase > 0 { - ALIGNMENT_EXACT_BONUS_MAX - } else { - 0.0 - }; - let coverage_bonus = - (coverage * ALIGNMENT_COVERAGE_BONUS_MAX).min(ALIGNMENT_COVERAGE_BONUS_MAX); - (exact_bonus + coverage_bonus).min(ALIGNMENT_BOOST_MAX) -} - -pub(crate) fn is_entity_stopword(token: &str) -> bool { - matches!( - token, - "the" - | "a" - | "an" - | "and" - | "or" - | "for" - | "with" - | "from" - | "into" - | "this" - | "that" - | "these" - | "those" - | "what" - | "which" - | "when" - | "where" - | "why" - | "how" - | "about" - | "around" - | "there" - | "their" - | "your" - | "our" - | "have" - | "has" - | "had" - | "will" - | "would" - | "could" - | "should" - ) -} - -pub(crate) fn is_short_technical_term(token: &str) -> bool { - matches!( - token, - "ai" | "ml" - | "db" - | "sql" - | "api" - | "jwt" - | "uid" - | "uuid" - | "id" - | "ip" - | "dns" - | "tls" - | "ssh" - | "http" - | "https" - | "url" - | "ui" - | "ux" - | "cpu" - | "gpu" - | "ram" - | "ios" - | "sdk" - ) -} - -pub(crate) fn extract_entity_like_terms(text: &str) -> HashSet { - let mut terms = HashSet::new(); - for raw in text - .split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.' | '/' | ':'))) - { - let token = raw.trim_matches(|c: char| !c.is_ascii_alphanumeric()); - if token.len() < 3 { - continue; - } - let lowered = token.to_ascii_lowercase(); - if is_entity_stopword(&lowered) { - continue; - } - let has_uppercase = token.chars().any(|c| c.is_ascii_uppercase()); - let has_digit = token.chars().any(|c| c.is_ascii_digit()); - let has_symbol = token - .chars() - .any(|c| matches!(c, '_' | '-' | '.' | '/' | ':')); - let long_specific = lowered.len() >= 9; - if has_uppercase || has_digit || has_symbol || long_specific { - terms.insert(lowered); - } - } - terms -} - -pub(crate) fn query_entity_terms(query_text: &str) -> HashSet { - let mut terms = extract_entity_like_terms(query_text); - if terms.is_empty() { - for term in query_focus_terms(query_text) { - if !is_entity_stopword(&term) && (term.len() >= 3 || is_short_technical_term(&term)) { - terms.insert(term); - } - } - } - terms -} - -pub(crate) fn entity_alignment_metrics_with_terms( - haystack: &str, - query_entities: &HashSet, -) -> (usize, f64) { - if query_entities.is_empty() { - return (0, 0.0); - } - let mut haystack_terms = extract_entity_like_terms(haystack); - if haystack_terms.is_empty() { - for term in extract_search_keywords(haystack) { - if !is_entity_stopword(&term) && (term.len() >= 3 || is_short_technical_term(&term)) { - haystack_terms.insert(term); - } - } - } - if haystack_terms.is_empty() { - return (0, 0.0); - } - let matches = query_entities - .iter() - .filter(|term| haystack_terms.contains(*term)) - .count(); - if matches == 0 { - return (0, 0.0); - } - let overlap = matches as f64 / query_entities.len().max(1) as f64; - (matches, overlap) -} - -pub(crate) fn entity_signal_boost(matches: usize, overlap: f64) -> f64 { - if matches == 0 { - return 0.0; - } - let overlap_component = overlap.clamp(0.0, 1.0) * ENTITY_SIGNAL_OVERLAP_WEIGHT; - let match_component = matches.min(3) as f64 * ENTITY_SIGNAL_MATCH_WEIGHT; - (overlap_component + match_component).min(ENTITY_SIGNAL_MAX_BOOST) -} - -// ─── Jaccard keyword similarity ────────────────────────────────────────────── - -/// Jaccard similarity on whitespace-tokenized keyword sets. -/// -/// Returns |A ∩ B| / |A ∪ B|. Returns 0.0 for empty inputs. -/// Used for Tier-1 fuzzy cache matching: queries with >= 0.6 Jaccard similarity -/// are considered close enough to reuse cached results. -pub(crate) fn jaccard_similarity(a: &str, b: &str) -> f64 { - let set_a: HashSet<&str> = a.split_whitespace().collect(); - let set_b: HashSet<&str> = b.split_whitespace().collect(); - if set_a.is_empty() && set_b.is_empty() { - return 1.0; - } - let intersection = set_a.intersection(&set_b).count(); - let union = set_a.union(&set_b).count(); - if union == 0 { - return 0.0; - } - intersection as f64 / union as f64 -} - -// ─── RRF fusion ────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct FusionWeights { - pub(crate) keyword: f64, - pub(crate) semantic: f64, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct QueryShapeProfile { - pub(crate) exactish: bool, - pub(crate) naturalish: bool, -} - -pub(crate) fn query_shape_profile(query_text: &str, source_prefix: Option<&str>) -> QueryShapeProfile { - let trimmed = query_text.trim(); - let token_count = trimmed.split_whitespace().count(); - let char_count = trimmed.chars().count(); - let lowered = trimmed.to_ascii_lowercase(); - let has_exact_markers = trimmed.contains('"') - || trimmed.contains('`') - || trimmed.contains("::") - || trimmed.contains('/') - || trimmed.contains('\\') - || lowered.contains(".rs") - || lowered.contains(".ts") - || lowered.contains(".tsx") - || lowered.contains(".js") - || lowered.contains(".py"); - QueryShapeProfile { - exactish: has_exact_markers - || token_count <= 3 - || char_count <= 24 - || source_prefix.is_some(), - naturalish: token_count >= 8 || char_count >= 56 || trimmed.ends_with('?'), - } -} - -pub(crate) fn adaptive_rrf_weights( - query_text: &str, - source_prefix: Option<&str>, - semantic_available: bool, -) -> FusionWeights { - if !semantic_available { - return FusionWeights { - keyword: 1.0, - semantic: 0.0, - }; - } - - let profile = query_shape_profile(query_text, source_prefix); - - let mut keyword = 1.0_f64; - let mut semantic = 1.0_f64; - if profile.exactish { - keyword += 0.35; - semantic -= 0.15; - } - if profile.naturalish { - semantic += 0.35; - keyword -= 0.15; - } - - FusionWeights { - keyword: keyword.clamp(0.35, 1.75), - semantic: semantic.clamp(0.35, 1.75), - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct FallbackRankingWeights { - pub(crate) keyword: f64, - pub(crate) score: f64, - pub(crate) recency: f64, - pub(crate) retrieval: f64, -} - -pub(crate) fn adaptive_fallback_ranking_weights( - query_text: &str, - term_group_count: usize, -) -> FallbackRankingWeights { - let profile = query_shape_profile(query_text, None); - let mut keyword = 0.40_f64; - let mut score = 0.25_f64; - let mut recency = 0.20_f64; - let mut retrieval = 0.15_f64; - - if profile.exactish && !profile.naturalish { - keyword += 0.12; - score -= 0.03; - recency -= 0.05; - retrieval -= 0.04; - } else if profile.naturalish && !profile.exactish { - keyword -= 0.08; - score += 0.05; - recency += 0.02; - retrieval += 0.01; - } - - if term_group_count <= 1 { - keyword += 0.05; - score += 0.01; - recency -= 0.03; - retrieval -= 0.03; - } else if term_group_count >= 5 { - keyword -= 0.04; - score += 0.02; - recency += 0.01; - retrieval += 0.01; - } - - keyword = keyword.max(0.05); - score = score.max(0.05); - recency = recency.max(0.05); - retrieval = retrieval.max(0.05); - - let total = keyword + score + recency + retrieval; - FallbackRankingWeights { - keyword: keyword / total, - score: score / total, - recency: recency / total, - retrieval: retrieval / total, - } -} - -pub(crate) fn fallback_ranking_score( - query_text: &str, - term_group_count: usize, - matched: i64, - effective_score: f64, - recency_days: i64, - retrievals: Option, -) -> f64 { - let keyword_weight = if term_group_count == 0 { - 0.0 - } else { - matched as f64 / term_group_count as f64 - }; - let recency_weight = 1.0 / (1.0 + recency_days.max(0) as f64 / 7.0); - let retrieval_weight = (retrievals.unwrap_or(0).clamp(0, 20) as f64) / 20.0; - let score_weight = effective_score.clamp(0.0, 1.0); - let weights = adaptive_fallback_ranking_weights(query_text, term_group_count); - (keyword_weight * weights.keyword) - + (score_weight * weights.score) - + (recency_weight * weights.recency) - + (retrieval_weight * weights.retrieval) -} - -/// Weighted Reciprocal Rank Fusion (Cormack et al., 2009). -/// -/// Fuses multiple ranked lists into a single list using the formula: -/// score(item) = Σ weight / (k + rank + 1) for each list containing item -/// -/// `k = 60.0` is the standard value from the original paper. -/// Items only in one list still accumulate their 1/(k+1) score. -/// Returns results sorted by fused score descending. -/// -/// # Arguments -/// * `lists` -- slice of ranked lists, each a `Vec<(id, score)>` in descending score order -/// * `weights` -- per-list weights in the same order as `lists` -/// * `k` -- smoothing constant (use `60.0` per Cormack et al.) -/// -pub(crate) fn rrf_fuse_weighted(lists: &[Vec<(i64, f64)>], weights: &[f64], k: f64) -> Vec<(i64, f64)> { - let smooth_k = if k.is_finite() && k >= 0.0 { k } else { 60.0 }; - let mut fused: HashMap = HashMap::new(); - for (list_index, list) in lists.iter().enumerate() { - let weight = match weights.get(list_index).copied() { - Some(value) if value.is_finite() => value.max(0.0), - Some(_) => 0.0, - None => 1.0, - }; - if weight == 0.0 { - continue; - } - for (rank, &(id, _score)) in list.iter().enumerate() { - *fused.entry(id).or_insert(0.0) += weight / (smooth_k + rank as f64 + 1.0); - } - } - let mut result: Vec<(i64, f64)> = fused.into_iter().collect(); - result.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - result -} - -#[cfg(test)] -pub(crate) fn rrf_fuse(lists: &[Vec<(i64, f64)>], k: f64) -> Vec<(i64, f64)> { - let default_weights = vec![1.0; lists.len()]; - rrf_fuse_weighted(lists, &default_weights, k) -} - -// ─── Compound scoring (Task 1.4) ───────────────────────────────────────────── - -/// Calculate elapsed days since an ISO 8601 timestamp. -/// Returns days as f64, handling invalid timestamps gracefully (returns very large value). -pub(crate) fn days_since(created_at: &str) -> f64 { - match chrono::DateTime::parse_from_rfc3339(created_at) { - Ok(dt) => { - let now = chrono::Utc::now(); - let duration = now.signed_duration_since(dt); - duration.num_days() as f64 + (duration.num_seconds() as f64 % 86400.0) / 86400.0 - } - Err(_) => f64::MAX, // Invalid timestamp: treat as very old - } -} - -/// Normalize importance score to 0.0-1.0 range. -/// Legacy records may use 0-100, while current records use 0-1. -pub(crate) fn normalize(importance: f64) -> f64 { - if !importance.is_finite() { - return 0.0; - } - let clamped = importance.clamp(0.0, 100.0); - if clamped <= 1.0 { - clamped - } else { - clamped / 100.0 - } -} - -/// Calculate compound score combining RRF rank, importance, and recency. -/// Formula: compound = rrf * 0.6 + importance_norm * 0.2 + recency * 0.2 -/// Recency follows 21-day half-life: exp(-days/30) -/// -/// # Arguments -/// * `rrf` -- fused RRF score from rrf_fuse() -/// * `importance` -- DB score field (typically 0-100) -/// * `created_at` -- ISO 8601 timestamp string -/// -/// Returns compound score in 0.0-1.0 range (approximately) -pub(crate) fn compound_score(rrf: f64, importance: f64, created_at: &str) -> f64 { - let days = days_since(created_at); - let recency = (-days / 30.0).exp(); // 21-day half-life - let importance_normalized = normalize(importance); - rrf * 0.6 + importance_normalized * 0.2 + recency * 0.2 -} - diff --git a/daemon-rs/src/handlers/recall/search.rs b/daemon-rs/src/handlers/recall/search.rs deleted file mode 100644 index 738c2b10..00000000 --- a/daemon-rs/src/handlers/recall/search.rs +++ /dev/null @@ -1,713 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── Search helpers ────────────────────────────────────────────────────────── - -pub(crate) fn search_memories( - conn: &Connection, - query_text: &str, - limit: usize, - source_prefix: Option<&str>, -) -> Result, String> { - let term_groups = build_search_term_groups(query_text); - let excerpt_focus_terms = query_focus_terms_for_excerpt(query_text); - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - - if term_groups.is_empty() { - let mut stmt = conn - .prepare( - "SELECT id, text, source, tags, score, trust_score, retrievals, last_accessed, created_at, compressed_text, age_tier \ - FROM memories WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - AND (?2 IS NULL OR COALESCE(source, 'memory::' || id) LIKE ?2) \ - ORDER BY COALESCE(last_accessed, created_at) DESC LIMIT ?1", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map(params![limit as i64, source_like.as_deref()], |row| { - let text: String = row.get(1)?; - let compressed: Option = row.get(9)?; - let age_tier: String = row - .get::<_, Option>(10)? - .unwrap_or_else(|| "fresh".to_string()); - let display = crate::aging::get_display_text(&text, &compressed, &age_tier); - let effective_score = - blend_importance(row.get::<_, Option>(4)?, row.get::<_, Option>(5)?); - Ok(SearchCandidate { - source: row.get::<_, Option>(2)?.unwrap_or_else(|| { - format!("memory::{}", row.get::<_, i64>(0).unwrap_or(0)) - }), - excerpt: query_focused_excerpt_with_terms(&display, &excerpt_focus_terms, 220), - alignment: (0, 0), - relevance: round4(0.5 * effective_score), - matched_keywords: 0, - score: effective_score, - ts: parse_timestamp_ms( - &row.get::<_, Option>(7)? - .or(row.get::<_, Option>(8)?) - .unwrap_or_default(), - ), - owner_id: None, - visibility: None, - }) - }) - .map_err(|e| e.to_string())?; - - return Ok(rows - .flatten() - .filter(|row| source_matches_prefix(&row.source, source_prefix)) - .collect()); - } - - let fts_query = build_fts_query(&term_groups); - let bm25 = bm25_weights(); - - let fts_result: Result, String> = (|| { - // Field-boosted BM25: memories_fts columns are (text, source, tags). - // Weight tuning favors rich content matches while preserving useful source/tag - // signal for code paths and metadata lookups. - // bm25() returns negative values (more negative = better match), so ORDER BY ASC. - let mut stmt = conn - .prepare( - "SELECT m.id, m.text, m.source, m.tags, m.score, m.trust_score, m.retrievals, m.last_accessed, m.created_at, m.compressed_text, m.age_tier, m.owner_id, m.visibility \ - FROM memories_fts fts \ - JOIN memories m ON m.id = fts.rowid \ - WHERE memories_fts MATCH ?1 AND m.status = 'active' \ - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) \ - AND (?6 IS NULL OR COALESCE(m.source, 'memory::' || m.id) LIKE ?6) \ - ORDER BY bm25(memories_fts, ?3, ?4, ?5) \ - LIMIT ?2", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map( - params![ - &fts_query, - limit as i64, - bm25.memories_text, - bm25.memories_source, - bm25.memories_tags, - source_like.as_deref() - ], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - row.get::<_, Option>(9)?, - row.get::<_, Option>(10)?, - row.get::<_, Option>(11)?, - row.get::<_, Option>(12)?, - )) - }, - ) - .map_err(|e| e.to_string())?; - - let mut ranked = Vec::new(); - for row in rows.flatten() { - let ( - id, - text, - source, - tags, - score, - trust_score, - retrievals, - last_accessed, - created_at, - compressed_text, - age_tier, - row_owner_id, - row_visibility, - ) = row; - let source_key = source - .as_deref() - .map(str::to_owned) - .unwrap_or_else(|| format!("memory::{id}")); - if !source_matches_prefix(&source_key, source_prefix) { - continue; - } - let effective_score = blend_importance(score, trust_score); - let ts = parse_timestamp_ms( - last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or(""), - ); - let display = crate::aging::get_display_text( - &text, - &compressed_text, - age_tier.as_deref().unwrap_or("fresh"), - ); - - let haystacks = [ - text.to_lowercase(), - source.as_deref().unwrap_or("").to_lowercase(), - tags.as_deref().unwrap_or("").to_lowercase(), - ]; - let matched = count_matching_term_groups(&haystacks, &term_groups); - let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); - let ranking = fallback_ranking_score( - query_text, - term_groups.len(), - matched, - effective_score, - recency_d, - retrievals, - ); - - ranked.push(SearchCandidate { - source: source_key, - excerpt: query_focused_excerpt_with_terms(&display, &excerpt_focus_terms, 280), - alignment: (0, 0), - relevance: round4(ranking), - matched_keywords: matched, - score: effective_score, - ts, - owner_id: row_owner_id, - visibility: row_visibility, - }); - } - - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then(b.matched_keywords.cmp(&a.matched_keywords)) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then_with(|| a.source.cmp(&b.source)) - }); - - ranked.truncate(limit); - Ok(ranked) - })(); - - match fts_result { - Ok(results) if !results.is_empty() => Ok(results), - _ => search_memories_fallback(conn, query_text, limit, source_prefix), - } -} - -pub(crate) fn search_memories_fallback( - conn: &Connection, - query_text: &str, - limit: usize, - source_prefix: Option<&str>, -) -> Result, String> { - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - let mut stmt = conn - .prepare( - "SELECT id, text, source, tags, score, trust_score, retrievals, last_accessed, created_at \ - FROM memories WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - AND (?1 IS NULL OR COALESCE(source, 'memory::' || id) LIKE ?1)", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map(params![source_like.as_deref()], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - )) - }) - .map_err(|e| e.to_string())?; - - let term_groups = build_search_term_groups(query_text); - let excerpt_focus_terms = query_focus_terms_for_excerpt(query_text); - let alignment_profile = QueryAlignmentProfile::from_query(query_text); - let mut ranked = Vec::new(); - - for row in rows.flatten() { - let (id, text, source, tags, score, trust_score, retrievals, last_accessed, created_at) = - row; - let source_key = source - .as_deref() - .map(str::to_owned) - .unwrap_or_else(|| format!("memory::{id}")); - if !source_matches_prefix(&source_key, source_prefix) { - continue; - } - let effective_score = blend_importance(score, trust_score); - let ts = parse_timestamp_ms( - last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or(""), - ); - - if term_groups.is_empty() { - let excerpt = query_focused_excerpt_with_terms(&text, &excerpt_focus_terms, 220); - ranked.push(SearchCandidate { - source: source_key, - alignment: alignment_profile.alignment_score(&excerpt), - excerpt, - relevance: round4(0.5 * effective_score), - matched_keywords: 0, - score: effective_score, - ts, - owner_id: None, - visibility: None, - }); - continue; - } - - let haystacks = [ - text.to_lowercase(), - source.as_deref().unwrap_or("").to_lowercase(), - tags.as_deref().unwrap_or("").to_lowercase(), - ]; - - let matched = count_matching_term_groups(&haystacks, &term_groups); - if matched == 0 { - continue; - } - - let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); - let ranking = fallback_ranking_score( - query_text, - term_groups.len(), - matched, - effective_score, - recency_d, - retrievals, - ); - - let excerpt = query_focused_excerpt_with_terms(&text, &excerpt_focus_terms, 260); - ranked.push(SearchCandidate { - source: source_key, - alignment: alignment_profile.alignment_score(&excerpt), - excerpt, - relevance: round4(ranking), - matched_keywords: matched, - score: effective_score, - ts, - owner_id: None, - visibility: None, - }); - } - - if term_groups.is_empty() { - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then(b.alignment.cmp(&a.alignment)) - .then_with(|| a.source.cmp(&b.source)) - }); - } else { - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then(b.matched_keywords.cmp(&a.matched_keywords)) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then(b.alignment.cmp(&a.alignment)) - .then_with(|| a.source.cmp(&b.source)) - }); - } - - ranked.truncate(limit); - Ok(ranked) -} - -pub(crate) fn search_decisions( - conn: &Connection, - query_text: &str, - limit: usize, - source_prefix: Option<&str>, -) -> Result, String> { - let term_groups = build_search_term_groups(query_text); - let excerpt_focus_terms = query_focus_terms_for_excerpt(query_text); - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - - if term_groups.is_empty() { - let mut stmt = conn - .prepare( - "SELECT id, decision, context, score, trust_score, retrievals, last_accessed, created_at \ - FROM decisions WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - AND (?2 IS NULL OR COALESCE(context, 'decision::' || id) LIKE ?2) \ - ORDER BY COALESCE(last_accessed, created_at) DESC LIMIT ?1", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map(params![limit as i64, source_like.as_deref()], |row| { - let effective_score = - blend_importance(row.get::<_, Option>(3)?, row.get::<_, Option>(4)?); - Ok(SearchCandidate { - source: row.get::<_, Option>(2)?.unwrap_or_else(|| { - format!("decision::{}", row.get::<_, i64>(0).unwrap_or(0)) - }), - excerpt: query_focused_excerpt_with_terms( - &row.get::<_, String>(1)?, - &excerpt_focus_terms, - 220, - ), - alignment: (0, 0), - relevance: round4(0.5 * effective_score), - matched_keywords: 0, - score: effective_score, - ts: parse_timestamp_ms( - &row.get::<_, Option>(6)? - .or(row.get::<_, Option>(7)?) - .unwrap_or_default(), - ), - owner_id: None, - visibility: None, - }) - }) - .map_err(|e| e.to_string())?; - - return Ok(rows - .flatten() - .filter(|row| source_matches_prefix(&row.source, source_prefix)) - .collect()); - } - - let fts_query = build_fts_query(&term_groups); - let bm25 = bm25_weights(); - - let fts_result: Result, String> = (|| { - // Field-boosted BM25: decisions_fts columns are (decision, context). - // Decision text carries most signal; context acts as a secondary anchor. - let mut stmt = conn - .prepare( - "SELECT d.id, d.decision, d.context, d.score, d.trust_score, d.retrievals, d.last_accessed, d.created_at, d.compressed_text, d.age_tier, d.owner_id, d.visibility \ - FROM decisions_fts fts \ - JOIN decisions d ON d.id = fts.rowid \ - WHERE decisions_fts MATCH ?1 AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) \ - AND (?5 IS NULL OR COALESCE(d.context, 'decision::' || d.id) LIKE ?5) \ - ORDER BY bm25(decisions_fts, ?3, ?4) \ - LIMIT ?2", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map( - params![ - &fts_query, - limit as i64, - bm25.decisions_text, - bm25.decisions_context, - source_like.as_deref() - ], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - row.get::<_, Option>(8)?, - row.get::<_, Option>(9)?, - row.get::<_, Option>(10)?, - row.get::<_, Option>(11)?, - )) - }, - ) - .map_err(|e| e.to_string())?; - - let mut ranked = Vec::new(); - for row in rows.flatten() { - let ( - id, - decision, - context, - score, - trust_score, - retrievals, - last_accessed, - created_at, - compressed_text, - age_tier, - row_owner_id, - row_visibility, - ) = row; - let source_key = context - .as_deref() - .map(str::to_owned) - .unwrap_or_else(|| format!("decision::{id}")); - if !source_matches_prefix(&source_key, source_prefix) { - continue; - } - let effective_score = blend_importance(score, trust_score); - let ts = parse_timestamp_ms( - last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or(""), - ); - let display = crate::aging::get_display_text( - &decision, - &compressed_text, - age_tier.as_deref().unwrap_or("fresh"), - ); - - let haystacks = [ - decision.to_lowercase(), - context.as_deref().unwrap_or("").to_lowercase(), - ]; - let matched = count_matching_term_groups(&haystacks, &term_groups); - let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); - let ranking = fallback_ranking_score( - query_text, - term_groups.len(), - matched, - effective_score, - recency_d, - retrievals, - ); - - ranked.push(SearchCandidate { - source: source_key, - excerpt: query_focused_excerpt_with_terms(&display, &excerpt_focus_terms, 280), - alignment: (0, 0), - relevance: round4(ranking), - matched_keywords: matched, - score: effective_score, - ts, - owner_id: row_owner_id, - visibility: row_visibility, - }); - } - - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then(b.matched_keywords.cmp(&a.matched_keywords)) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then_with(|| a.source.cmp(&b.source)) - }); - - ranked.truncate(limit); - Ok(ranked) - })(); - - match fts_result { - Ok(results) if !results.is_empty() => Ok(results), - _ => search_decisions_fallback(conn, query_text, limit, source_prefix), - } -} - -pub(crate) fn search_decisions_fallback( - conn: &Connection, - query_text: &str, - limit: usize, - source_prefix: Option<&str>, -) -> Result, String> { - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - let mut stmt = conn - .prepare( - "SELECT id, decision, context, score, trust_score, retrievals, last_accessed, created_at \ - FROM decisions WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - AND (?1 IS NULL OR COALESCE(context, 'decision::' || id) LIKE ?1)", - ) - .map_err(|e| e.to_string())?; - - let rows = stmt - .query_map(params![source_like.as_deref()], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - )) - }) - .map_err(|e| e.to_string())?; - - let term_groups = build_search_term_groups(query_text); - let excerpt_focus_terms = query_focus_terms_for_excerpt(query_text); - let alignment_profile = QueryAlignmentProfile::from_query(query_text); - let mut ranked = Vec::new(); - - for row in rows.flatten() { - let (id, decision, context, score, trust_score, retrievals, last_accessed, created_at) = - row; - let source_key = context - .as_deref() - .map(str::to_owned) - .unwrap_or_else(|| format!("decision::{id}")); - if !source_matches_prefix(&source_key, source_prefix) { - continue; - } - let effective_score = blend_importance(score, trust_score); - let ts = parse_timestamp_ms( - last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or(""), - ); - - if term_groups.is_empty() { - let excerpt = query_focused_excerpt_with_terms(&decision, &excerpt_focus_terms, 220); - ranked.push(SearchCandidate { - source: source_key, - alignment: alignment_profile.alignment_score(&excerpt), - excerpt, - relevance: round4(0.5 * effective_score), - matched_keywords: 0, - score: effective_score, - ts, - owner_id: None, - visibility: None, - }); - continue; - } - - let haystacks = [ - decision.to_lowercase(), - context.as_deref().unwrap_or("").to_lowercase(), - ]; - let matched = count_matching_term_groups(&haystacks, &term_groups); - if matched == 0 { - continue; - } - - let recency_d = recency_days(last_accessed.as_deref().or(created_at.as_deref())); - let ranking = fallback_ranking_score( - query_text, - term_groups.len(), - matched, - effective_score, - recency_d, - retrievals, - ); - - let excerpt = query_focused_excerpt_with_terms(&decision, &excerpt_focus_terms, 260); - ranked.push(SearchCandidate { - source: source_key, - alignment: alignment_profile.alignment_score(&excerpt), - excerpt, - relevance: round4(ranking), - matched_keywords: matched, - score: effective_score, - ts, - owner_id: None, - visibility: None, - }); - } - - if term_groups.is_empty() { - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then(b.alignment.cmp(&a.alignment)) - .then_with(|| a.source.cmp(&b.source)) - }); - } else { - ranked.sort_by(|a, b| { - b.relevance - .partial_cmp(&a.relevance) - .unwrap_or(std::cmp::Ordering::Equal) - .then(b.matched_keywords.cmp(&a.matched_keywords)) - .then( - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal), - ) - .then(b.ts.cmp(&a.ts)) - .then(b.alignment.cmp(&a.alignment)) - .then_with(|| a.source.cmp(&b.source)) - }); - } - - ranked.truncate(limit); - Ok(ranked) -} - diff --git a/daemon-rs/src/handlers/recall/semantic.rs b/daemon-rs/src/handlers/recall/semantic.rs deleted file mode 100644 index eb19c384..00000000 --- a/daemon-rs/src/handlers/recall/semantic.rs +++ /dev/null @@ -1,690 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -pub(crate) fn collect_semantic_candidates( - conn: &Connection, - query_vector: &[f32], - query_text: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, -) -> Vec { - let selected_model = crate::embeddings::selected_model_key(); - let expected_vector_bytes = std::mem::size_of_val(query_vector) as i64; - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - let scale_sim = |sim: f32| -> f64 { - SEMANTIC_SCALE_BASE - + (sim as f64 - SEMANTIC_SIM_FLOOR) - * ((1.0 - SEMANTIC_SCALE_BASE) / (1.0 - SEMANTIC_SIM_FLOOR)) - }; - let keyword_terms = extract_search_keywords(query_text); - let semantic_floor = if keyword_terms.len() >= 3 { - SEMANTIC_SIM_FLOOR + 0.12 - } else { - SEMANTIC_SIM_FLOOR - }; - - let mut candidates: HashMap = HashMap::new(); - - let semantic_memory_query_with_acl = "SELECT e.vector, m.text, m.source, m.owner_id, m.visibility, m.score, m.trust_score, m.last_accessed, m.created_at \ - FROM embeddings e \ - JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id AND m.status = 'active' \ - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR m.source LIKE ?3)"; - let semantic_memory_query_without_acl = "SELECT e.vector, m.text, m.source, NULL AS owner_id, NULL AS visibility, m.score, m.trust_score, m.last_accessed, m.created_at \ - FROM embeddings e \ - JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id AND m.status = 'active' \ - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR m.source LIKE ?3)"; - let semantic_memory_stmt = match conn.prepare(semantic_memory_query_with_acl) { - Ok(stmt) => Some(stmt), - Err(err) if is_missing_team_visibility_columns(&err) => { - conn.prepare(semantic_memory_query_without_acl).ok() - } - Err(_) => None, - }; - if let Some(mut stmt) = semantic_memory_stmt { - if let Ok(rows) = stmt.query_map( - params![ - selected_model, - expected_vector_bytes, - source_like.as_deref() - ], - |row| -> rusqlite::Result { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - row.get(6)?, - row.get(7)?, - row.get(8)?, - )) - }, - ) { - for ( - blob, - text, - source, - owner_id, - visibility, - score, - trust_score, - last_accessed, - created_at, - ) in rows.flatten() - { - if !is_visible(owner_id, visibility.as_deref(), ctx) { - continue; - } - if !source_matches_prefix(&source, source_prefix) { - continue; - } - let existing_vec = crate::embeddings::blob_to_vector(&blob); - let sim = crate::embeddings::cosine_similarity(query_vector, &existing_vec); - if sim <= semantic_floor as f32 { - continue; - } - - let mut scaled = scale_sim(sim); - if !keyword_terms.is_empty() { - let haystack = text.to_lowercase(); - let overlap = keyword_terms - .iter() - .filter(|term| haystack.contains(term.as_str())) - .count(); - if overlap == 0 { - scaled *= 0.82; - } else { - let ratio = overlap as f64 / keyword_terms.len().max(1) as f64; - scaled *= 1.0 + ratio * 0.08; - } - } - let excerpt = query_focused_excerpt(&text, query_text, 280); - let importance = blend_importance(score, trust_score); - let ts_source = last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or_default(); - let ts = parse_timestamp_ms(ts_source); - let entry = candidates - .entry(source.clone()) - .or_insert(SemanticCandidate { - source, - excerpt: excerpt.clone(), - relevance: scaled, - importance, - ts, - }); - if scaled > entry.relevance { - *entry = SemanticCandidate { - source: entry.source.clone(), - excerpt, - relevance: scaled, - importance, - ts, - }; - } - } - } - } - - let semantic_decision_query_with_acl = "SELECT e.vector, d.decision, d.context, d.owner_id, d.visibility, d.score, d.trust_score, d.last_accessed, d.created_at \ - FROM embeddings e \ - JOIN decisions d ON e.target_type = 'decision' AND e.target_id = d.id AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR d.context LIKE ?3)"; - let semantic_decision_query_without_acl = "SELECT e.vector, d.decision, d.context, NULL AS owner_id, NULL AS visibility, d.score, d.trust_score, d.last_accessed, d.created_at \ - FROM embeddings e \ - JOIN decisions d ON e.target_type = 'decision' AND e.target_id = d.id AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR d.context LIKE ?3)"; - let semantic_decision_stmt = match conn.prepare(semantic_decision_query_with_acl) { - Ok(stmt) => Some(stmt), - Err(err) if is_missing_team_visibility_columns(&err) => { - conn.prepare(semantic_decision_query_without_acl).ok() - } - Err(_) => None, - }; - if let Some(mut stmt) = semantic_decision_stmt { - if let Ok(rows) = stmt.query_map( - params![ - selected_model, - expected_vector_bytes, - source_like.as_deref() - ], - |row| -> rusqlite::Result { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - row.get(6)?, - row.get(7)?, - row.get(8)?, - )) - }, - ) { - for ( - blob, - decision, - context, - owner_id, - visibility, - score, - trust_score, - last_accessed, - created_at, - ) in rows.flatten() - { - if !is_visible(owner_id, visibility.as_deref(), ctx) { - continue; - } - let existing_vec = crate::embeddings::blob_to_vector(&blob); - let sim = crate::embeddings::cosine_similarity(query_vector, &existing_vec); - if sim <= semantic_floor as f32 { - continue; - } - - let source = context.unwrap_or_else(|| { - format!( - "decision::{}", - decision.chars().take(40).collect::() - ) - }); - if !source_matches_prefix(&source, source_prefix) { - continue; - } - let mut scaled = scale_sim(sim); - if !keyword_terms.is_empty() { - let haystack = decision.to_lowercase(); - let overlap = keyword_terms - .iter() - .filter(|term| haystack.contains(term.as_str())) - .count(); - if overlap == 0 { - scaled *= 0.82; - } else { - let ratio = overlap as f64 / keyword_terms.len().max(1) as f64; - scaled *= 1.0 + ratio * 0.08; - } - } - let excerpt = query_focused_excerpt(&decision, query_text, 280); - let importance = blend_importance(score, trust_score); - let ts_source = last_accessed - .as_deref() - .or(created_at.as_deref()) - .unwrap_or_default(); - let ts = parse_timestamp_ms(ts_source); - let entry = candidates - .entry(source.clone()) - .or_insert(SemanticCandidate { - source, - excerpt: excerpt.clone(), - relevance: scaled, - importance, - ts, - }); - if scaled > entry.relevance { - *entry = SemanticCandidate { - source: entry.source.clone(), - excerpt, - relevance: scaled, - importance, - ts, - }; - } - } - } - } - - let mut sorted: Vec = candidates.into_values().collect(); - sorted.sort_by(|a, b| { - compare_relevance_desc_source_asc(a.relevance, &a.source, b.relevance, &b.source) - }); - sorted.truncate(MAX_SEMANTIC_RRF_CANDIDATES); - sorted -} - -pub(crate) fn collect_shadow_semantic_rows( - conn: &Connection, - ctx: &RecallContext, - source_prefix: Option<&str>, - expected_dimension: usize, -) -> Vec { - let selected_model = crate::embeddings::selected_model_key(); - let expected_vector_bytes = (expected_dimension * std::mem::size_of::()) as i64; - let source_like = source_prefix.map(|prefix| format!("{prefix}%")); - let mut rows_by_source: HashMap> = HashMap::new(); - - let memory_query_with_acl = "SELECT e.vector, m.source, m.owner_id, m.visibility \ - FROM embeddings e \ - JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id AND m.status = 'active' \ - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR m.source LIKE ?3)"; - let memory_query_without_acl = "SELECT e.vector, m.source, NULL AS owner_id, NULL AS visibility \ - FROM embeddings e \ - JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id AND m.status = 'active' \ - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR m.source LIKE ?3)"; - let memory_stmt = match conn.prepare(memory_query_with_acl) { - Ok(stmt) => Some(stmt), - Err(err) if is_missing_team_visibility_columns(&err) => { - conn.prepare(memory_query_without_acl).ok() - } - Err(_) => None, - }; - if let Some(mut stmt) = memory_stmt { - if let Ok(rows) = stmt.query_map( - params![ - selected_model, - expected_vector_bytes, - source_like.as_deref() - ], - |row| -> rusqlite::Result { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) - }, - ) { - for (blob, source, owner_id, visibility) in rows.flatten() { - if !is_visible(owner_id, visibility.as_deref(), ctx) { - continue; - } - if !source_matches_prefix(&source, source_prefix) { - continue; - } - rows_by_source - .entry(source) - .or_insert_with(|| crate::embeddings::blob_to_vector(&blob)); - } - } - } - - let decision_query_with_acl = "SELECT e.vector, d.decision, d.context, d.owner_id, d.visibility \ - FROM embeddings e \ - JOIN decisions d ON e.target_type = 'decision' AND e.target_id = d.id AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR d.context LIKE ?3)"; - let decision_query_without_acl = "SELECT e.vector, d.decision, d.context, NULL AS owner_id, NULL AS visibility \ - FROM embeddings e \ - JOIN decisions d ON e.target_type = 'decision' AND e.target_id = d.id AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) \ - AND (e.model IS NULL OR LOWER(e.model) = ?1) \ - AND (length(e.vector) = ?2 OR length(e.vector) = ?2/4 + 6) \ - AND (?3 IS NULL OR d.context LIKE ?3)"; - let decision_stmt = match conn.prepare(decision_query_with_acl) { - Ok(stmt) => Some(stmt), - Err(err) if is_missing_team_visibility_columns(&err) => { - conn.prepare(decision_query_without_acl).ok() - } - Err(_) => None, - }; - if let Some(mut stmt) = decision_stmt { - if let Ok(rows) = stmt.query_map( - params![ - selected_model, - expected_vector_bytes, - source_like.as_deref() - ], - |row| -> rusqlite::Result { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - )) - }, - ) { - for (blob, decision, context, owner_id, visibility) in rows.flatten() { - if !is_visible(owner_id, visibility.as_deref(), ctx) { - continue; - } - let source = context.unwrap_or_else(|| { - format!( - "decision::{}", - decision.chars().take(40).collect::() - ) - }); - if !source_matches_prefix(&source, source_prefix) { - continue; - } - rows_by_source - .entry(source) - .or_insert_with(|| crate::embeddings::blob_to_vector(&blob)); - } - } - } - - let mut rows: Vec = rows_by_source - .into_iter() - .map(|(source, vector)| ShadowSemanticRow { source, vector }) - .collect(); - rows.sort_by(|a, b| a.source.cmp(&b.source)); - rows -} - -pub(crate) fn vector_to_vec0_literal(vector: &[f32]) -> String { - let mut literal = String::with_capacity(vector.len().saturating_mul(12).saturating_add(2)); - literal.push('['); - for (idx, value) in vector.iter().enumerate() { - if idx > 0 { - literal.push_str(", "); - } - let stable = if value.is_finite() { *value } else { 0.0 }; - let _ = write!(&mut literal, "{stable}"); - } - literal.push(']'); - literal -} - -pub(crate) fn run_sqlite_vec_shadow_knn_sources( - conn: &Connection, - query_vector: &[f32], - candidates: &[ShadowSemanticRow], - top_k: usize, -) -> Result, String> { - if query_vector.is_empty() || candidates.is_empty() { - return Ok(Vec::new()); - } - - const SHADOW_TABLE: &str = "cortex_shadow_semantic_knn"; - let k = top_k.max(1).min(candidates.len()); - let query_literal = vector_to_vec0_literal(query_vector); - let result = (|| -> Result, String> { - conn.execute_batch(&format!("DROP TABLE IF EXISTS {SHADOW_TABLE};")) - .map_err(|err| format!("sqlite-vec shadow drop failed: {err}"))?; - conn.execute_batch(&format!( - "CREATE VIRTUAL TABLE {SHADOW_TABLE} USING vec0(\ - candidate_id INTEGER PRIMARY KEY,\ - embedding FLOAT[{}]\ - );", - query_vector.len() - )) - .map_err(|err| format!("sqlite-vec shadow create failed: {err}"))?; - - let insert_sql = - format!("INSERT INTO {SHADOW_TABLE}(candidate_id, embedding) VALUES (?1, ?2)"); - let mut insert_stmt = conn - .prepare(&insert_sql) - .map_err(|err| format!("sqlite-vec shadow insert prepare failed: {err}"))?; - for (candidate_idx, candidate) in candidates.iter().enumerate() { - let candidate_id = i64::try_from(candidate_idx + 1) - .map_err(|_| "sqlite-vec shadow candidate id overflow".to_string())?; - let embedding_literal = vector_to_vec0_literal(&candidate.vector); - insert_stmt - .execute(params![candidate_id, embedding_literal]) - .map_err(|err| format!("sqlite-vec shadow insert failed: {err}"))?; - } - - let k_i64 = i64::try_from(k).map_err(|_| "sqlite-vec shadow k overflow".to_string())?; - let query_sql = format!( - "SELECT candidate_id, distance \ - FROM {SHADOW_TABLE} \ - WHERE embedding MATCH ?1 AND k = ?2" - ); - let mut query_stmt = conn - .prepare(&query_sql) - .map_err(|err| format!("sqlite-vec shadow query prepare failed: {err}"))?; - let rows = query_stmt - .query_map(params![query_literal, k_i64], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, f64>(1)?)) - }) - .map_err(|err| format!("sqlite-vec shadow query failed: {err}"))?; - - let mut sources = Vec::new(); - let mut seen = HashSet::new(); - for row in rows { - let (candidate_id, _distance) = - row.map_err(|err| format!("sqlite-vec shadow row decode failed: {err}"))?; - if candidate_id <= 0 { - continue; - } - let Some(candidate) = candidates.get((candidate_id - 1) as usize) else { - continue; - }; - if seen.insert(candidate.source.clone()) { - sources.push(candidate.source.clone()); - } - } - - Ok(sources) - })(); - - let _ = conn.execute_batch(&format!("DROP TABLE IF EXISTS {SHADOW_TABLE};")); - result -} - -pub(crate) fn shadow_error_to_unavailable_reason(error: &str) -> Option<&'static str> { - let normalized = error.to_ascii_lowercase(); - if normalized.contains("no such module: vec0") { - return Some("sqlite_vec_not_available"); - } - None -} - -pub(crate) fn build_shadow_semantic_explain( - conn: &Connection, - query_vector: Option<&[f32]>, - query_text: &str, - ctx: &RecallContext, - source_prefix: Option<&str>, - top_k: usize, - baseline_override: Option<&ShadowSemanticBaseline>, -) -> Value { - let top_k = top_k.clamp(1, MAX_SEMANTIC_RRF_CANDIDATES); - let Some(query_vector) = query_vector else { - return json!({ - "enabled": true, - "status": "unavailable", - "reason": "query_embedding_unavailable", - "topK": top_k - }); - }; - if query_vector.is_empty() { - return json!({ - "enabled": true, - "status": "unavailable", - "reason": "query_embedding_empty", - "topK": top_k - }); - } - - let (baseline_candidate_count, baseline_top_sources) = if let Some(baseline) = baseline_override - { - (baseline.candidate_count, baseline.top_sources(top_k)) - } else { - let baseline = - collect_semantic_candidates(conn, query_vector, query_text, ctx, source_prefix); - let top_sources = baseline - .iter() - .take(top_k) - .map(|candidate| candidate.source.clone()) - .collect(); - (baseline.len(), top_sources) - }; - - let rows = collect_shadow_semantic_rows(conn, ctx, source_prefix, query_vector.len()); - if rows.is_empty() { - return json!({ - "enabled": true, - "status": "unavailable", - "reason": "no_shadow_candidates", - "topK": top_k, - "baselineCandidateCount": baseline_candidate_count, - "baselineTopSources": baseline_top_sources, - }); - } - - let vector_dim = query_vector.len(); - let compatible_rows: Vec = rows - .into_iter() - .filter(|row| row.vector.len() == vector_dim) - .collect(); - if compatible_rows.is_empty() { - return json!({ - "enabled": true, - "status": "unavailable", - "reason": "no_dimension_compatible_candidates", - "topK": top_k, - "vectorDimension": vector_dim, - "baselineCandidateCount": baseline_candidate_count, - "baselineTopSources": baseline_top_sources, - }); - } - - let compatible_count = compatible_rows.len(); - let shadow_top_sources = - match run_sqlite_vec_shadow_knn_sources(conn, query_vector, &compatible_rows, top_k) { - Ok(sources) => sources, - Err(error) => { - if let Some(reason) = shadow_error_to_unavailable_reason(&error) { - return json!({ - "enabled": true, - "status": "unavailable", - "reason": reason, - "detail": error, - "topK": top_k, - "vectorDimension": vector_dim, - "baselineCandidateCount": baseline_candidate_count, - "shadowCandidateCount": compatible_count, - "baselineTopSources": baseline_top_sources, - }); - } - return json!({ - "enabled": true, - "status": "error", - "reason": error, - "topK": top_k, - "vectorDimension": vector_dim, - "baselineCandidateCount": baseline_candidate_count, - "shadowCandidateCount": compatible_count, - "baselineTopSources": baseline_top_sources, - }); - } - }; - - let baseline_set: HashSet<&str> = baseline_top_sources.iter().map(String::as_str).collect(); - let shadow_set: HashSet<&str> = shadow_top_sources.iter().map(String::as_str).collect(); - let overlap_count = baseline_set.intersection(&shadow_set).count(); - let union_count = baseline_set.union(&shadow_set).count(); - let overlap_ratio = if top_k == 0 { - 0.0 - } else { - round4(overlap_count as f64 / top_k as f64) - }; - let jaccard = if union_count == 0 { - 1.0 - } else { - round4(overlap_count as f64 / union_count as f64) - }; - let baseline_index: HashMap<&str, usize> = baseline_top_sources - .iter() - .enumerate() - .map(|(idx, source)| (source.as_str(), idx)) - .collect(); - let shadow_index: HashMap<&str, usize> = shadow_top_sources - .iter() - .enumerate() - .map(|(idx, source)| (source.as_str(), idx)) - .collect(); - let mut matched_rank_pairs: usize = 0; - let mut rank_delta_sum: usize = 0; - for (source, baseline_rank) in &baseline_index { - if let Some(shadow_rank) = shadow_index.get(source) { - matched_rank_pairs += 1; - rank_delta_sum += baseline_rank.abs_diff(*shadow_rank); - } - } - let mean_abs_rank_delta = if matched_rank_pairs > 0 { - Some(round4(rank_delta_sum as f64 / matched_rank_pairs as f64)) - } else { - None - }; - let top1_match = match ( - baseline_top_sources.first().map(String::as_str), - shadow_top_sources.first().map(String::as_str), - ) { - (Some(left), Some(right)) => Some(left == right), - _ => None, - }; - - json!({ - "enabled": true, - "status": "ok", - "topK": top_k, - "vectorDimension": vector_dim, - "baselineCandidateCount": baseline_candidate_count, - "shadowCandidateCount": compatible_count, - "baselineTopSources": baseline_top_sources, - "shadowTopSources": shadow_top_sources, - "overlapCount": overlap_count, - "overlapRatio": overlap_ratio, - "jaccard": jaccard, - "matchedRankPairs": matched_rank_pairs, - "meanAbsRankDelta": mean_abs_rank_delta, - "top1Match": top1_match, - }) -} - diff --git a/daemon-rs/src/handlers/recall/telemetry.rs b/daemon-rs/src/handlers/recall/telemetry.rs deleted file mode 100644 index a790d491..00000000 --- a/daemon-rs/src/handlers/recall/telemetry.rs +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── Unified recall pipeline ───────────────────────────────────────────────── - -pub(crate) fn is_benchmark_recall_scope(agent: &str, source_prefix: Option<&str>) -> bool { - if agent - .trim() - .to_ascii_lowercase() - .starts_with(BENCHMARK_SOURCE_AGENT_PREFIX) - { - return true; - } - source_prefix - .map(str::trim) - .unwrap_or_default() - .to_ascii_lowercase() - .starts_with(BENCHMARK_SOURCE_SCOPE_PREFIX) -} - -pub(crate) async fn emit_recall_query_event( - state: &RuntimeState, - agent: &str, - source_prefix: Option<&str>, - payload: Value, -) { - if is_benchmark_recall_scope(agent, source_prefix) { - return; - } - let conn = state.db.lock().await; - if crate::handlers::log_event(&conn, "recall_query", payload, agent).is_ok() { - checkpoint_wal_best_effort(&conn); - } -} - -pub(crate) fn build_method_breakdown(results: &[RecallItem]) -> Value { - let mut counts: BTreeMap = BTreeMap::new(); - for item in results { - *counts.entry(item.method.clone()).or_insert(0) += 1; - } - json!(counts) -} - -pub(crate) fn method_count(methods: &Value, method: &str) -> i64 { - methods.get(method).and_then(|v| v.as_i64()).unwrap_or(0) -} - -pub(crate) fn classify_recall_tier(cached: bool, mode: &str, methods: &Value) -> &'static str { - if cached { - return "cache_hit"; - } - if mode == "headlines" { - return "headlines"; - } - if mode == "semantic" { - return "semantic_only"; - } - - let keyword = method_count(methods, "keyword"); - let semantic = method_count(methods, "semantic"); - let hybrid = method_count(methods, "hybrid"); - let crystal = method_count(methods, "crystal"); - let associative = method_count(methods, "associative"); - - if hybrid > 0 || (keyword > 0 && semantic > 0) { - if crystal > 0 { - return "hybrid_crystal"; - } - return "hybrid_fusion"; - } - if associative > 0 && (keyword > 0 || semantic > 0 || crystal > 0) { - return "associative_blend"; - } - if keyword > 0 { - if crystal > 0 { - return "keyword_crystal"; - } - return "keyword_only"; - } - if semantic > 0 { - if crystal > 0 { - return "semantic_crystal"; - } - return "semantic_only"; - } - if crystal > 0 { - return "crystal_only"; - } - if associative > 0 { - return "associative_only"; - } - "unknown" -} - -pub(crate) fn shadow_semantic_telemetry_summary(shadow_semantic: &Value) -> Value { - let status = shadow_semantic - .get("status") - .and_then(Value::as_str) - .unwrap_or("error"); - - let mut summary = json!({ - "status": status, - }); - if let Some(reason) = shadow_semantic.get("reason").and_then(Value::as_str) { - summary["reason"] = json!(reason); - } - for key in [ - "topK", - "vectorDimension", - "baselineCandidateCount", - "shadowCandidateCount", - "overlapCount", - "overlapRatio", - "jaccard", - "matchedRankPairs", - "meanAbsRankDelta", - "top1Match", - ] { - if let Some(value) = shadow_semantic.get(key) { - summary[key] = value.clone(); - } - } - if status == "error" && summary.get("reason").is_none() { - summary["reason"] = json!("shadow_payload_invalid"); - } - summary -} - diff --git a/daemon-rs/src/handlers/recall/tests/core.rs b/daemon-rs/src/handlers/recall/tests/core.rs index 3b248534..781e47f5 100644 --- a/daemon-rs/src/handlers/recall/tests/core.rs +++ b/daemon-rs/src/handlers/recall/tests/core.rs @@ -1,10 +1,8 @@ // SPDX-License-Identifier: MIT #[cfg(test)] mod tests { + use crate::handlers::recall::tests::support::{solo_ctx, store_decision_with_embedding, team_ctx, test_conn}; use crate::handlers::recall::*; - use crate::handlers::recall::tests::support::{ - solo_ctx, store_decision_with_embedding, team_ctx, test_conn, - }; #[test] fn search_memories_excludes_temporally_invalid_rows() { @@ -60,11 +58,26 @@ mod tests { &[0.0, 0.0, 0.0, 0.0, 1.0], ); - let results = - run_budget_recall(&mut conn, "write buffer", 400, 5, &solo_ctx(), None).unwrap(); + let results = run_budget_recall(&mut conn, "write buffer", 400, 5, &solo_ctx(), None).unwrap(); assert_eq!(results[0].source, "decision::write-buffer"); } + #[test] + fn semantic_candidates_include_current_pq8_embeddings() { + let mut conn = test_conn(); + let vector = [0.0, 0.0, 0.0, 0.0, 1.0]; + store_decision_with_embedding( + &mut conn, + "Semantic recall should read current PQ8 embeddings without scanning bad blobs.", + "decision::semantic-pq8", + &vector, + ); + + let candidates = collect_semantic_candidates(&conn, &vector, "semantic recall pq8", &solo_ctx(), Some("decision::semantic")); + + assert!(candidates.iter().any(|candidate| candidate.source == "decision::semantic-pq8")); + } + #[test] fn is_visible_team_private_hidden_from_other() { let ctx = team_ctx(2); diff --git a/daemon-rs/src/handlers/recall/tests/mod.rs b/daemon-rs/src/handlers/recall/tests/mod.rs index 24f0a157..68b44495 100644 --- a/daemon-rs/src/handlers/recall/tests/mod.rs +++ b/daemon-rs/src/handlers/recall/tests/mod.rs @@ -2,5 +2,5 @@ //! Recall tests: user-visible search, scoping, and store→recall flows only. //! Internal scoring/NLP/RRF/cache math is not unit-tested here — see Info/testing-philosophy.md. -mod support; mod core; +mod support; diff --git a/daemon-rs/src/handlers/recall/tests/support.rs b/daemon-rs/src/handlers/recall/tests/support.rs index 56c23845..217baf30 100644 --- a/daemon-rs/src/handlers/recall/tests/support.rs +++ b/daemon-rs/src/handlers/recall/tests/support.rs @@ -1,72 +1,13 @@ // SPDX-License-Identifier: MIT use crate::handlers::recall::*; use crate::handlers::store::{persist_decision_embedding, store_decision_with_input_embedding}; -use crate::state::RuntimeState; -use rusqlite::params; -use serde_json::Value; -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::sync::{broadcast, Mutex}; - -static SHARED_TEST_DB_COUNTER: AtomicU64 = AtomicU64::new(0); - -struct StaticReranker; - -impl crate::rerank::Reranker for StaticReranker { - fn name(&self) -> &'static str { - "static_test_reranker" - } - - fn model_size_mb(&self) -> u64 { - 1 - } - - fn rerank( - &self, - _query: &str, - candidates: &[crate::rerank::RerankCandidate], - fusion_alpha: f64, - ) -> Result, String> { - let scores = candidates - .iter() - .map(|candidate| { - let score = if candidate.id == "memory::winner" { - 10.0 - } else { - -10.0 - }; - (candidate.id.clone(), score) - }) - .collect::>(); - Ok(crate::rerank::fuse_scores( - candidates, - &scores, - fusion_alpha, - )) - } -} - -// ── is_visible tests ─────────────────────────────────────────── pub(crate) fn solo_ctx() -> RecallContext { - RecallContext { - caller_id: None, - team_mode: false, - } + RecallContext { caller_id: None, team_mode: false } } + pub(crate) fn team_ctx(caller: i64) -> RecallContext { - RecallContext { - caller_id: Some(caller), - team_mode: true, - } -} -fn team_ctx_no_caller() -> RecallContext { - RecallContext { - caller_id: None, - team_mode: true, - } + RecallContext { caller_id: Some(caller), team_mode: true } } pub(crate) fn test_conn() -> rusqlite::Connection { @@ -77,285 +18,10 @@ pub(crate) fn test_conn() -> rusqlite::Connection { conn } -fn shared_test_state() -> RuntimeState { - let unique_id = SHARED_TEST_DB_COUNTER.fetch_add(1, Ordering::Relaxed); - let db_path = std::env::temp_dir().join(format!( - "cortex-recall-shared-{}-{}-{}.db", - std::process::id(), - unique_id, - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let write_conn = rusqlite::Connection::open(&db_path).unwrap(); - crate::db::configure(&write_conn).unwrap(); - crate::db::initialize_schema(&write_conn).unwrap(); - crate::db::run_pending_migrations(&write_conn); - - let read_conn = rusqlite::Connection::open(&db_path).unwrap(); - crate::db::configure(&read_conn).unwrap(); - crate::db::initialize_schema(&read_conn).unwrap(); - crate::db::run_pending_migrations(&read_conn); - - let (events, _) = broadcast::channel(8); - let (brain_firing, _) = broadcast::channel(8); - RuntimeState { - db: Arc::new(Mutex::new(write_conn)), - db_read: Arc::new(Mutex::new(read_conn)), - token: Arc::new("test-token".to_string()), - events, - brain_firing, - mcp_calls: Arc::new(AtomicU64::new(0)), - mcp_sessions: Arc::new(Mutex::new(HashMap::new())), - recall_history: Arc::new(Mutex::new(HashMap::new())), - pre_cache: Arc::new(Mutex::new(HashMap::new())), - served_content: Arc::new(Mutex::new(HashMap::new())), - shutdown_tx: Arc::new(Mutex::new(None)), - home: PathBuf::from("."), - db_path: db_path.clone(), - token_path: PathBuf::from("cortex.token"), - pid_path: PathBuf::from("cortex.pid"), - port: 7437, - embedding_engine: None, - rate_limiter: crate::rate_limit::RateLimiter::new(), - team_mode: false, - default_owner_id: None, - team_api_key_hashes: Arc::new(std::sync::RwLock::new(Vec::new())), - degraded_mode: Arc::new(AtomicBool::new(false)), - db_corrupted: Arc::new(AtomicBool::new(false)), - readiness: Arc::new(AtomicBool::new(true)), - last_activity_unix_secs: Arc::new(AtomicU64::new(0)), - write_buffer_path: PathBuf::from("write_buffer.jsonl"), - sqlite_vec_canary: crate::state::SqliteVecCanaryConfig { - trial_percent: 0, - force_off: false, - route_mode: crate::state::SqliteVecRouteMode::Trial, - }, - rerank_config: crate::rerank::RerankConfig::off(), - reranker: None, - } -} - -fn latest_recall_query_event(conn: &rusqlite::Connection) -> Value { - let raw: String = conn - .query_row( - "SELECT data FROM events WHERE type = 'recall_query' ORDER BY id DESC LIMIT 1", - [], - |row| row.get(0), - ) - .expect("latest recall_query event should exist"); - serde_json::from_str(&raw).expect("recall_query event should be valid json") -} - -fn recall_item_for_rerank(source: &str, relevance: f64) -> RecallItem { - RecallItem { - source: source.to_string(), - relevance, - excerpt: format!("rerank fixture for {source}"), - method: "hybrid".to_string(), - tokens: Some(10), - entropy: Some(0.5), - family_members: Vec::new(), - collapsed_sources: Vec::new(), - collapsed_source_scores: Vec::new(), - } -} - -#[test] -fn primary_rerank_reorders_top_window_and_marks_method() { - let mut state = shared_test_state(); - state.rerank_config = crate::rerank::RerankConfig { - mode: crate::rerank::RerankMode::Primary, - top_n: 2, - fusion_alpha: 0.90, - }; - state.reranker = Some(Arc::new(StaticReranker)); - let results = vec![ - recall_item_for_rerank("memory::baseline", 0.95), - recall_item_for_rerank("memory::winner", 0.70), - recall_item_for_rerank("memory::outside", 0.60), - ]; - - let (reranked, route) = maybe_apply_rerank(&state, "winner query", results, 240); - - assert_eq!(route["status"], "ok"); - assert_eq!(route["mode"], "primary"); - assert_eq!(route["applied"], true); - assert_eq!(route["baselineTopSources"][0], "memory::baseline"); - assert_eq!(route["rerankedTopSources"][0], "memory::winner"); - assert_eq!(reranked[0].source, "memory::winner"); - assert_eq!(reranked[2].source, "memory::outside"); - assert!(reranked[0].method.ends_with("+rerank")); -} - -#[test] -fn shadow_rerank_reports_route_without_reordering() { - let mut state = shared_test_state(); - state.rerank_config = crate::rerank::RerankConfig { - mode: crate::rerank::RerankMode::Shadow, - top_n: 2, - fusion_alpha: 0.90, - }; - state.reranker = Some(Arc::new(StaticReranker)); - let results = vec![ - recall_item_for_rerank("memory::baseline", 0.95), - recall_item_for_rerank("memory::winner", 0.70), - recall_item_for_rerank("memory::outside", 0.60), - ]; - - let (reranked, route) = maybe_apply_rerank(&state, "winner query", results, 240); - - assert_eq!(route["status"], "ok"); - assert_eq!(route["mode"], "shadow"); - assert_eq!(route["applied"], false); - assert_eq!(route["baselineTopSources"][0], "memory::baseline"); - assert_eq!(route["rerankedTopSources"][0], "memory::winner"); - assert_eq!(reranked[0].source, "memory::baseline"); - assert!(!reranked[0].method.ends_with("+rerank")); -} - -fn insert_memory_with_embedding( - conn: &rusqlite::Connection, - text: &str, - source: &str, - vector: &[f32], -) -> i64 { - let model_key = crate::embeddings::selected_model_key(); - conn.execute( - "INSERT INTO memories (text, source, type, status, score, created_at, updated_at) - VALUES (?1, ?2, 'note', 'active', 1.0, datetime('now'), datetime('now'))", - params![text, source], - ) - .unwrap(); - let id = conn.last_insert_rowid(); - conn.execute( - "INSERT INTO embeddings (target_type, target_id, vector, model) - VALUES ('memory', ?1, ?2, ?3)", - params![id, crate::embeddings::vector_to_blob(vector), model_key], - ) - .unwrap(); - id -} - -fn insert_memory_with_optional_source_and_embedding( - conn: &rusqlite::Connection, - text: &str, - source: Option<&str>, - vector: &[f32], -) -> i64 { - let model_key = crate::embeddings::selected_model_key(); - conn.execute( - "INSERT INTO memories (text, source, type, status, score, created_at, updated_at) - VALUES (?1, ?2, 'note', 'active', 1.0, datetime('now'), datetime('now'))", - params![text, source], - ) - .unwrap(); - let id = conn.last_insert_rowid(); - conn.execute( - "INSERT INTO embeddings (target_type, target_id, vector, model) - VALUES ('memory', ?1, ?2, ?3)", - params![id, crate::embeddings::vector_to_blob(vector), model_key], - ) - .unwrap(); - id -} - -pub(crate) fn store_decision_with_embedding( - conn: &mut rusqlite::Connection, - decision: &str, - context: &str, - vector: &[f32], -) { - let (_, new_id) = store_decision_with_input_embedding( - conn, - decision, - Some(context.to_string()), - None, - "tester".to_string(), - None, - None, - Some(vector), - None, - ) - .unwrap(); - +pub(crate) fn store_decision_with_embedding(conn: &mut rusqlite::Connection, decision: &str, context: &str, vector: &[f32]) { + let (_, new_id) = + store_decision_with_input_embedding(conn, decision, Some(context.to_string()), None, "tester".to_string(), None, None, Some(vector), None).unwrap(); if let Some(id) = new_id { - persist_decision_embedding(conn, id, vector, crate::embeddings::selected_model_key()) - .unwrap(); - } -} - -fn insert_crystal_with_memory_members( - conn: &rusqlite::Connection, - label: &str, - consolidated_text: &str, - crystal_vector: &[f32], - members: &[(&str, &str, &[f32])], -) -> (i64, String, Vec) { - let mut member_sources = Vec::with_capacity(members.len()); - let mut member_ids = Vec::with_capacity(members.len()); - for (text, source, vector) in members { - let id = insert_memory_with_embedding(conn, text, source, vector); - member_ids.push(id); - member_sources.push((*source).to_string()); - } - - if conn - .execute( - "INSERT INTO memory_clusters ( - label, - centroid, - consolidated_text, - member_count, - owner_id, - visibility, - created_at, - updated_at - ) VALUES (?1, NULL, ?2, ?3, 1, 'shared', datetime('now'), datetime('now'))", - params![label, consolidated_text, members.len() as i64], - ) - .is_err() - { - conn.execute( - "INSERT INTO memory_clusters ( - label, - centroid, - consolidated_text, - member_count, - created_at, - updated_at - ) VALUES (?1, NULL, ?2, ?3, datetime('now'), datetime('now'))", - params![label, consolidated_text, members.len() as i64], - ) - .unwrap(); - } - let crystal_id = conn.last_insert_rowid(); - - for member_id in member_ids { - conn.execute( - "INSERT INTO cluster_members (cluster_id, target_type, target_id, similarity) - VALUES (?1, 'memory', ?2, 1.0)", - params![crystal_id, member_id], - ) - .unwrap(); + persist_decision_embedding(conn, id, vector, crate::embeddings::selected_model_key()).unwrap(); } - - conn.execute( - "INSERT INTO embeddings (target_type, target_id, vector, model) - VALUES ('crystal', ?1, ?2, ?3)", - params![ - crystal_id, - crate::embeddings::vector_to_blob(crystal_vector), - crate::embeddings::selected_model_key() - ], - ) - .unwrap(); - - ( - crystal_id, - crystal_source(crystal_id, label), - member_sources, - ) } - diff --git a/daemon-rs/src/handlers/recall/types.rs b/daemon-rs/src/handlers/recall/types.rs deleted file mode 100644 index 3ff0f422..00000000 --- a/daemon-rs/src/handlers/recall/types.rs +++ /dev/null @@ -1,715 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -use super::query_shape_profile; - -// ─── Constants ─────────────────────────────────────────────────────────────── - -pub(crate) const MAX_RECALL_HISTORY: usize = 50; -pub(crate) const PRECACHE_TTL_MS: i64 = 5 * 60 * 1000; -pub(crate) const SEMANTIC_SIM_FLOOR: f64 = 0.3; -pub(crate) const SEMANTIC_SCALE_BASE: f64 = 0.55; -pub(crate) const MAX_SEMANTIC_RRF_CANDIDATES: usize = 120; -pub(crate) const MIN_BUDGET_HEADROOM_TOKENS: usize = 8; -pub(crate) const MIN_EXCERPT_CHARS: usize = 24; -pub(crate) const ASSOCIATIVE_MIN_BUDGET_TOKENS: usize = 260; -pub(crate) const MEMORIES_BM25_TEXT_WEIGHT: f64 = 4.6; -pub(crate) const MEMORIES_BM25_SOURCE_WEIGHT: f64 = 1.7; -pub(crate) const MEMORIES_BM25_TAGS_WEIGHT: f64 = 2.2; -pub(crate) const DECISIONS_BM25_DECISION_WEIGHT: f64 = 6.6; -pub(crate) const DECISIONS_BM25_CONTEXT_WEIGHT: f64 = 1.0; -pub(crate) const BM25_WEIGHT_MIN: f64 = 0.1; -pub(crate) const BM25_WEIGHT_MAX: f64 = 12.0; -pub(crate) const SQLITE_VEC_TRIAL_MIN_OVERLAP_RATIO: f64 = 0.60; -pub(crate) const SQLITE_VEC_TRIAL_MIN_JACCARD: f64 = 0.45; -pub(crate) const SQLITE_VEC_TRIAL_MAX_MEAN_ABS_RANK_DELTA: f64 = 1.25; -pub(crate) const SQLITE_VEC_TRIAL_TOP1_MATCH_REQUIRED: bool = true; -pub(crate) const ENTITY_SIGNAL_OVERLAP_WEIGHT: f64 = 0.10; -pub(crate) const ENTITY_SIGNAL_MATCH_WEIGHT: f64 = 0.01; -pub(crate) const ENTITY_SIGNAL_MAX_BOOST: f64 = 0.12; -pub(crate) const ALIGNMENT_EXACT_BONUS_MAX: f64 = 0.08; -pub(crate) const ALIGNMENT_COVERAGE_BONUS_MAX: f64 = 0.07; -pub(crate) const ALIGNMENT_BOOST_MAX: f64 = 0.15; -pub(crate) const TEMPORAL_INTENT_MULTIPLIER_RANGE: f64 = 0.16; -pub(crate) const BENCHMARK_SOURCE_AGENT_PREFIX: &str = "amb-cortex::"; -pub(crate) const BENCHMARK_SOURCE_SCOPE_PREFIX: &str = "amb::"; -pub(crate) const DEFAULT_RECALL_BUDGET_FAST: usize = 180; -pub(crate) const DEFAULT_RECALL_BUDGET_BALANCED: usize = 320; -pub(crate) const DEFAULT_RECALL_BUDGET_DEEP: usize = 560; -pub(crate) const DEFAULT_RECALL_LATENCY_FAST_MS: u128 = 900; -pub(crate) const DEFAULT_RECALL_LATENCY_BALANCED_MS: u128 = 1800; -pub(crate) const DEFAULT_RECALL_LATENCY_DEEP_MS: u128 = 3500; -pub(crate) const BUDGET_REDUNDANCY_SIMILARITY_THRESHOLD: f64 = 0.84; -pub(crate) const BUDGET_PRESSURE_EARLY_STOP_THRESHOLD: f64 = 0.82; - -// ─── Internal types ────────────────────────────────────────────────────────── - -#[derive(Clone, Copy, Debug)] -pub(crate) struct Bm25Weights { - pub(crate) memories_text: f64, - pub(crate) memories_source: f64, - pub(crate) memories_tags: f64, - pub(crate) decisions_text: f64, - pub(crate) decisions_context: f64, -} - -pub(crate) static BM25_WEIGHTS: OnceLock = OnceLock::new(); - -pub(crate) fn parse_bm25_weight(raw: Option, default: f64) -> f64 { - raw.and_then(|value| value.trim().parse::().ok()) - .filter(|value| value.is_finite() && *value > 0.0) - .unwrap_or(default) - .clamp(BM25_WEIGHT_MIN, BM25_WEIGHT_MAX) -} - -pub(crate) fn bm25_weights_from_resolver(mut resolve_env: impl FnMut(&str) -> Option) -> Bm25Weights { - Bm25Weights { - memories_text: parse_bm25_weight( - resolve_env("CORTEX_BM25_MEM_TEXT_WEIGHT"), - MEMORIES_BM25_TEXT_WEIGHT, - ), - memories_source: parse_bm25_weight( - resolve_env("CORTEX_BM25_MEM_SOURCE_WEIGHT"), - MEMORIES_BM25_SOURCE_WEIGHT, - ), - memories_tags: parse_bm25_weight( - resolve_env("CORTEX_BM25_MEM_TAGS_WEIGHT"), - MEMORIES_BM25_TAGS_WEIGHT, - ), - decisions_text: parse_bm25_weight( - resolve_env("CORTEX_BM25_DECISION_WEIGHT"), - DECISIONS_BM25_DECISION_WEIGHT, - ), - decisions_context: parse_bm25_weight( - resolve_env("CORTEX_BM25_CONTEXT_WEIGHT"), - DECISIONS_BM25_CONTEXT_WEIGHT, - ), - } -} - -pub(crate) fn bm25_weights() -> &'static Bm25Weights { - BM25_WEIGHTS.get_or_init(|| bm25_weights_from_resolver(|name| std::env::var(name).ok())) -} - -#[derive(Clone, Debug)] -pub(crate) struct RecallItem { - pub(crate) source: String, - pub(crate) relevance: f64, - pub(crate) excerpt: String, - pub(crate) method: String, - pub(crate) tokens: Option, - pub(crate) entropy: Option, - pub(crate) family_members: Vec, - pub(crate) collapsed_sources: Vec, - pub(crate) collapsed_source_scores: Vec<(String, f64)>, -} - -/// Shannon entropy of text (bits per character). -/// English prose: ~4.0-4.5, boilerplate: ~2.0-3.0, code/decisions: ~4.5-5.0. -pub fn shannon_entropy(text: &str) -> f64 { - if text.is_empty() { - return 0.0; - } - let mut freq = [0u32; 256]; - let len = text.len() as f64; - for &b in text.as_bytes() { - freq[b as usize] += 1; - } - let mut h = 0.0f64; - for &count in &freq { - if count > 0 { - let p = count as f64 / len; - h -= p * p.log2(); - } - } - h -} - -#[derive(Clone)] -pub(crate) struct SearchCandidate { - pub(crate) source: String, - pub(crate) excerpt: String, - pub(crate) alignment: (usize, usize), - pub(crate) relevance: f64, - pub(crate) matched_keywords: i64, - pub(crate) score: f64, - pub(crate) ts: i64, - pub(crate) owner_id: Option, - pub(crate) visibility: Option, -} - -#[derive(Clone)] -pub(crate) struct SemanticCandidate { - pub(crate) source: String, - pub(crate) excerpt: String, - pub(crate) relevance: f64, - pub(crate) importance: f64, - pub(crate) ts: i64, -} - -#[derive(Clone)] -pub(crate) struct ShadowSemanticRow { - pub(crate) source: String, - pub(crate) vector: Vec, -} - -#[derive(Clone)] -pub(crate) struct ShadowSemanticBaseline { - pub(crate) candidate_count: usize, - pub(crate) ranked_sources: Vec, -} - -impl ShadowSemanticBaseline { - pub(crate) fn top_sources(&self, top_k: usize) -> Vec { - self.ranked_sources - .iter() - .take(top_k.clamp(1, MAX_SEMANTIC_RRF_CANDIDATES)) - .cloned() - .collect() - } -} - -pub(crate) struct RecallWithVectorTrace { - pub(crate) ranked: Vec, - pub(crate) semantic_baseline: Option, - pub(crate) semantic_route: Value, -} - -pub(crate) type MemorySemanticRow = ( - Vec, - String, - String, - Option, - Option, - Option, - Option, - Option, - Option, -); -pub(crate) type DecisionSemanticRow = ( - Vec, - String, - Option, - Option, - Option, - Option, - Option, - Option, - Option, -); -pub(crate) type CrystalMemberSourceRow = (Option, Option, Option); -pub(crate) type ShadowMemoryRow = (Vec, String, Option, Option); -pub(crate) type ShadowDecisionRow = (Vec, String, Option, Option, Option); - -// ─── Visibility context ───────────────────────────────────────────────────── - -/// Caller identity + team mode flag, threaded through the recall pipeline -/// so visibility filtering can gate results without changing SQL queries. -#[derive(Clone, Copy)] -pub struct RecallContext { - pub caller_id: Option, - pub team_mode: bool, -} - -impl RecallContext { - /// Build from already-resolved caller_id (avoids double argon2). - pub fn from_caller(caller_id: Option, state: &RuntimeState) -> Self { - Self { - caller_id, - team_mode: state.team_mode, - } - } - - /// Build from runtime state (for MCP/non-HTTP callers). Uses default_owner_id. - #[allow(dead_code)] - pub fn from_state(state: &RuntimeState) -> Self { - Self { - caller_id: state.default_owner_id, - team_mode: state.team_mode, - } - } - - /// Solo-mode context where everything is visible (no filtering). - #[allow(dead_code)] - pub fn solo() -> Self { - Self { - caller_id: None, - team_mode: false, - } - } -} - -#[allow(clippy::result_large_err)] -pub(crate) fn require_team_caller( - state: &RuntimeState, - caller_id: Option, -) -> Result, Response> { - if state.team_mode && caller_id.is_none() { - return Err(json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - )); - } - Ok(caller_id) -} - -/// Check whether a record is visible to the current caller. -/// Solo mode: everything visible (no filtering). -/// Team mode (fail closed): -/// - caller_id=None → deny (unidentified caller sees nothing) -/// - owner_id=None → deny (unowned data hidden until backfilled) -/// - owner == caller → allow -/// - visibility shared/team → allow -/// - otherwise → deny -pub(crate) fn is_visible(owner_id: Option, visibility: Option<&str>, ctx: &RecallContext) -> bool { - if !ctx.team_mode { - return true; - } - let caller = match ctx.caller_id { - Some(c) => c, - None => return false, - }; - let owner = match owner_id { - Some(o) => o, - None => return false, - }; - if owner == caller { - return true; - } - matches!(visibility, Some("shared") | Some("team")) -} - -pub(crate) fn source_matches_prefix(source: &str, source_prefix: Option<&str>) -> bool { - match source_prefix { - Some(prefix) => source.starts_with(prefix), - None => true, - } -} - -pub(crate) fn crystal_source(crystal_id: i64, label: &str) -> String { - format!("crystal::{crystal_id}::{label}") -} - -pub(crate) fn dedup_preserve_order(values: &mut Vec) { - let mut seen = HashSet::new(); - values.retain(|value| seen.insert(value.clone())); -} - -pub(crate) fn normalize_collapsed_source_rank(item: &mut RecallItem) { - let mut best_scores: HashMap = HashMap::new(); - for (order, source) in item.collapsed_sources.iter().enumerate() { - best_scores.entry(source.clone()).or_insert((0.0, order)); - } - for (order, (source, score)) in item.collapsed_source_scores.iter().enumerate() { - best_scores - .entry(source.clone()) - .and_modify(|entry| { - entry.0 = entry.0.max(*score); - entry.1 = entry.1.min(order); - }) - .or_insert((*score, order)); - } - let mut ranked: Vec<(String, f64, usize)> = best_scores - .into_iter() - .map(|(source, (score, order))| (source, score, order)) - .collect(); - ranked.sort_by(|a, b| { - b.1.partial_cmp(&a.1) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.2.cmp(&b.2)) - }); - item.collapsed_source_scores = ranked - .iter() - .map(|(source, score, _)| (source.clone(), *score)) - .collect(); - item.collapsed_sources = item - .collapsed_source_scores - .iter() - .map(|(source, _)| source.clone()) - .collect(); -} - -pub(crate) fn parse_crystal_source_id(source: &str) -> Option { - let rest = source.strip_prefix("crystal::")?; - let (id, _) = rest.split_once("::")?; - id.parse::().ok() -} - -pub(crate) fn crystal_member_sources(conn: &Connection, crystal_id: i64, ctx: &RecallContext) -> Vec { - let query_rows = |sql: &str, - with_visibility: bool| - -> Result, rusqlite::Error> { - let mut stmt = conn.prepare(sql)?; - let mapped = stmt.query_map(params![crystal_id], |row| { - Ok(( - row.get::<_, Option>(0)?, - if with_visibility { - row.get::<_, Option>(1)? - } else { - None - }, - if with_visibility { - row.get::<_, Option>(2)? - } else { - None - }, - )) - })?; - Ok(mapped.flatten().collect()) - }; - - let sql_with_visibility = "SELECT CASE - WHEN cm.target_type = 'memory' THEN COALESCE(m.source, 'memory::' || m.id) - ELSE COALESCE(d.context, 'decision::' || d.id) - END AS source, - CASE - WHEN cm.target_type = 'memory' THEN m.owner_id - ELSE d.owner_id - END AS owner_id, - CASE - WHEN cm.target_type = 'memory' THEN m.visibility - ELSE d.visibility - END AS visibility - FROM cluster_members cm - LEFT JOIN memories m - ON cm.target_type = 'memory' - AND cm.target_id = m.id - AND m.status = 'active' - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) - LEFT JOIN decisions d - ON cm.target_type = 'decision' - AND cm.target_id = d.id - AND d.status = 'active' - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) - WHERE cm.cluster_id = ?1 - ORDER BY cm.target_type, cm.target_id"; - - let sql_legacy = "SELECT CASE - WHEN cm.target_type = 'memory' THEN COALESCE(m.source, 'memory::' || m.id) - ELSE COALESCE(d.context, 'decision::' || d.id) - END AS source - FROM cluster_members cm - LEFT JOIN memories m - ON cm.target_type = 'memory' - AND cm.target_id = m.id - AND m.status = 'active' - AND (m.expires_at IS NULL OR m.expires_at > datetime('now')) \ - AND (m.valid_from IS NULL OR m.valid_from <= datetime('now')) \ - AND (m.valid_until IS NULL OR m.valid_until > datetime('now')) - LEFT JOIN decisions d - ON cm.target_type = 'decision' - AND cm.target_id = d.id - AND d.status = 'active' - AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ - AND (d.valid_from IS NULL OR d.valid_from <= datetime('now')) \ - AND (d.valid_until IS NULL OR d.valid_until > datetime('now')) - WHERE cm.cluster_id = ?1 - ORDER BY cm.target_type, cm.target_id"; - - let rows = match query_rows(sql_with_visibility, true) { - Ok(rows) => rows, - Err(err) if is_missing_team_visibility_columns(&err) => { - match query_rows(sql_legacy, false) { - Ok(rows) => rows, - Err(_) => return Vec::new(), - } - } - Err(_) => return Vec::new(), - }; - - let mut sources = Vec::new(); - let mut seen = HashSet::new(); - for (source, owner_id, visibility) in rows { - let Some(source) = source else { - continue; - }; - if !is_visible(owner_id, visibility.as_deref(), ctx) { - continue; - } - if seen.insert(source.clone()) { - sources.push(source); - } - } - sources -} - -pub(crate) type CrystalUnfoldRow = (String, String, i64, Option, Option); - -pub(crate) fn query_crystal_for_unfold(conn: &Connection, crystal_id: i64) -> Option { - let sql_with_visibility = "SELECT label, consolidated_text, member_count, owner_id, visibility - FROM memory_clusters - WHERE id = ?1"; - match conn.query_row(sql_with_visibility, params![crystal_id], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - )) - }) { - Ok(row) => Some(row), - Err(err) if is_missing_team_visibility_columns(&err) => conn - .query_row( - "SELECT label, consolidated_text, member_count - FROM memory_clusters - WHERE id = ?1", - params![crystal_id], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - None, - None, - )) - }, - ) - .ok(), - Err(_) => None, - } -} - -pub(crate) fn is_missing_team_visibility_columns(err: &rusqlite::Error) -> bool { - let normalized = err.to_string().to_ascii_lowercase(); - normalized.contains("no such column") - && (normalized.contains("owner_id") || normalized.contains("visibility")) -} - -// ─── Query types ───────────────────────────────────────────────────────────── - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RecallPolicyMode { - Headlines, - Fast, - Balanced, - Deep, -} - -impl RecallPolicyMode { - pub fn as_str(self) -> &'static str { - match self { - Self::Headlines => "headlines", - Self::Fast => "fast", - Self::Balanced => "balanced", - Self::Deep => "deep", - } - } -} - -pub(crate) fn parse_env_usize(name: &str, default: usize, min: usize, max: usize) -> usize { - std::env::var(name) - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .map(|value| value.clamp(min, max)) - .unwrap_or(default) -} - -pub(crate) fn recall_default_budget_for_mode(mode: RecallPolicyMode) -> usize { - match mode { - RecallPolicyMode::Headlines => 0, - RecallPolicyMode::Fast => parse_env_usize( - "CORTEX_RECALL_FAST_BUDGET", - DEFAULT_RECALL_BUDGET_FAST, - 1, - 2000, - ), - RecallPolicyMode::Balanced => parse_env_usize( - "CORTEX_RECALL_BALANCED_BUDGET", - DEFAULT_RECALL_BUDGET_BALANCED, - 1, - 4000, - ), - RecallPolicyMode::Deep => parse_env_usize( - "CORTEX_RECALL_DEEP_BUDGET", - DEFAULT_RECALL_BUDGET_DEEP, - 1, - 8000, - ), - } -} - -pub(crate) fn recall_default_k_for_mode(mode: RecallPolicyMode) -> usize { - match mode { - RecallPolicyMode::Headlines => 10, - RecallPolicyMode::Fast => 16, - RecallPolicyMode::Balanced => 12, - RecallPolicyMode::Deep => 10, - } -} - -pub(crate) fn recall_latency_budget_ms_for_mode(mode: RecallPolicyMode) -> u128 { - match mode { - RecallPolicyMode::Headlines => parse_env_usize( - "CORTEX_RECALL_HEADLINES_MAX_LATENCY_MS", - DEFAULT_RECALL_LATENCY_FAST_MS as usize, - 0, - 60_000, - ) as u128, - RecallPolicyMode::Fast => parse_env_usize( - "CORTEX_RECALL_FAST_MAX_LATENCY_MS", - DEFAULT_RECALL_LATENCY_FAST_MS as usize, - 0, - 60_000, - ) as u128, - RecallPolicyMode::Balanced => parse_env_usize( - "CORTEX_RECALL_BALANCED_MAX_LATENCY_MS", - DEFAULT_RECALL_LATENCY_BALANCED_MS as usize, - 0, - 60_000, - ) as u128, - RecallPolicyMode::Deep => parse_env_usize( - "CORTEX_RECALL_DEEP_MAX_LATENCY_MS", - DEFAULT_RECALL_LATENCY_DEEP_MS as usize, - 0, - 120_000, - ) as u128, - } -} - -pub(crate) fn recall_mode_for_budget(budget: usize) -> RecallPolicyMode { - if budget == 0 { - RecallPolicyMode::Headlines - } else if budget <= 220 { - RecallPolicyMode::Fast - } else if budget <= 500 { - RecallPolicyMode::Balanced - } else { - RecallPolicyMode::Deep - } -} - -pub fn parse_recall_policy_mode(raw: Option<&str>) -> Result, String> { - let Some(raw) = raw.map(str::trim).filter(|value| !value.is_empty()) else { - return Ok(None); - }; - let normalized = raw.to_ascii_lowercase(); - let mode = match normalized.as_str() { - "headlines" => RecallPolicyMode::Headlines, - "fast" => RecallPolicyMode::Fast, - "balanced" => RecallPolicyMode::Balanced, - "deep" => RecallPolicyMode::Deep, - _ => { - return Err( - "Invalid policy mode. Expected one of: headlines, fast, balanced, deep".to_string(), - ); - } - }; - Ok(Some(mode)) -} - -pub fn resolve_recall_budget_k( - requested_mode: Option, - budget: Option, - k: Option, -) -> (usize, usize, RecallPolicyMode) { - let resolved_budget = match (requested_mode, budget) { - (_, Some(explicit_budget)) => explicit_budget, - (Some(mode), None) => recall_default_budget_for_mode(mode), - (None, None) => recall_default_budget_for_mode(RecallPolicyMode::Balanced), - }; - let resolved_mode = recall_mode_for_budget(resolved_budget); - let resolved_k = k.unwrap_or_else(|| recall_default_k_for_mode(resolved_mode)); - (resolved_budget, resolved_k.max(1), resolved_mode) -} - -pub(crate) fn adaptive_default_budget_for_query( - query_text: &str, - resolved_k: usize, - default_budget: usize, -) -> usize { - if default_budget == 0 { - return 0; - } - let profile = query_shape_profile(query_text, None); - let token_count = query_text.split_whitespace().count(); - let base: usize = if profile.exactish && !profile.naturalish { - 180 - } else if profile.naturalish && !profile.exactish { - if token_count >= 14 { - 300 - } else { - 270 - } - } else { - 240 - }; - let scaled = if resolved_k <= 3 { - base.saturating_sub(40) - } else if resolved_k <= 6 { - base - } else if resolved_k <= 10 { - base.saturating_add(30) - } else { - base.saturating_add(60) - }; - scaled.clamp(140, default_budget.max(140)) -} - -pub(crate) fn maybe_apply_adaptive_default_budget( - query_text: &str, - requested_mode: Option, - requested_budget: Option, - resolved_budget: usize, - resolved_k: usize, -) -> usize { - if requested_mode.is_some() || requested_budget.is_some() { - return resolved_budget; - } - adaptive_default_budget_for_query(query_text, resolved_k, resolved_budget) -} - -#[derive(Deserialize, Default)] -pub struct RecallQuery { - pub q: Option, - pub k: Option, - pub budget: Option, - pub agent: Option, - pub source_prefix: Option, - pub pool_k: Option, - #[serde(alias = "policyMode")] - pub policy_mode: Option, -} - -#[derive(Deserialize, Default)] -pub struct RecallBody { - pub q: Option, - pub k: Option, - pub budget: Option, - pub agent: Option, - pub source_prefix: Option, - #[serde(alias = "policyMode")] - pub policy_mode: Option, -} - diff --git a/daemon-rs/src/handlers/recall/unfold.rs b/daemon-rs/src/handlers/recall/unfold.rs deleted file mode 100644 index 901155d4..00000000 --- a/daemon-rs/src/handlers/recall/unfold.rs +++ /dev/null @@ -1,330 +0,0 @@ -// SPDX-License-Identifier: MIT -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use chrono::{TimeZone, Utc}; -use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; -use serde_json::{json, Value}; -use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; -use std::time::Instant; - -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget}; -use crate::handlers::{ - estimate_tokens, json_response, now_iso, parse_timestamp_ms, resolve_source_identity, - truncate_chars, -}; - -use super::*; -use crate::budgets::BudgetEndpoint; -use crate::co_occurrence; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::rerank::{RerankCandidate, RerankedScore}; -use crate::state::{ - PreCacheEntry, RecallHistoryEntry, RuntimeState, SqliteVecCanaryConfig, SqliteVecRouteMode, -}; - -// ─── GET /unfold ──────────────────────────────────────────────────────────── - -#[derive(Deserialize, Default)] -pub struct UnfoldQuery { - pub sources: Option, -} - -pub(crate) const MAX_UNFOLD_SOURCES: usize = 50; - -/// Unfold specific items by source string. Returns full text for each requested -/// source without re-running search. Designed for progressive disclosure: -/// peek (headlines) → unfold (full text of selected items). -pub async fn handle_unfold( - State(state): State, - Query(query): Query, - headers: HeaderMap, -) -> Response { - let caller_id = - match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Recall).await - { - Ok(id) => id, - Err(resp) => return resp, - }; - let caller_id = match require_team_caller(&state, caller_id) { - Ok(caller_id) => caller_id, - Err(resp) => return resp, - }; - let ctx = RecallContext::from_caller(caller_id, &state); - let sources_str = match &query.sources { - Some(s) if !s.trim().is_empty() => s.trim().to_string(), - _ => { - return json_response( - StatusCode::BAD_REQUEST, - json!({"error": "Missing query parameter: sources (comma-separated)"}), - ); - } - }; - - let requested: Vec<&str> = sources_str - .split(',') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .collect(); - if requested.is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({"error": "No valid sources provided"}), - ); - } - if requested.len() > MAX_UNFOLD_SOURCES { - return json_response( - StatusCode::BAD_REQUEST, - json!({"error": format!("Too many sources (max {MAX_UNFOLD_SOURCES})")}), - ); - } - - let agent = resolve_source_identity(&headers, "http").agent; - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Recall, &agent).await - { - return resp; - } - - let conn = state.db_read.lock().await; - let mut results: Vec = Vec::new(); - let mut total_tokens = 0usize; - - for source in &requested { - if let Some(mut item) = unfold_source(&conn, source, &ctx) { - let tokens = estimate_tokens(item["text"].as_str().unwrap_or("")); - total_tokens += tokens; - if let Value::Object(ref mut map) = item { - if !map.contains_key("source") { - map.insert("source".to_string(), Value::String(source.to_string())); - } - map.insert("tokens".to_string(), Value::Number((tokens as u64).into())); - } - results.push(item); - } else { - results.push(json!({ - "source": source, - "text": null, - "type": "not_found", - "tokens": 0, - })); - } - } - - json_response( - StatusCode::OK, - json!({ - "results": results, - "totalTokens": total_tokens, - "count": results.iter().filter(|r| r["type"] != "not_found").count(), - }), - ) -} - -/// Look up the full text of a single source string (team visibility applied when `ctx.team_mode`). -pub fn unfold_source(conn: &Connection, source: &str, ctx: &RecallContext) -> Option { - if let Some(crystal_id) = parse_crystal_source_id(source) { - if let Some((label, consolidated_text, member_count, owner_id, visibility)) = - query_crystal_for_unfold(conn, crystal_id) - { - if is_visible(owner_id, visibility.as_deref(), ctx) { - let members = crystal_member_sources(conn, crystal_id, ctx); - let mut full_text = consolidated_text.clone(); - if !members.is_empty() { - full_text.push_str("\n\nFamily members:\n"); - for member in members.iter().take(16) { - full_text.push_str("- "); - full_text.push_str(member); - full_text.push('\n'); - } - if member_count as usize > members.len() { - full_text.push_str(&format!( - "... plus {} more hidden or archived member(s)", - (member_count as usize).saturating_sub(members.len()) - )); - } - } - return Some(json!({ - "source": crystal_source(crystal_id, &label), - "text": full_text.trim_end().to_string(), - "type": "crystal", - "label": label, - "clusterId": crystal_id, - "members": members, - "memberCount": member_count, - })); - } - } - } - - if let Some((text, ty, owner_id, visibility)) = query_memory_for_unfold(conn, source) { - if is_visible(owner_id, visibility.as_deref(), ctx) { - return Some(json!({"text": text, "type": ty})); - } - } - - if let Some(id_str) = source.strip_prefix("decision::") { - if let Ok(id) = id_str.parse::() { - if let Some((decision, context, owner_id, visibility)) = - query_decision_by_id_for_unfold(conn, id) - { - if is_visible(owner_id, visibility.as_deref(), ctx) { - let full = match context { - Some(c) => format!("{decision}\n\nContext: {c}"), - None => decision, - }; - return Some(json!({"text": full, "type": "decision"})); - } - } - } - } - - if let Some((decision, context, owner_id, visibility)) = - query_decision_by_context_for_unfold(conn, source) - { - if is_visible(owner_id, visibility.as_deref(), ctx) { - let full = match context { - Some(c) => format!("{decision}\n\nContext: {c}"), - None => decision, - }; - return Some(json!({"text": full, "type": "decision"})); - } - } - - let stripped = source.strip_prefix("memory::").unwrap_or(source); - if stripped != source { - if let Some((text, ty, owner_id, visibility)) = query_memory_for_unfold(conn, stripped) { - if is_visible(owner_id, visibility.as_deref(), ctx) { - return Some(json!({"text": text, "type": ty})); - } - } - } - - None -} - -pub(crate) type MemoryUnfoldRow = (String, String, Option, Option); -pub(crate) type DecisionUnfoldRow = (String, Option, Option, Option); - -pub(crate) fn query_memory_for_unfold(conn: &Connection, source: &str) -> Option { - let sql_with_visibility = - "SELECT text, type, owner_id, visibility FROM memories WHERE source = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - ORDER BY score DESC LIMIT 1"; - match conn.query_row(sql_with_visibility, params![source], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - )) - }) { - Ok(row) => Some(row), - Err(err) if is_missing_team_visibility_columns(&err) => conn - .query_row( - "SELECT text, type FROM memories WHERE source = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - ORDER BY score DESC LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - None, - None, - )) - }, - ) - .ok(), - Err(_) => None, - } -} - -pub(crate) fn query_decision_by_id_for_unfold(conn: &Connection, id: i64) -> Option { - let sql_with_visibility = - "SELECT decision, context, owner_id, visibility FROM decisions WHERE id = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now'))"; - match conn.query_row(sql_with_visibility, params![id], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - )) - }) { - Ok(row) => Some(row), - Err(err) if is_missing_team_visibility_columns(&err) => conn - .query_row( - "SELECT decision, context FROM decisions WHERE id = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now'))", - params![id], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - None, - None, - )) - }, - ) - .ok(), - Err(_) => None, - } -} - -pub(crate) fn query_decision_by_context_for_unfold( - conn: &Connection, - source: &str, -) -> Option { - let sql_with_visibility = - "SELECT decision, context, owner_id, visibility FROM decisions WHERE context = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - ORDER BY score DESC LIMIT 1"; - match conn.query_row(sql_with_visibility, params![source], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - )) - }) { - Ok(row) => Some(row), - Err(err) if is_missing_team_visibility_columns(&err) => conn - .query_row( - "SELECT decision, context FROM decisions WHERE context = ?1 \ - AND status = 'active' AND (expires_at IS NULL OR expires_at > datetime('now')) \ - AND (valid_from IS NULL OR valid_from <= datetime('now')) \ - AND (valid_until IS NULL OR valid_until > datetime('now')) \ - ORDER BY score DESC LIMIT 1", - params![source], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - None, - None, - )) - }, - ) - .ok(), - Err(_) => None, - } -} - diff --git a/daemon-rs/src/handlers/redaction.rs b/daemon-rs/src/handlers/redaction.rs index d5f6c890..36e8102b 100644 --- a/daemon-rs/src/handlers/redaction.rs +++ b/daemon-rs/src/handlers/redaction.rs @@ -1,13 +1,8 @@ -// SPDX-License-Identifier: MIT use regex::Regex; use std::sync::OnceLock; - static BEARER_REDACTION_RE: OnceLock> = OnceLock::new(); static HASH_REDACTION_RE: OnceLock> = OnceLock::new(); static CREDENTIAL_REDACTION_RE: OnceLock> = OnceLock::new(); - -// Apply redactions in three passes so broad credential masking does not hide -// structured bearer/hash patterns from earlier, more specific replacements. pub fn redact_secrets(text: &str) -> String { let bearer = BEARER_REDACTION_RE .get_or_init(|| Regex::new(r"Bearer\s+[a-f0-9]{32,}").ok()) diff --git a/daemon-rs/src/handlers/store/core.rs b/daemon-rs/src/handlers/store/core.rs index 3cc07112..436fe536 100644 --- a/daemon-rs/src/handlers/store/core.rs +++ b/daemon-rs/src/handlers/store/core.rs @@ -1,85 +1,23 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - use super::*; -pub fn store_decision( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - confidence: Option, - owner_id: Option, -) -> Result<(Value, Option), String> { - let provenance = DecisionProvenance::from_fields(&source_agent, None, None); - store_decision_internal( - conn, - decision, - context, - entry_type, - source_agent, - provenance, - confidence, - None, - None, - None, - owner_id, - ) - .map_err(|err| err.to_string()) -} - +use crate::api_types::RetentionClass; +use crate::conflict::{fetch_recent_decision_candidates, jaccard_token_set, scan_recent_decision_candidates, ConflictClassification}; +use crate::db::checkpoint_wal_best_effort; +use crate::handlers::{log_event, now_iso, truncate_chars}; +use rusqlite::Connection; +use serde_json::{json, Value}; #[allow(clippy::too_many_arguments, dead_code)] pub fn store_decision_with_ttl( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - confidence: Option, - ttl_seconds: Option, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: Option, source_agent: String, confidence: Option, + ttl_seconds: Option, owner_id: Option, ) -> Result<(Value, Option), String> { let provenance = DecisionProvenance::from_fields(&source_agent, None, None); - store_decision_internal( - conn, - decision, - context, - entry_type, - source_agent, - provenance, - confidence, - ttl_seconds, - None, - None, - owner_id, - ) - .map_err(|err| err.to_string()) + store_decision_internal(conn, decision, context, entry_type, source_agent, provenance, confidence, ttl_seconds, None, None, owner_id) + .map_err(|err| err.to_string()) } - #[allow(clippy::too_many_arguments, dead_code)] pub(crate) fn store_decision_with_input_embedding( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - confidence: Option, - ttl_seconds: Option, - query_embedding: Option<&[f32]>, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: Option, source_agent: String, confidence: Option, + ttl_seconds: Option, query_embedding: Option<&[f32]>, owner_id: Option, ) -> Result<(Value, Option), StoreError> { let provenance = DecisionProvenance::from_fields(&source_agent, None, None); store_decision_with_input_embedding_and_provenance( @@ -95,19 +33,10 @@ pub(crate) fn store_decision_with_input_embedding( owner_id, ) } - #[allow(clippy::too_many_arguments)] pub(crate) fn store_decision_with_input_embedding_and_provenance( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - provenance: DecisionProvenance, - confidence: Option, - ttl_seconds: Option, - query_embedding: Option<&[f32]>, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: Option, source_agent: String, provenance: DecisionProvenance, + confidence: Option, ttl_seconds: Option, query_embedding: Option<&[f32]>, owner_id: Option, ) -> Result<(Value, Option), StoreError> { store_decision_with_input_embedding_and_provenance_retention( conn, @@ -123,57 +52,23 @@ pub(crate) fn store_decision_with_input_embedding_and_provenance( owner_id, ) } - #[allow(clippy::too_many_arguments)] pub(crate) fn store_decision_with_input_embedding_and_provenance_retention( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - provenance: DecisionProvenance, - confidence: Option, - ttl_seconds: Option, - retention_class: Option, - query_embedding: Option<&[f32]>, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: Option, source_agent: String, provenance: DecisionProvenance, + confidence: Option, ttl_seconds: Option, retention_class: Option, query_embedding: Option<&[f32]>, owner_id: Option, ) -> Result<(Value, Option), StoreError> { - store_decision_internal( - conn, - decision, - context, - entry_type, - source_agent, - provenance, - confidence, - ttl_seconds, - retention_class, - query_embedding, - owner_id, - ) + store_decision_internal(conn, decision, context, entry_type, source_agent, provenance, confidence, ttl_seconds, retention_class, query_embedding, owner_id) } - #[allow(clippy::too_many_arguments)] pub(crate) fn store_decision_internal( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: Option, - source_agent: String, - provenance: DecisionProvenance, - confidence: Option, - ttl_seconds: Option, - retention_class: Option, - query_embedding: Option<&[f32]>, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: Option, source_agent: String, provenance: DecisionProvenance, + confidence: Option, ttl_seconds: Option, retention_class: Option, query_embedding: Option<&[f32]>, owner_id: Option, ) -> Result<(Value, Option), StoreError> { let entry_type = entry_type.unwrap_or_else(|| "decision".to_string()); - let suppress_benchmark_events = - is_benchmark_entry_type(&entry_type) || is_benchmark_source_agent(&source_agent); + let suppress_benchmark_events = is_benchmark_entry_type(&entry_type) || is_benchmark_source_agent(&source_agent); let mut decision_text = decision.trim().to_string(); let decision_chars = decision_text.chars().count(); - let decision_truncated = - !is_benchmark_entry_type(&entry_type) && decision_chars > MAX_DECISION_CHARS; + let decision_truncated = !is_benchmark_entry_type(&entry_type) && decision_chars > MAX_DECISION_CHARS; if decision_truncated { decision_text = truncate_chars(&decision_text, MAX_DECISION_CHARS); } @@ -182,29 +77,20 @@ pub(crate) fn store_decision_internal( let confidence = confidence.unwrap_or(0.8); let trust_score = provenance.trust_score(confidence); let ts = now_iso(); - let retention_class = - RetentionClass::classify(retention_class, &entry_type, decision, context.as_deref()); + let retention_class = RetentionClass::classify(retention_class, &entry_type, decision, context.as_deref()); let ttl_seconds = validate_explicit_ttl_seconds(ttl_seconds)?; let effective_ttl_seconds = ttl_seconds.or_else(|| retention_class.default_ttl_seconds()); - let expires_at = - compute_expires_at(conn, effective_ttl_seconds).map_err(StoreError::Internal)?; - + let expires_at = compute_expires_at(conn, effective_ttl_seconds).map_err(StoreError::Internal)?; if decision_truncated { let _ = log_event( conn, "decision_truncated", json!({ - "source_agent": source_agent, - "entry_type": entry_type.as_str(), - "original_chars": decision_chars, - "stored_chars": MAX_DECISION_CHARS, - "preview": truncate_chars(decision, 180), - }), +"source_agent":source_agent,"entry_type":entry_type.as_str(),"original_chars":decision_chars,"stored_chars":MAX_DECISION_CHARS, +"preview":truncate_chars(decision,180),}), "rust-daemon", ); } - - // Benchmark ingestion must preserve corpus fidelity (no dedup/conflict collapse). if is_benchmark_entry_type(&entry_type) { return insert_decision( conn, @@ -224,29 +110,14 @@ pub(crate) fn store_decision_internal( !suppress_benchmark_events, ); } - if quality.score < TOO_VAGUE_THRESHOLD { - return Err(StoreError::Validation { - message: "Memory too vague", - quality: quality.score, - factors: quality.factors, - }); + return Err(StoreError::Validation { message: "Memory too vague", quality: quality.score, factors: quality.factors }); } - if let Some(query_vector) = query_embedding { let candidates = fetch_top_semantic_candidates(conn, query_vector, owner_id)?; let dedup_action = choose_semantic_dedup_action(&candidates, decision); - let best_similarity = candidates - .first() - .map(|candidate| candidate.similarity as f64) - .unwrap_or(0.0); - - if let SemanticDedupAction::Merge { - target_id, - similarity, - jaccard, - } = dedup_action - { + let best_similarity = candidates.first().map(|candidate| candidate.similarity as f64).unwrap_or(0.0); + if let SemanticDedupAction::Merge { target_id, similarity, jaccard } = dedup_action { return merge_into_existing_decision( conn, target_id, @@ -260,7 +131,6 @@ pub(crate) fn store_decision_internal( owner_id, ); } - return insert_decision( conn, decision, @@ -279,7 +149,6 @@ pub(crate) fn store_decision_internal( !suppress_benchmark_events, ); } - store_decision_legacy( conn, decision, @@ -296,26 +165,15 @@ pub(crate) fn store_decision_internal( owner_id, ) } - #[allow(clippy::too_many_arguments)] pub(crate) fn store_decision_legacy( - conn: &mut Connection, - decision: &str, - context: Option, - entry_type: &str, - source_agent: &str, - provenance: &DecisionProvenance, - confidence: f64, - trust_score: f64, - quality: i32, - retention_class: RetentionClass, - expires_at: Option, - ts: &str, - owner_id: Option, + conn: &mut Connection, decision: &str, context: Option, entry_type: &str, source_agent: &str, provenance: &DecisionProvenance, confidence: f64, + trust_score: f64, quality: i32, retention_class: RetentionClass, expires_at: Option, ts: &str, owner_id: Option, ) -> Result<(Value, Option), StoreError> { - let relation = - detect_conflict(conn, decision, source_agent, owner_id).map_err(StoreError::Internal)?; - + let decision_tokens = jaccard_token_set(decision); + let recent_candidates = fetch_recent_decision_candidates(conn, owner_id).map_err(StoreError::Internal)?; + let recent_scan = scan_recent_decision_candidates(&recent_candidates, decision, source_agent, &decision_tokens); + let relation = recent_scan.relation; match relation.classification { ConflictClassification::Contradicts => { return handle_contradiction_policy( @@ -336,15 +194,7 @@ pub(crate) fn store_decision_legacy( ); } ConflictClassification::Agrees => { - return handle_agreement_policy( - conn, - decision, - context.as_deref(), - source_agent, - quality, - ts, - &relation, - ); + return handle_agreement_policy(conn, decision, context.as_deref(), source_agent, quality, ts, &relation); } ConflictClassification::Refines => { return handle_refinement_policy( @@ -366,65 +216,21 @@ pub(crate) fn store_decision_legacy( } ConflictClassification::Unrelated => {} } - - let existing: Vec = if let Some(owner_id) = owner_id { - let mut stmt = conn - .prepare( - "SELECT decision FROM decisions \ - WHERE owner_id = ?1 \ - AND status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - ORDER BY created_at DESC LIMIT 50", - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; - let rows = stmt - .query_map(params![owner_id], |row| row.get(0)) - .map_err(|e| StoreError::Internal(e.to_string()))?; - rows.filter_map(|row| row.ok()).collect() - } else { - let mut stmt = conn - .prepare( - "SELECT decision FROM decisions \ - WHERE status = 'active' \ - AND (expires_at IS NULL OR expires_at > datetime('now')) \ - ORDER BY created_at DESC LIMIT 50", - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; - let rows = stmt - .query_map([], |row| row.get(0)) - .map_err(|e| StoreError::Internal(e.to_string()))?; - rows.filter_map(|row| row.ok()).collect() - }; - - let max_sim = existing - .iter() - .map(|text| jaccard_similarity(decision, text)) - .fold(0.0_f64, f64::max); - let surprise = 1.0 - max_sim; - + let surprise = 1.0 - recent_scan.max_jaccard; if surprise < 0.25 { let _ = log_event( conn, "decision_rejected_duplicate", json!({ - "decision": &decision[..decision.len().min(100)], - "surprise": surprise, - "source_agent": source_agent, - "quality": quality, - }), +"decision":&decision[..decision.len().min(100)],"surprise":surprise,"source_agent":source_agent,"quality":quality,}), "rust-daemon", ); checkpoint_wal_best_effort(conn); - let mut entry = json!({ - "stored": false, - "reason": "duplicate", - "surprise": surprise, - "quality": quality, - }); + let mut entry = json!({"stored":false,"reason":"duplicate","surprise":surprise,"quality":quality +,}); decorate_entry_with_relation(&mut entry, &relation, None); return Ok((entry, None)); } - let (mut entry, new_id) = insert_decision( conn, decision, @@ -445,4 +251,3 @@ pub(crate) fn store_decision_legacy( decorate_entry_with_relation(&mut entry, &relation, None); Ok((entry, new_id)) } - diff --git a/daemon-rs/src/handlers/store/embedding.rs b/daemon-rs/src/handlers/store/embedding.rs index 9eb25e3d..2ed6cb9c 100644 --- a/daemon-rs/src/handlers/store/embedding.rs +++ b/daemon-rs/src/handlers/store/embedding.rs @@ -1,26 +1,5 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; use rusqlite::{params, Connection}; -use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; -pub fn persist_decision_embedding( - conn: &Connection, - decision_id: i64, - vector: &[f32], - model_key: &str, -) -> Result<(), String> { +pub fn persist_decision_embedding(conn: &Connection, decision_id: i64, vector: &[f32], model_key: &str) -> Result<(), String> { let blob = crate::embeddings::vector_to_blob(vector); conn.execute( "INSERT OR REPLACE INTO embeddings (target_type, target_id, vector, model) \ @@ -30,4 +9,3 @@ pub fn persist_decision_embedding( .map(|_| ()) .map_err(|e| format!("Failed to persist decision embedding: {e}")) } - diff --git a/daemon-rs/src/handlers/store/handler.rs b/daemon-rs/src/handlers/store/handler.rs index 06b01f6e..7e46e53a 100644 --- a/daemon-rs/src/handlers/store/handler.rs +++ b/daemon-rs/src/handlers/store/handler.rs @@ -1,87 +1,43 @@ -// SPDX-License-Identifier: MIT +use super::*; +use crate::api_types::StoreRequest; +use crate::budgets::BudgetEndpoint; +use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, require_team_caller, resolve_source_identity}; +use crate::rate_limit::RequestClass; +use crate::state::RuntimeState; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; use axum::Json; -use rusqlite::{params, Connection}; -use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; -pub async fn handle_store( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Response { - let caller_id = match ensure_auth_with_caller_rated_for_class( - &headers, - &state, - RequestClass::Store, - ) - .await - { +use serde_json::json; +pub async fn handle_store(State(state): State, headers: HeaderMap, Json(body): Json) -> Response { + let caller_id = match ensure_auth_with_caller_rated_for_class(&headers, &state, RequestClass::Store).await { Ok(id) => id, Err(resp) => return resp, }; - if state.team_mode && caller_id.is_none() { - return json_response( - StatusCode::FORBIDDEN, - json!({ "error": "Team mode requires a caller-scoped ctx_ API key" }), - ); + if let Err(resp) = require_team_caller(&state, caller_id) { + return resp; } - let decision = body.decision.unwrap_or_default(); if decision.trim().is_empty() { - return json_response( - StatusCode::BAD_REQUEST, - json!({ "error": "Missing field: decision" }), - ); + return json_response(StatusCode::BAD_REQUEST, json!({"error":"Missing field: decision"})); } - - let source_identity = - resolve_source_identity(&headers, body.source_agent.as_deref().unwrap_or("http")); + let source_identity = resolve_source_identity(&headers, body.source_agent.as_deref().unwrap_or("http")); let source_agent = source_identity.agent.clone(); - if let Err(resp) = - ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Store, &source_agent).await - { + if let Err(resp) = ensure_endpoint_budget(&headers, &state, BudgetEndpoint::Store, &source_agent).await { return resp; } - let benchmark_store = body - .entry_type - .as_deref() - .map(is_benchmark_entry_type) - .unwrap_or(false) - || is_benchmark_source_agent(&source_agent); - let provenance = DecisionProvenance::from_fields( - &source_agent, - body.source_model - .as_deref() - .or(source_identity.model.as_deref()), - body.reasoning_depth.as_deref(), - ); - + let benchmark_store = body.entry_type.as_deref().map(is_benchmark_entry_type).unwrap_or(false) || is_benchmark_source_agent(&source_agent); + let provenance = + DecisionProvenance::from_fields(&source_agent, body.source_model.as_deref().or(source_identity.model.as_deref()), body.reasoning_depth.as_deref()); if let Err(StoreError::BadRequest(message)) = validate_explicit_ttl_seconds(body.ttl_seconds) { - return json_response(StatusCode::BAD_REQUEST, json!({ "error": message })); + return json_response(StatusCode::BAD_REQUEST, json!({"error":message})); } - let decision_text = decision.trim().to_string(); - let embedding_model_key = state - .embedding_engine - .as_ref() - .map(|engine| engine.model_key()) - .unwrap_or(crate::embeddings::selected_model_key()); + let embedding_model_key = state.embedding_engine.as_ref().map(|engine| engine.model_key()).unwrap_or(crate::embeddings::selected_model_key()); let decision_embedding = match state.embedding_engine.clone() { Some(engine) => engine.embed_async(decision_text.clone()).await, None => None, }; - let mut conn = state.db.lock().await; let result = store_decision_with_input_embedding_and_provenance_retention( &mut conn, @@ -96,17 +52,12 @@ pub async fn handle_store( decision_embedding.as_deref(), caller_id, ); - match result { Ok((entry, new_id)) => { if let Some(id) = new_id { if let Some(vec) = decision_embedding.as_deref() { - if let Err(err) = - persist_decision_embedding(&conn, id, vec, embedding_model_key) - { - eprintln!( - "[store] Warning: failed to persist decision embedding for {id}: {err}" - ); + if let Err(err) = persist_decision_embedding(&conn, id, vec, embedding_model_key) { + eprintln!("[store] Warning: failed to persist decision embedding for {id}: {err}"); } } else if let Some(engine) = state.embedding_engine.clone() { let db = state.db.clone(); @@ -120,31 +71,25 @@ pub async fn handle_store( }); } } - if !benchmark_store { crate::focus::focus_append(&conn, &source_agent, &decision_text); } - json_response(StatusCode::OK, json!({ "stored": true, "entry": entry })) - } - Err(StoreError::BadRequest(message)) => { - json_response(StatusCode::BAD_REQUEST, json!({ "error": message })) + json_response( + StatusCode::OK, + json!({"stored":true, +"entry":entry}), + ) } - Err(StoreError::Validation { - message, - quality, - factors, - }) => json_response( + Err(StoreError::BadRequest(message)) => json_response(StatusCode::BAD_REQUEST, json!({"error":message})), + Err(StoreError::Validation { message, quality, factors }) => json_response( StatusCode::BAD_REQUEST, - json!({ - "error": message, - "quality": quality, - "factors": factors.as_json(), - }), + json!({"error":message,"quality":quality, +"factors":factors.as_json(),}), ), Err(StoreError::Internal(err)) => json_response( StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": format!("Store failed: {err}") }), + json!({"error": +format!("Store failed: {err}")}), ), } } - diff --git a/daemon-rs/src/handlers/store/insert.rs b/daemon-rs/src/handlers/store/insert.rs index 18a3218d..4633eb81 100644 --- a/daemon-rs/src/handlers/store/insert.rs +++ b/daemon-rs/src/handlers/store/insert.rs @@ -1,183 +1,67 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; +use super::*; +use crate::api_types::RetentionClass; +use crate::conflict::{jaccard_similarity, ConflictClassification, ConflictResult}; use rusqlite::{params, Connection}; use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; pub(crate) fn insert_decision_with_state( - tx: &rusqlite::Transaction<'_>, - decision: &str, - context: Option<&str>, - entry_type: &str, - source_agent: &str, - provenance: &DecisionProvenance, - confidence: f64, - trust_score: f64, - quality: i32, - retention_class: RetentionClass, - expires_at: Option<&str>, - ts: &str, - owner_id: Option, - status: &str, - disputes_id: Option, - supersedes_id: Option, - surprise: Option, + tx: &rusqlite::Transaction<'_>, decision: &str, context: Option<&str>, entry_type: &str, source_agent: &str, provenance: &DecisionProvenance, + confidence: f64, trust_score: f64, quality: i32, retention_class: RetentionClass, expires_at: Option<&str>, ts: &str, owner_id: Option, status: &str, + disputes_id: Option, supersedes_id: Option, surprise: Option, ) -> Result { let surprise = surprise.map(round4); - if let Some(oid) = owner_id { - tx.execute( - "INSERT INTO decisions \ + if let Some +(oid)=owner_id{tx.execute( +"INSERT INTO decisions \ (decision, context, type, source_agent, confidence, surprise, status, disputes_id, supersedes_id, owner_id, quality, retention_class, expires_at, created_at, updated_at, source_client, source_model, reasoning_depth, trust_score) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18)", - params![ - decision, - context, - entry_type, - source_agent, - confidence, - surprise, - status, - disputes_id, - supersedes_id, - oid, - quality, - retention_class.as_str(), - expires_at, - ts, - provenance.source_client.as_str(), - provenance.source_model.as_deref(), - provenance.reasoning_depth.as_str(), - trust_score, - ], - ) - } else { - tx.execute( - "INSERT INTO decisions \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18)" +,params![decision,context,entry_type,source_agent,confidence,surprise,status,disputes_id,supersedes_id,oid,quality,retention_class +.as_str(),expires_at,ts,provenance.source_client.as_str(),provenance.source_model.as_deref(),provenance.reasoning_depth.as_str(), +trust_score,],)}else{tx.execute( +"INSERT INTO decisions \ (decision, context, type, source_agent, confidence, surprise, status, disputes_id, supersedes_id, quality, retention_class, expires_at, created_at, updated_at, source_client, source_model, reasoning_depth, trust_score) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?13, ?14, ?15, ?16, ?17)", - params![ - decision, - context, - entry_type, - source_agent, - confidence, - surprise, - status, - disputes_id, - supersedes_id, - quality, - retention_class.as_str(), - expires_at, - ts, - provenance.source_client.as_str(), - provenance.source_model.as_deref(), - provenance.reasoning_depth.as_str(), - trust_score, - ], - ) - } - .map_err(|e| StoreError::Internal(e.to_string()))?; - + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?13, ?14, ?15, ?16, ?17)" +,params![decision,context,entry_type,source_agent,confidence,surprise,status,disputes_id,supersedes_id,quality,retention_class. +as_str(),expires_at,ts,provenance.source_client.as_str(),provenance.source_model.as_deref(),provenance.reasoning_depth.as_str(), +trust_score,],)}.map_err(|e|StoreError::Internal(e.to_string()))?; Ok(tx.last_insert_rowid()) } - #[allow(clippy::too_many_arguments)] pub(crate) fn insert_conflict_record( - tx: &rusqlite::Transaction<'_>, - source_decision_id: Option, - target_decision_id: i64, - classification: ConflictClassification, - similarity_jaccard: f64, - similarity_cosine: Option, - status: &str, - resolution_strategy: Option<&str>, - resolved_by: Option<&str>, - ts: &str, + tx: &rusqlite::Transaction<'_>, source_decision_id: Option, target_decision_id: i64, classification: ConflictClassification, similarity_jaccard: f64, + similarity_cosine: Option, status: &str, resolution_strategy: Option<&str>, resolved_by: Option<&str>, ts: &str, ) -> Result { let resolved_at = if status == "open" { None } else { Some(ts) }; tx.execute( - "INSERT INTO decision_conflicts \ +"INSERT INTO decision_conflicts \ (source_decision_id, target_decision_id, classification, similarity_jaccard, similarity_cosine, status, resolution_strategy, resolved_by, resolved_at, created_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", - params![ - source_decision_id, - target_decision_id, - classification.as_str(), - round4(similarity_jaccard), - similarity_cosine.map(round4), - status, - resolution_strategy, - resolved_by, - resolved_at, - ts, - ], - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)" +,params![source_decision_id,target_decision_id,classification.as_str(),round4(similarity_jaccard),similarity_cosine.map(round4), +status,resolution_strategy,resolved_by,resolved_at,ts,],).map_err(|e|StoreError::Internal(e.to_string()))?; Ok(tx.last_insert_rowid()) } - -pub(crate) fn decorate_entry_with_relation( - entry: &mut Value, - relation: &ConflictResult, - conflict_record: Option, -) { +pub(crate) fn decorate_entry_with_relation(entry: &mut Value, relation: &ConflictResult, conflict_record: Option) { if let Some(object) = entry.as_object_mut() { - object.insert( - "classification".to_string(), - json!(relation.classification.as_str()), - ); + object.insert("classification".to_string(), json!(relation.classification.as_str())); object.insert("relation".to_string(), relation_to_json(relation)); if let Some(conflict_record) = conflict_record { object.insert("conflict".to_string(), conflict_record); } } } - pub(crate) fn relation_to_json(relation: &ConflictResult) -> Value { - json!({ - "matched_id": relation.matched_id, - "matched_agent": relation.matched_agent, - "matched_trust_score": relation.matched_trust_score.map(round4), - "similarity": { - "jaccard": round4(relation.similarity_jaccard), - "cosine": relation.similarity_cosine.map(round4), - }, - }) + json!({"matched_id":relation.matched_id, +"matched_agent":relation.matched_agent,"matched_trust_score":relation.matched_trust_score.map(round4),"similarity":{"jaccard": +round4(relation.similarity_jaccard),"cosine":relation.similarity_cosine.map(round4),},}) } - pub(crate) fn conflict_record_json( - record_id: i64, - source_decision_id: Option, - target_decision_id: i64, - classification: ConflictClassification, - status: &str, - strategy: Option<&str>, + record_id: i64, source_decision_id: Option, target_decision_id: i64, classification: ConflictClassification, status: &str, strategy: Option<&str>, ) -> Value { - json!({ - "id": record_id, - "source_decision_id": source_decision_id, - "target_decision_id": target_decision_id, - "classification": classification.as_str(), - "status": status, - "resolution_strategy": strategy, - }) + json!({"id":record_id,"source_decision_id":source_decision_id,"target_decision_id":target_decision_id, +"classification":classification.as_str(),"status":status,"resolution_strategy":strategy,}) } - pub(crate) fn round4(value: f64) -> f64 { (value * 10_000.0).round() / 10_000.0 } - pub(crate) fn assess_quality(text: &str) -> QualityAssessment { let trimmed = text.trim(); let len = trimmed.chars().count(); @@ -190,161 +74,101 @@ pub(crate) fn assess_quality(text: &str) -> QualityAssessment { } else { 100 }; - - let specificity_bonus = if has_specificity_markers(trimmed) { - 20 - } else { - 0 - }; + let specificity_bonus = if has_specificity_markers(trimmed) { 20 } else { 0 }; let question_penalty = if trimmed.ends_with('?') { -30 } else { 0 }; let score = (length_score + specificity_bonus + question_penalty).clamp(0, 100); - - QualityAssessment { - score, - factors: QualityFactors { - length_score, - specificity_bonus, - question_penalty, - }, - } + QualityAssessment { score, factors: QualityFactors { length_score, specificity_bonus, question_penalty } } } - pub(crate) fn has_specificity_markers(text: &str) -> bool { let lower = text.to_lowercase(); - let file_extensions = [ - ".rs", ".go", ".py", ".ts", ".tsx", ".js", ".jsx", ".json", ".toml", ".yaml", ".yml", - ".sql", ".md", - ]; - let code_prefixes = [ - "fn ", "func ", "def ", "class ", "struct ", "impl ", "select ", "insert ", "update ", - "delete ", - ]; - + let file_extensions = [".rs", ".go", ".py", ".ts", ".tsx", ".js", ".jsx", ".json", ".toml", ".yaml", ".yml", ".sql", ".md"]; + let code_prefixes = ["fn ", "func ", "def ", "class ", "struct ", "impl ", "select ", "insert ", "update ", "delete "]; let has_path = text.contains('/') || text.contains('\\'); let has_extension = file_extensions.iter().any(|ext| lower.contains(ext)); - let has_function = text.contains("::") - || text.contains("()") - || text.contains("->") - || code_prefixes.iter().any(|needle| lower.contains(needle)); - let has_identifier = text - .split_whitespace() - .any(|token| token.contains('_') && token.chars().any(|ch| ch.is_ascii_alphabetic())); - + let has_function = text.contains("::") || text.contains("()") || text.contains("->") || code_prefixes.iter().any(|needle| lower.contains(needle)); + let has_identifier = text.split_whitespace().any(|token| token.contains('_') && token.chars().any(|ch| ch.is_ascii_alphabetic())); has_path || has_extension || has_function || has_identifier } - -pub(crate) fn choose_semantic_dedup_action( - candidates: &[SemanticCandidate], - incoming_text: &str, -) -> SemanticDedupAction { +pub(crate) fn choose_semantic_dedup_action(candidates: &[SemanticCandidate], incoming_text: &str) -> SemanticDedupAction { for candidate in candidates { let jaccard = jaccard_similarity(incoming_text, &candidate.decision); if should_merge_candidate(candidate.similarity, jaccard) { - return SemanticDedupAction::Merge { - target_id: candidate.id, - similarity: candidate.similarity, - jaccard, - }; + return SemanticDedupAction::Merge { target_id: candidate.id, similarity: candidate.similarity, jaccard }; } } SemanticDedupAction::Insert } - pub(crate) fn should_merge_candidate(similarity: f32, jaccard: f64) -> bool { if similarity > HARD_MERGE_THRESHOLD { return true; } - (REVIEW_MERGE_THRESHOLD..=HARD_MERGE_THRESHOLD).contains(&similarity) - && jaccard > JACCARD_MERGE_THRESHOLD + (REVIEW_MERGE_THRESHOLD..=HARD_MERGE_THRESHOLD).contains(&similarity) && jaccard > JACCARD_MERGE_THRESHOLD } -pub(crate) fn fetch_top_semantic_candidates( - conn: &Connection, - query_vector: &[f32], - owner_id: Option, -) -> Result, StoreError> { +pub(crate) fn fetch_top_semantic_candidates(conn: &Connection, query_vector: &[f32], owner_id: Option) -> Result, StoreError> { + let selected_model = crate::embeddings::selected_model_key().to_ascii_lowercase(); + let legacy_vector_bytes = std::mem::size_of_val(query_vector) as i64; + let pq8_vector_bytes = (crate::embeddings::PQ8_HEADER_BYTES + query_vector.len()) as i64; let (sql, has_owner_scope) = if owner_id.is_some() { ( - "SELECT d.id, d.decision, d.context, e.vector \ + "SELECT d.id, d.decision, e.vector \ FROM decisions d \ JOIN embeddings e ON e.target_type = 'decision' AND e.target_id = d.id \ WHERE d.owner_id = ?1 \ AND d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now'))", + AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ + AND LOWER(COALESCE(e.model, '')) = ?2 \ + AND length(e.vector) IN (?3, ?4)", true, ) } else { ( - "SELECT d.id, d.decision, d.context, e.vector \ + "SELECT d.id, d.decision, e.vector \ FROM decisions d \ JOIN embeddings e ON e.target_type = 'decision' AND e.target_id = d.id \ WHERE d.status = 'active' \ - AND (d.expires_at IS NULL OR d.expires_at > datetime('now'))", + AND (d.expires_at IS NULL OR d.expires_at > datetime('now')) \ + AND LOWER(COALESCE(e.model, '')) = ?1 \ + AND length(e.vector) IN (?2, ?3)", false, ) }; - let mut stmt = conn - .prepare(sql) - .map_err(|e| StoreError::Internal(e.to_string()))?; - + let mut stmt = conn.prepare(sql).map_err(|error| StoreError::Internal(error.to_string()))?; let mut candidates = Vec::new(); if has_owner_scope { let rows = stmt - .query_map([owner_id.unwrap_or_default()], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Vec>(3)?, - )) + .query_map(params![owner_id.unwrap_or_default(), selected_model, legacy_vector_bytes, pq8_vector_bytes], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Vec>(2)?)) }) - .map_err(|e| StoreError::Internal(e.to_string()))?; + .map_err(|error| StoreError::Internal(error.to_string()))?; for row in rows.flatten() { - let (id, decision, _context, blob) = row; + let (id, decision, blob) = row; let existing_vec = crate::embeddings::blob_to_vector(&blob); - if existing_vec.len() != query_vector.len() { - continue; - } let similarity = crate::embeddings::cosine_similarity(query_vector, &existing_vec); - candidates.push(SemanticCandidate { - id, - decision, - similarity, - }); + let candidate = SemanticCandidate { id, decision, similarity }; + if similarity >= 1.0 { + return Ok(vec![candidate]); + } + candidates.push(candidate); } } else { let rows = stmt - .query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Vec>(3)?, - )) + .query_map(params![selected_model, legacy_vector_bytes, pq8_vector_bytes], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Vec>(2)?)) }) - .map_err(|e| StoreError::Internal(e.to_string()))?; + .map_err(|error| StoreError::Internal(error.to_string()))?; for row in rows.flatten() { - let (id, decision, _context, blob) = row; + let (id, decision, blob) = row; let existing_vec = crate::embeddings::blob_to_vector(&blob); - if existing_vec.len() != query_vector.len() { - continue; - } let similarity = crate::embeddings::cosine_similarity(query_vector, &existing_vec); - candidates.push(SemanticCandidate { - id, - decision, - similarity, - }); + let candidate = SemanticCandidate { id, decision, similarity }; + if similarity >= 1.0 { + return Ok(vec![candidate]); + } + candidates.push(candidate); } } - - candidates.sort_by(|left, right| { - right - .similarity - .partial_cmp(&left.similarity) - .unwrap_or(std::cmp::Ordering::Equal) - }); + candidates.sort_by(|left, right| right.similarity.partial_cmp(&left.similarity).unwrap_or(std::cmp::Ordering::Equal)); candidates.truncate(3); Ok(candidates) } - diff --git a/daemon-rs/src/handlers/store/merge.rs b/daemon-rs/src/handlers/store/merge.rs index ef191c8c..6291ff7d 100644 --- a/daemon-rs/src/handlers/store/merge.rs +++ b/daemon-rs/src/handlers/store/merge.rs @@ -1,40 +1,15 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; +use super::*; +use crate::api_types::RetentionClass; +use crate::db::checkpoint_wal_best_effort; +use crate::handlers::log_event; use rusqlite::{params, Connection}; use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; pub(crate) fn merge_into_existing_decision( - conn: &mut Connection, - target_id: i64, - incoming_text: &str, - incoming_context: Option<&str>, - source_agent: &str, - quality: i32, - similarity: f32, - jaccard: f64, - ts: &str, - owner_id: Option, + conn: &mut Connection, target_id: i64, incoming_text: &str, incoming_context: Option<&str>, source_agent: &str, quality: i32, similarity: f32, + jaccard: f64, ts: &str, owner_id: Option, ) -> Result<(Value, Option), StoreError> { - let tx = conn - .transaction() - .map_err(|e| StoreError::Internal(e.to_string()))?; - let (existing_decision, existing_context, previous_merged_count): ( - String, - Option, - i64, - ) = if let Some(owner_id) = owner_id { + let tx = conn.transaction().map_err(|e| StoreError::Internal(e.to_string()))?; + let (existing_decision, existing_context, previous_merged_count): (String, Option, i64) = if let Some(owner_id) = owner_id { tx.query_row( "SELECT decision, context, COALESCE(merged_count, 0) \ FROM decisions WHERE id = ?1 AND owner_id = ?2", @@ -43,20 +18,12 @@ pub(crate) fn merge_into_existing_decision( ) .map_err(|e| StoreError::Internal(e.to_string()))? } else { - tx.query_row( - "SELECT decision, context, COALESCE(merged_count, 0) FROM decisions WHERE id = ?1", - params![target_id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) + tx.query_row("SELECT decision, context, COALESCE(merged_count, 0) FROM decisions WHERE id = ?1", params![target_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) .map_err(|e| StoreError::Internal(e.to_string()))? }; - - let merged_context = merge_context( - existing_context, - &existing_decision, - incoming_context, - incoming_text, - ); + let merged_context = merge_context(existing_context, &existing_decision, incoming_context, incoming_text); let merged_count = previous_merged_count + 1; if let Some(owner_id) = owner_id { tx.execute( @@ -67,15 +34,7 @@ pub(crate) fn merge_into_existing_decision( quality = MAX(COALESCE(quality, 50), ?4), \ updated_at = ?5 \ WHERE id = ?6 AND owner_id = ?7", - params![ - merged_context, - MERGE_SCORE_BONUS, - merged_count, - quality, - ts, - target_id, - owner_id - ], + params![merged_context, MERGE_SCORE_BONUS, merged_count, quality, ts, target_id, owner_id], ) .map_err(|e| StoreError::Internal(e.to_string()))?; } else { @@ -87,70 +46,37 @@ pub(crate) fn merge_into_existing_decision( quality = MAX(COALESCE(quality, 50), ?4), \ updated_at = ?5 \ WHERE id = ?6", - params![ - merged_context, - MERGE_SCORE_BONUS, - merged_count, - quality, - ts, - target_id - ], + params![merged_context, MERGE_SCORE_BONUS, merged_count, quality, ts, target_id], ) .map_err(|e| StoreError::Internal(e.to_string()))?; } - let _ = log_event( &tx, "merge", - json!({ - "source_id": Value::Null, - "target_id": target_id, - "target_type": "decision", - "incoming_text": incoming_text, - "similarity": similarity, - "jaccard": jaccard, - "source_agent": source_agent, - }), + json!({"source_id":Value::Null,"target_id":target_id,"target_type":"decision","incoming_text": +incoming_text,"similarity":similarity,"jaccard":jaccard,"source_agent":source_agent,}), "rust-daemon", ); - tx.commit() - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.commit().map_err(|e| StoreError::Internal(e.to_string()))?; checkpoint_wal_best_effort(conn); - Ok(( - json!({ - "action": "merged", - "target_id": target_id, - "merged_count": merged_count, - "quality": quality, - "similarity": similarity, - "jaccard": jaccard, - }), + json!({"action":"merged","target_id":target_id, +"merged_count":merged_count,"quality":quality,"similarity":similarity,"jaccard":jaccard,}), None, )) } - -pub(crate) fn merge_context( - existing_context: Option, - existing_decision: &str, - incoming_context: Option<&str>, - incoming_text: &str, -) -> Option { +pub(crate) fn merge_context(existing_context: Option, existing_decision: &str, incoming_context: Option<&str>, incoming_text: &str) -> Option { let incoming_note = incoming_context .map(str::trim) .filter(|text| !text.is_empty()) .map(ToOwned::to_owned) .unwrap_or_else(|| incoming_text.trim().to_string()); - if incoming_note.is_empty() || incoming_note.eq_ignore_ascii_case(existing_decision.trim()) { return existing_context; } - match existing_context { Some(existing) if !existing.trim().is_empty() => { - let already_present = existing - .split("\n\n") - .any(|part| part.trim().eq_ignore_ascii_case(&incoming_note)); + let already_present = existing.split("\n\n").any(|part| part.trim().eq_ignore_ascii_case(&incoming_note)); if already_present { Some(existing) } else { @@ -160,115 +86,44 @@ pub(crate) fn merge_context( _ => Some(incoming_note), } } - #[allow(clippy::too_many_arguments)] pub(crate) fn insert_decision( - conn: &Connection, - decision: &str, - context: Option, - entry_type: &str, - source_agent: &str, - provenance: &DecisionProvenance, - confidence: f64, - trust_score: f64, - quality: i32, - retention_class: RetentionClass, - expires_at: Option, - ts: &str, - owner_id: Option, - surprise: f64, + conn: &Connection, decision: &str, context: Option, entry_type: &str, source_agent: &str, provenance: &DecisionProvenance, confidence: f64, + trust_score: f64, quality: i32, retention_class: RetentionClass, expires_at: Option, ts: &str, owner_id: Option, surprise: f64, emit_decision_stored_event: bool, ) -> Result<(Value, Option), StoreError> { let surprise = (surprise * 10_000.0).round() / 10_000.0; - if let Some(oid) = owner_id { - conn.execute( - "INSERT INTO decisions \ + if let +Some(oid)=owner_id{conn.execute( +"INSERT INTO decisions \ (decision, context, type, source_agent, confidence, surprise, status, owner_id, quality, retention_class, expires_at, created_at, updated_at, source_client, source_model, reasoning_depth, trust_score) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'active', ?7, ?8, ?9, ?10, ?11, ?11, ?12, ?13, ?14, ?15)", - params![ - decision, - context, - entry_type, - source_agent, - confidence, - surprise, - oid, - quality, - retention_class.as_str(), - expires_at, - ts, - provenance.source_client.as_str(), - provenance.source_model.as_deref(), - provenance.reasoning_depth.as_str(), - trust_score, - ], - ) - } else { - conn.execute( - "INSERT INTO decisions \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'active', ?7, ?8, ?9, ?10, ?11, ?11, ?12, ?13, ?14, ?15)" +,params![decision,context,entry_type,source_agent,confidence,surprise,oid,quality,retention_class.as_str(),expires_at,ts, +provenance.source_client.as_str(),provenance.source_model.as_deref(),provenance.reasoning_depth.as_str(),trust_score,],)}else{conn +.execute( +"INSERT INTO decisions \ (decision, context, type, source_agent, confidence, surprise, status, quality, retention_class, expires_at, created_at, updated_at, source_client, source_model, reasoning_depth, trust_score) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'active', ?7, ?8, ?9, ?10, ?10, ?11, ?12, ?13, ?14)", - params![ - decision, - context, - entry_type, - source_agent, - confidence, - surprise, - quality, - retention_class.as_str(), - expires_at, - ts, - provenance.source_client.as_str(), - provenance.source_model.as_deref(), - provenance.reasoning_depth.as_str(), - trust_score, - ], - ) - } - .map_err(|e| StoreError::Internal(e.to_string()))?; - + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'active', ?7, ?8, ?9, ?10, ?10, ?11, ?12, ?13, ?14)" +,params![decision,context,entry_type,source_agent,confidence,surprise,quality,retention_class.as_str(),expires_at,ts,provenance. +source_client.as_str(),provenance.source_model.as_deref(),provenance.reasoning_depth.as_str(),trust_score,],)}.map_err(|e| +StoreError::Internal(e.to_string()))?; let id = conn.last_insert_rowid(); if emit_decision_stored_event { - let _ = log_event( - conn, - "decision_stored", - json!({ - "id": id, - "source_agent": source_agent, - "surprise": surprise, - "quality": quality, - }), - "rust-daemon", - ); + let _ = log_event(conn, "decision_stored", json!({"id":id,"source_agent":source_agent,"surprise":surprise,"quality":quality,}), "rust-daemon"); } checkpoint_wal_best_effort(conn); - Ok(( - json!({ - "action": "inserted", - "id": id, - "status": "active", - "retention_class": retention_class.as_str(), - "surprise": surprise, - "quality": quality, - }), + json!({"action":"inserted","id":id,"status":"active","retention_class":retention_class.as_str +(),"surprise":surprise,"quality":quality,}), Some(id), )) } - -pub(crate) fn compute_expires_at( - conn: &Connection, - ttl_seconds: Option, -) -> Result, String> { +pub(crate) fn compute_expires_at(conn: &Connection, ttl_seconds: Option) -> Result, String> { let Some(ttl_seconds) = ttl_seconds else { return Ok(None); }; let modifier = format!("+{ttl_seconds} seconds"); - conn.query_row("SELECT datetime('now', ?1)", params![modifier], |row| { - row.get(0) - }) - .map(Some) - .map_err(|e| format!("Failed to compute expires_at: {e}")) + conn.query_row("SELECT datetime('now', ?1)", params![modifier], |row| row.get(0)) + .map(Some) + .map_err(|e| format!("Failed to compute expires_at: {e}")) } - diff --git a/daemon-rs/src/handlers/store/mod.rs b/daemon-rs/src/handlers/store/mod.rs index 1950cbd5..5be45c37 100644 --- a/daemon-rs/src/handlers/store/mod.rs +++ b/daemon-rs/src/handlers/store/mod.rs @@ -1,7 +1,19 @@ -// SPDX-License-Identifier: MIT -mod core; mod embedding; mod handler; mod insert; mod merge; mod policies; mod types; -#[cfg(test)] mod tests; -pub(crate) use types::*; pub(crate) use core::*; pub(crate) use policies::*; pub(crate) use insert::*; pub(crate) use merge::*; -pub use handler::handle_store; pub use core::{store_decision, store_decision_with_ttl}; pub use embedding::persist_decision_embedding; -pub(crate) use core::{store_decision_with_input_embedding, store_decision_with_input_embedding_and_provenance, store_decision_with_input_embedding_and_provenance_retention}; -pub(crate) use types::{DecisionProvenance, validate_explicit_ttl_seconds}; +mod core; +mod embedding; +mod handler; +mod insert; +mod merge; +mod policies; +#[cfg(test)] +mod tests; +mod types; +pub(crate) use core::store_decision_with_input_embedding_and_provenance_retention; +#[cfg(test)] +pub(crate) use core::{store_decision_with_input_embedding, store_decision_with_ttl}; +pub use embedding::persist_decision_embedding; +pub use handler::handle_store; +pub(crate) use insert::*; +pub(crate) use merge::*; +pub(crate) use policies::*; +pub(crate) use types::*; +pub(crate) use types::{validate_explicit_ttl_seconds, DecisionProvenance}; diff --git a/daemon-rs/src/handlers/store/policies.rs b/daemon-rs/src/handlers/store/policies.rs index 90dd3b1f..2c8887cc 100644 --- a/daemon-rs/src/handlers/store/policies.rs +++ b/daemon-rs/src/handlers/store/policies.rs @@ -1,67 +1,28 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; +use super::*; +use crate::api_types::RetentionClass; +use crate::conflict::ConflictResult; +use crate::db::checkpoint_wal_best_effort; +use crate::handlers::log_event; use rusqlite::{params, Connection}; use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; pub(crate) fn handle_contradiction_policy( - conn: &mut Connection, - decision: &str, - context: Option<&str>, - entry_type: &str, - source_agent: &str, - provenance: &DecisionProvenance, - confidence: f64, - trust_score: f64, - quality: i32, - retention_class: RetentionClass, - expires_at: Option<&str>, - ts: &str, - owner_id: Option, - relation: &ConflictResult, + conn: &mut Connection, decision: &str, context: Option<&str>, entry_type: &str, source_agent: &str, provenance: &DecisionProvenance, confidence: f64, + trust_score: f64, quality: i32, retention_class: RetentionClass, expires_at: Option<&str>, ts: &str, owner_id: Option, relation: &ConflictResult, ) -> Result<(Value, Option), StoreError> { - let existing_id = relation - .matched_id - .ok_or_else(|| StoreError::Internal("Missing conflict target id".to_string()))?; + let existing_id = relation.matched_id.ok_or_else(|| StoreError::Internal("Missing conflict target id".to_string()))?; let existing_trust = relation.matched_trust_score.unwrap_or(0.8); let incoming_wins = trust_score > existing_trust; - let strategy = if incoming_wins { - "trust_score_source_wins" - } else { - "trust_score_target_wins" - }; - - let tx = conn - .transaction() - .map_err(|e| StoreError::Internal(e.to_string()))?; - + let strategy = if incoming_wins { "trust_score_source_wins" } else { "trust_score_target_wins" }; + let tx = conn.transaction().map_err(|e| StoreError::Internal(e.to_string()))?; if incoming_wins { if let Some(owner_id) = owner_id { - tx.execute( - "UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2 AND owner_id = ?3", - params![ts, existing_id, owner_id], - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.execute("UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2 AND owner_id = ?3", params![ts, existing_id, owner_id]) + .map_err(|e| StoreError::Internal(e.to_string()))?; } else { - tx.execute( - "UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2", - params![ts, existing_id], - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.execute("UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2", params![ts, existing_id]) + .map_err(|e| StoreError::Internal(e.to_string()))?; } } - let new_id = insert_decision_with_state( &tx, decision, @@ -77,19 +38,10 @@ pub(crate) fn handle_contradiction_policy( ts, owner_id, if incoming_wins { "active" } else { "disputed" }, - if incoming_wins { - None - } else { - Some(existing_id) - }, - if incoming_wins { - Some(existing_id) - } else { - None - }, + if incoming_wins { None } else { Some(existing_id) }, + if incoming_wins { Some(existing_id) } else { None }, Some((1.0 - relation.similarity_jaccard).clamp(0.0, 1.0)), )?; - let conflict_record_id = insert_conflict_record( &tx, Some(new_id), @@ -102,83 +54,40 @@ pub(crate) fn handle_contradiction_policy( Some("policy_engine"), ts, )?; - let _ = log_event( &tx, "decision_conflict", - json!({ - "newId": new_id, - "existingId": existing_id, - "source_agent": source_agent, - "matchedAgent": relation.matched_agent, - "strategy": strategy, - "source_trust_score": trust_score, - "target_trust_score": existing_trust, - "conflict_record_id": conflict_record_id, - }), + json +!({"newId":new_id,"existingId":existing_id,"source_agent":source_agent,"matchedAgent":relation.matched_agent,"strategy":strategy, +"source_trust_score":trust_score,"target_trust_score":existing_trust,"conflict_record_id":conflict_record_id,}), "rust-daemon", ); - - tx.commit() - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.commit().map_err(|e| StoreError::Internal(e.to_string()))?; checkpoint_wal_best_effort(conn); - - let mut entry = json!({ - "action": "inserted", - "id": new_id, - "status": if incoming_wins { "active" } else { "disputed" }, - "retention_class": retention_class.as_str(), - "quality": quality, - "conflictWith": existing_id, - "resolution_strategy": strategy, - }); + let mut entry = json!({"action": +"inserted","id":new_id,"status":if incoming_wins{"active"}else{"disputed"},"retention_class":retention_class.as_str(),"quality": +quality,"conflictWith":existing_id,"resolution_strategy":strategy,}); if incoming_wins { entry["supersedes"] = json!(existing_id); } decorate_entry_with_relation( &mut entry, relation, - Some(conflict_record_json( - conflict_record_id, - Some(new_id), - existing_id, - relation.classification, - "auto_resolved", - Some(strategy), - )), + Some(conflict_record_json(conflict_record_id, Some(new_id), existing_id, relation.classification, "auto_resolved", Some(strategy))), ); Ok((entry, Some(new_id))) } - #[allow(clippy::too_many_arguments)] pub(crate) fn handle_agreement_policy( - conn: &mut Connection, - decision: &str, - context: Option<&str>, - source_agent: &str, - quality: i32, - ts: &str, - relation: &ConflictResult, + conn: &mut Connection, decision: &str, context: Option<&str>, source_agent: &str, quality: i32, ts: &str, relation: &ConflictResult, ) -> Result<(Value, Option), StoreError> { - let target_id = relation - .matched_id - .ok_or_else(|| StoreError::Internal("Missing agreement target id".to_string()))?; - let tx = conn - .transaction() - .map_err(|e| StoreError::Internal(e.to_string()))?; - - let (existing_decision, existing_context, previous_merged_count): ( - String, - Option, - i64, - ) = tx - .query_row( - "SELECT decision, context, COALESCE(merged_count, 0) FROM decisions WHERE id = ?1", - params![target_id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) + let target_id = relation.matched_id.ok_or_else(|| StoreError::Internal("Missing agreement target id".to_string()))?; + let tx = conn.transaction().map_err(|e| StoreError::Internal(e.to_string()))?; + let (existing_decision, existing_context, previous_merged_count): (String, Option, i64) = tx + .query_row("SELECT decision, context, COALESCE(merged_count, 0) FROM decisions WHERE id = ?1", params![target_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) .map_err(|e| StoreError::Internal(e.to_string()))?; - let merged_context = merge_context(existing_context, &existing_decision, context, decision); let merged_count = previous_merged_count + 1; tx.execute( @@ -189,17 +98,9 @@ pub(crate) fn handle_agreement_policy( quality = MAX(COALESCE(quality, 50), ?4), \ updated_at = ?5 \ WHERE id = ?6", - params![ - merged_context, - MERGE_SCORE_BONUS, - merged_count, - quality, - ts, - target_id - ], + params![merged_context, MERGE_SCORE_BONUS, merged_count, quality, ts, target_id], ) .map_err(|e| StoreError::Internal(e.to_string()))?; - let conflict_record_id = insert_conflict_record( &tx, None, @@ -212,88 +113,42 @@ pub(crate) fn handle_agreement_policy( Some("policy_engine"), ts, )?; - let _ = log_event( &tx, "decision_agreement_merge", - json!({ - "targetId": target_id, - "source_agent": source_agent, - "similarity_jaccard": relation.similarity_jaccard, - "conflict_record_id": conflict_record_id, - }), + json!({"targetId":target_id,"source_agent":source_agent,"similarity_jaccard":relation. +similarity_jaccard,"conflict_record_id":conflict_record_id,}), "rust-daemon", ); - - tx.commit() - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.commit().map_err(|e| StoreError::Internal(e.to_string()))?; checkpoint_wal_best_effort(conn); - - let mut entry = json!({ - "action": "merged", - "target_id": target_id, - "merged_count": merged_count, - "quality": quality, - }); + let mut entry = json!({"action":"merged","target_id":target_id,"merged_count": +merged_count,"quality":quality,}); decorate_entry_with_relation( &mut entry, relation, - Some(conflict_record_json( - conflict_record_id, - None, - target_id, - relation.classification, - "auto_resolved", - Some("deduplicated_merge"), - )), + Some(conflict_record_json(conflict_record_id, None, target_id, relation.classification, "auto_resolved", Some("deduplicated_merge"))), ); Ok((entry, None)) } - #[allow(clippy::too_many_arguments)] pub(crate) fn handle_refinement_policy( - conn: &mut Connection, - decision: &str, - context: Option<&str>, - entry_type: &str, - source_agent: &str, - provenance: &DecisionProvenance, - confidence: f64, - trust_score: f64, - quality: i32, - retention_class: RetentionClass, - expires_at: Option<&str>, - ts: &str, - owner_id: Option, - relation: &ConflictResult, + conn: &mut Connection, decision: &str, context: Option<&str>, entry_type: &str, source_agent: &str, provenance: &DecisionProvenance, confidence: f64, + trust_score: f64, quality: i32, retention_class: RetentionClass, expires_at: Option<&str>, ts: &str, owner_id: Option, relation: &ConflictResult, ) -> Result<(Value, Option), StoreError> { - let target_id = relation - .matched_id - .ok_or_else(|| StoreError::Internal("Missing refinement target id".to_string()))?; + let target_id = relation.matched_id.ok_or_else(|| StoreError::Internal("Missing refinement target id".to_string()))?; let target_trust = relation.matched_trust_score.unwrap_or(0.8); - let should_supersede = - relation.matched_agent.as_deref() == Some(source_agent) || trust_score >= target_trust; - - let tx = conn - .transaction() - .map_err(|e| StoreError::Internal(e.to_string()))?; - + let should_supersede = relation.matched_agent.as_deref() == Some(source_agent) || trust_score >= target_trust; + let tx = conn.transaction().map_err(|e| StoreError::Internal(e.to_string()))?; if should_supersede { if let Some(owner_id) = owner_id { - tx.execute( - "UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2 AND owner_id = ?3", - params![ts, target_id, owner_id], - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.execute("UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2 AND owner_id = ?3", params![ts, target_id, owner_id]) + .map_err(|e| StoreError::Internal(e.to_string()))?; } else { - tx.execute( - "UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2", - params![ts, target_id], - ) - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.execute("UPDATE decisions SET status = 'superseded', updated_at = ?1 WHERE id = ?2", params![ts, target_id]) + .map_err(|e| StoreError::Internal(e.to_string()))?; } } - let new_id = insert_decision_with_state( &tx, decision, @@ -308,35 +163,13 @@ pub(crate) fn handle_refinement_policy( expires_at, ts, owner_id, - if should_supersede { - "active" - } else { - "disputed" - }, - if should_supersede { - None - } else { - Some(target_id) - }, - if should_supersede { - Some(target_id) - } else { - None - }, + if should_supersede { "active" } else { "disputed" }, + if should_supersede { None } else { Some(target_id) }, + if should_supersede { Some(target_id) } else { None }, Some((1.0 - relation.similarity_jaccard).clamp(0.0, 1.0)), )?; - - let conflict_status = if should_supersede { - "auto_resolved" - } else { - "open" - }; - let strategy = if should_supersede { - Some("refine_supersede") - } else { - Some("requires_user_review") - }; - + let conflict_status = if should_supersede { "auto_resolved" } else { "open" }; + let strategy = if should_supersede { Some("refine_supersede") } else { Some("requires_user_review") }; let conflict_record_id = insert_conflict_record( &tx, Some(new_id), @@ -346,43 +179,21 @@ pub(crate) fn handle_refinement_policy( relation.similarity_cosine, conflict_status, strategy, - if should_supersede { - Some("policy_engine") - } else { - None - }, + if should_supersede { Some("policy_engine") } else { None }, ts, )?; - - let event_name = if should_supersede { - "decision_supersede" - } else { - "decision_refine_pending" - }; + let event_name = if should_supersede { "decision_supersede" } else { "decision_refine_pending" }; let _ = log_event( &tx, event_name, - json!({ - "newId": new_id, - "targetId": target_id, - "source_agent": source_agent, - "strategy": strategy, - "conflict_record_id": conflict_record_id, - }), + json!({"newId":new_id,"targetId":target_id,"source_agent":source_agent,"strategy":strategy,"conflict_record_id": +conflict_record_id,}), "rust-daemon", ); - - tx.commit() - .map_err(|e| StoreError::Internal(e.to_string()))?; + tx.commit().map_err(|e| StoreError::Internal(e.to_string()))?; checkpoint_wal_best_effort(conn); - - let mut entry = json!({ - "action": "inserted", - "id": new_id, - "status": if should_supersede { "superseded_old" } else { "disputed" }, - "retention_class": retention_class.as_str(), - "quality": quality, - }); + let mut entry = json!({"action":"inserted","id":new_id,"status":if should_supersede{"superseded_old"}else{"disputed"}, +"retention_class":retention_class.as_str(),"quality":quality,}); if should_supersede { entry["supersedes"] = json!(target_id); } else { @@ -391,15 +202,7 @@ pub(crate) fn handle_refinement_policy( decorate_entry_with_relation( &mut entry, relation, - Some(conflict_record_json( - conflict_record_id, - Some(new_id), - target_id, - relation.classification, - conflict_status, - strategy, - )), + Some(conflict_record_json(conflict_record_id, Some(new_id), target_id, relation.classification, conflict_status, strategy)), ); Ok((entry, Some(new_id))) } - diff --git a/daemon-rs/src/handlers/store/tests.rs b/daemon-rs/src/handlers/store/tests.rs deleted file mode 100644 index c8dd3378..00000000 --- a/daemon-rs/src/handlers/store/tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Store data-integrity boundaries only. - -use super::*; -use rusqlite::{params, Connection}; - -fn test_conn() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - crate::db::configure(&conn).unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - crate::db::run_pending_migrations(&conn); - conn -} - -fn insert_existing_decision(conn: &Connection, decision: &str, context: Option<&str>, vector: &[f32]) -> i64 { - conn.execute( - "INSERT INTO decisions (decision, context, source_agent, status, score, merged_count, quality, created_at, updated_at) - VALUES (?1, ?2, 'tester', 'active', 1.0, 0, 50, datetime('now'), datetime('now'))", - params![decision, context], - ) - .unwrap(); - let id = conn.last_insert_rowid(); - persist_decision_embedding(conn, id, vector, crate::embeddings::selected_model_key()).unwrap(); - id -} - -#[test] -fn benchmark_entries_bypass_semantic_merge() { - let mut conn = test_conn(); - insert_existing_decision( - &conn, - "store benchmark messages without dedup collapsing", - Some("seed"), - &[1.0, 0.0], - ); - - let (_entry, new_id) = store_decision_with_input_embedding( - &mut conn, - "store benchmark messages without dedup collapsing", - Some("bench-doc".to_string()), - Some("benchmark".to_string()), - "tester".to_string(), - None, - None, - Some(&[1.0, 0.0]), - None, - ) - .unwrap(); - - assert!(new_id.is_some()); - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM decisions", [], |row| row.get(0)) - .unwrap(); - assert_eq!(count, 2); -} - -#[test] -fn store_decision_rejects_invalid_explicit_ttl() { - let mut conn = test_conn(); - let err = store_decision_with_ttl( - &mut conn, - "ttl smoke", - Some("ctx".to_string()), - Some("decision".to_string()), - "tester".to_string(), - None, - Some(-1), - None, - ) - .unwrap_err(); - assert!(err.contains("ttl") || err.contains("TTL")); -} diff --git a/daemon-rs/src/handlers/store/tests/mod.rs b/daemon-rs/src/handlers/store/tests/mod.rs new file mode 100644 index 00000000..5328d7ac --- /dev/null +++ b/daemon-rs/src/handlers/store/tests/mod.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MIT +use super::*; +use rusqlite::{params, Connection}; +fn test_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::configure(&conn).unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + crate::db::run_pending_migrations(&conn); + conn +} +fn insert_existing_decision(conn: &Connection, decision: &str, context: Option<&str>, vector: &[f32]) -> i64 { + conn.execute( + "INSERT INTO decisions (decision, context, source_agent, status, score, merged_count, quality, created_at, updated_at) + VALUES (?1, ?2, 'tester', 'active', 1.0, 0, 50, datetime('now'), datetime('now'))", + params![decision, context], + ) + .unwrap(); + let id = conn.last_insert_rowid(); + persist_decision_embedding(conn, id, vector, crate::embeddings::selected_model_key()).unwrap(); + id +} +fn insert_existing_decision_with_model(conn: &Connection, decision: &str, context: Option<&str>, vector: &[f32], model: &str) -> i64 { + conn.execute( + "INSERT INTO decisions (decision, context, source_agent, status, score, merged_count, quality, created_at, updated_at) + VALUES (?1, ?2, 'tester', 'active', 1.0, 0, 50, datetime('now'), datetime('now'))", + params![decision, context], + ) + .unwrap(); + let id = conn.last_insert_rowid(); + let blob = crate::embeddings::vector_to_blob(vector); + conn.execute( + "INSERT OR REPLACE INTO embeddings (target_type, target_id, vector, model) + VALUES ('decision', ?1, ?2, ?3)", + params![id, blob, model], + ) + .unwrap(); + id +} +#[test] +fn benchmark_entries_bypass_semantic_merge() { + let mut conn = test_conn(); + insert_existing_decision(&conn, "store benchmark messages without dedup collapsing", Some("seed"), &[1.0, 0.0]); + let (_entry, new_id) = store_decision_with_input_embedding( + &mut conn, + "store benchmark messages without dedup collapsing", + Some("bench-doc".to_string()), + Some("benchmark".to_string()), + "tester".to_string(), + None, + None, + Some(&[1.0, 0.0]), + None, + ) + .unwrap(); + assert!(new_id.is_some()); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM decisions", [], |row| row.get(0)).unwrap(); + assert_eq!(count, 2); +} +#[test] +fn semantic_candidates_filter_model_and_vector_length() { + let conn = test_conn(); + let selected_id = + insert_existing_decision_with_model(&conn, "selected model embedding candidate", Some("seed"), &[1.0, 0.0], crate::embeddings::selected_model_key()); + insert_existing_decision_with_model(&conn, "wrong model embedding candidate", Some("seed"), &[1.0, 0.0], "other-embedding-model"); + insert_existing_decision_with_model( + &conn, + "wrong vector length embedding candidate", + Some("seed"), + &[1.0, 0.0, 0.0], + crate::embeddings::selected_model_key(), + ); + + let candidates = fetch_top_semantic_candidates(&conn, &[1.0, 0.0], None).unwrap(); + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].id, selected_id); +} +#[test] +fn store_decision_rejects_invalid_explicit_ttl() { + let mut conn = test_conn(); + let err = + store_decision_with_ttl(&mut conn, "ttl smoke", Some("ctx".to_string()), Some("decision".to_string()), "tester".to_string(), None, Some(-1), None) + .unwrap_err(); + assert!(err.contains("ttl") || err.contains("TTL")); +} diff --git a/daemon-rs/src/handlers/store/types.rs b/daemon-rs/src/handlers/store/types.rs index 8bb703d8..67e67a76 100644 --- a/daemon-rs/src/handlers/store/types.rs +++ b/daemon-rs/src/handlers/store/types.rs @@ -1,20 +1,4 @@ -// SPDX-License-Identifier: MIT -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::Json; -use rusqlite::{params, Connection}; use serde_json::{json, Value}; -use crate::handlers::{ensure_auth_with_caller_rated_for_class, ensure_endpoint_budget, json_response, log_event, now_iso, resolve_source_identity, truncate_chars}; -use crate::api_types::{RetentionClass, StoreRequest}; -use crate::budgets::BudgetEndpoint; -use crate::conflict::{detect_conflict, jaccard_similarity, ConflictClassification, ConflictResult}; -use crate::db::checkpoint_wal_best_effort; -use crate::rate_limit::RequestClass; -use crate::state::RuntimeState; - - -use super::*; pub(crate) const HARD_MERGE_THRESHOLD: f32 = 0.92; pub(crate) const REVIEW_MERGE_THRESHOLD: f32 = 0.90; pub(crate) const JACCARD_MERGE_THRESHOLD: f64 = 0.70; @@ -24,134 +8,88 @@ pub(crate) const BENCHMARK_ENTRY_TYPE: &str = "benchmark"; pub(crate) const BENCHMARK_SOURCE_AGENT_PREFIX: &str = "amb-cortex::"; pub(crate) const MAX_DECISION_CHARS: usize = 4096; pub(crate) const MAX_EXPLICIT_TTL_SECONDS: i64 = 365 * 24 * 60 * 60; - pub(crate) fn is_benchmark_entry_type(entry_type: &str) -> bool { entry_type.eq_ignore_ascii_case(BENCHMARK_ENTRY_TYPE) } - pub(crate) fn is_benchmark_source_agent(source_agent: &str) -> bool { - source_agent - .trim() - .to_ascii_lowercase() - .starts_with(BENCHMARK_SOURCE_AGENT_PREFIX) + source_agent.trim().to_ascii_lowercase().starts_with(BENCHMARK_SOURCE_AGENT_PREFIX) } - #[derive(Debug, Clone, PartialEq)] pub(crate) struct DecisionProvenance { pub(crate) source_client: String, pub(crate) source_model: Option, pub(crate) reasoning_depth: String, } - impl DecisionProvenance { - pub(crate) fn from_fields( - source_agent: &str, - source_model: Option<&str>, - reasoning_depth: Option<&str>, - ) -> Self { - let normalized_model = source_model - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); + pub(crate) fn from_fields(source_agent: &str, source_model: Option<&str>, reasoning_depth: Option<&str>) -> Self { + let normalized_model = source_model.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string); Self { source_client: normalize_source_client(source_agent), source_model: normalized_model, reasoning_depth: normalize_reasoning_depth(reasoning_depth), } } - pub(crate) fn trust_score(&self, confidence: f64) -> f64 { compute_trust_score(confidence, self.source_model.as_deref()) } } - #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct QualityFactors { pub(crate) length_score: i32, pub(crate) specificity_bonus: i32, pub(crate) question_penalty: i32, } - impl QualityFactors { pub(crate) fn as_json(&self) -> Value { - json!({ - "length_score": self.length_score, - "specificity_bonus": self.specificity_bonus, - "question_penalty": self.question_penalty, - }) + json!({"length_score":self.length_score, +"specificity_bonus":self.specificity_bonus,"question_penalty":self.question_penalty,}) } } - #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct QualityAssessment { pub(crate) score: i32, pub(crate) factors: QualityFactors, } - #[derive(Debug, Clone)] pub(crate) struct SemanticCandidate { pub(crate) id: i64, pub(crate) decision: String, pub(crate) similarity: f32, } - #[derive(Debug, Clone, PartialEq)] pub(crate) enum SemanticDedupAction { Insert, - Merge { - target_id: i64, - similarity: f32, - jaccard: f64, - }, + Merge { target_id: i64, similarity: f32, jaccard: f64 }, } - #[derive(Debug)] pub(crate) enum StoreError { BadRequest(&'static str), - Validation { - message: &'static str, - quality: i32, - factors: QualityFactors, - }, + Validation { message: &'static str, quality: i32, factors: QualityFactors }, Internal(String), } - impl std::fmt::Display for StoreError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { StoreError::BadRequest(message) => write!(f, "{message}"), - StoreError::Validation { - message, quality, .. - } => write!(f, "{message} (quality {quality})"), + StoreError::Validation { message, quality, .. } => write!(f, "{message} (quality {quality})"), StoreError::Internal(message) => write!(f, "{message}"), } } } - impl From for StoreError { fn from(value: String) -> Self { StoreError::Internal(value) } } - pub(crate) fn normalize_source_client(raw: &str) -> String { - let before_model = raw - .split('(') - .next() - .unwrap_or(raw) - .trim() - .to_ascii_lowercase(); - let normalized: String = before_model - .chars() - .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') - .collect(); + let before_model = raw.split('(').next().unwrap_or(raw).trim().to_ascii_lowercase(); + let normalized: String = before_model.chars().filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_').collect(); if normalized.is_empty() { "unknown".to_string() } else { normalized } } - pub(crate) fn normalize_reasoning_depth(raw: Option<&str>) -> String { let normalized = raw .map(str::trim) @@ -174,15 +112,11 @@ pub(crate) fn normalize_reasoning_depth(raw: Option<&str>) -> String { }) .filter(|value| !value.is_empty()) .unwrap_or_else(|| "single-shot".to_string()); - match normalized.as_str() { - "chain-of-thought" | "single-shot" | "tool-assisted" | "multi-step" | "user-stated" => { - normalized - } + "chain-of-thought" | "single-shot" | "tool-assisted" | "multi-step" | "user-stated" => normalized, _ => "single-shot".to_string(), } } - pub(crate) fn model_weight(source_model: Option<&str>) -> f64 { let Some(model) = source_model.map(|value| value.to_ascii_lowercase()) else { return 0.70; @@ -201,16 +135,12 @@ pub(crate) fn model_weight(source_model: Option<&str>) -> f64 { 0.70 } } - pub(crate) fn compute_trust_score(confidence: f64, source_model: Option<&str>) -> f64 { let bounded_confidence = confidence.clamp(0.0, 1.0); let raw = bounded_confidence * model_weight(source_model); ((raw * 10_000.0).round() / 10_000.0).clamp(0.0, 1.0) } - -pub(crate) fn validate_explicit_ttl_seconds( - ttl_seconds: Option, -) -> Result, StoreError> { +pub(crate) fn validate_explicit_ttl_seconds(ttl_seconds: Option) -> Result, StoreError> { let Some(ttl_seconds) = ttl_seconds else { return Ok(None); }; @@ -218,10 +148,7 @@ pub(crate) fn validate_explicit_ttl_seconds( return Err(StoreError::BadRequest("ttl_seconds must be > 0")); } if ttl_seconds > MAX_EXPLICIT_TTL_SECONDS { - return Err(StoreError::BadRequest( - "ttl_seconds must be <= 31536000 (365 days)", - )); + return Err(StoreError::BadRequest("ttl_seconds must be <= 31536000 (365 days)")); } Ok(Some(ttl_seconds)) } - diff --git a/daemon-rs/src/handlers/tests/mod.rs b/daemon-rs/src/handlers/tests/mod.rs new file mode 100644 index 00000000..9cfc2b4e --- /dev/null +++ b/daemon-rs/src/handlers/tests/mod.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +#[test] +fn parse_duration_to_seconds_bounds_fuzzed_inputs() { + assert_eq!(parse_duration_to_seconds("15m"), 15 * 60); + assert_eq!(parse_duration_to_seconds("2h"), 2 * 60 * 60); + assert_eq!(parse_duration_to_seconds("3d"), 3 * 24 * 60 * 60); + assert_eq!(parse_duration_to_seconds("36500d"), MAX_PARSED_DURATION_SECONDS); + for raw in [ + "", + "m", + "-5h", + "10x", + "36501d", + "9223372036854775807m", + "9223372036854775807h", + "9223372036854775807d", + "999999999999999999999999999999d", + ] { + assert_eq!(parse_duration_to_seconds(raw), DEFAULT_PARSED_DURATION_SECONDS, "duration parser should fall back for fuzzed input {raw:?}",); + } +} +#[test] +fn estimate_tokens_from_chars_matches_estimate_tokens() { + for char_count in [0usize, 1, 3, 4, 38, 379, 10_000] { + let text = "x".repeat(char_count); + assert_eq!(estimate_tokens_from_chars(char_count), estimate_tokens(&text), "char-count estimator should match text estimator for {char_count} chars"); + } +} diff --git a/daemon-rs/src/hook_boot.rs b/daemon-rs/src/hook_boot.rs index adbc4914..96a05cfe 100644 --- a/daemon-rs/src/hook_boot.rs +++ b/daemon-rs/src/hook_boot.rs @@ -1,34 +1,16 @@ -// SPDX-License-Identifier: MIT -//! Hook subcommands -- replaces brain-boot.js and statusline JS. -//! -//! `cortex hook-boot [--agent NAME]` -- Claude Code SessionStart hook -//! `cortex hook-status` -- Statusline one-liner output -//! -//! Both are short-lived CLI invocations that HTTP-call the running daemon. -//! No RuntimeState, no DB, no ONNX -- just a thin HTTP client. - use serde_json::json; use std::path::PathBuf; - const DEFAULT_BUDGET: u32 = 600; - -// ---- Internal types --------------------------------------------------------- - struct BootResult { boot_prompt: String, token_estimate: Option, savings: Option, } - struct HealthResult { memories: i64, decisions: i64, embeddings: i64, } - -// ---- HTTP helpers ----------------------------------------------------------- - -/// Read the auth token from ~/.cortex/cortex.token for authenticated requests. fn read_auth_token() -> Option { let path = crate::auth::CortexPaths::resolve().token; match std::fs::read_to_string(path) { @@ -43,44 +25,26 @@ fn read_auth_token() -> Option { Err(_) => None, } } - -async fn fetch_boot( - agent: &str, - budget: u32, - paths: &crate::auth::CortexPaths, -) -> Option { +async fn fetch_boot(agent: &str, budget: u32, paths: &crate::auth::CortexPaths) -> Option { let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(3)) .timeout(std::time::Duration::from_secs(7)) .build() .ok()?; - let base_url = crate::transport::local_http_base_url(paths); let mut url = reqwest::Url::parse(&format!("{}/boot", base_url.trim_end_matches('/'))).ok()?; - url.query_pairs_mut() - .append_pair("agent", agent) - .append_pair("budget", &budget.to_string()); - + url.query_pairs_mut().append_pair("agent", agent).append_pair("budget", &budget.to_string()); let mut headers = vec![("x-cortex-request".to_string(), "true".to_string())]; if let Some(token) = read_auth_token() { headers.push(("authorization".to_string(), format!("Bearer {token}"))); } - - let (status, body) = crate::transport::request_url_with_local_ipc_fallback( - &client, - "GET", - url.as_ref(), - paths, - &headers, - None, - std::time::Duration::from_secs(7), - ) - .await - .ok()?; + let (status, body) = + crate::transport::request_url_with_local_ipc_fallback(&client, "GET", url.as_ref(), paths, &headers, None, std::time::Duration::from_secs(7)) + .await + .ok()?; if !status.is_success() { return None; } - let data: serde_json::Value = serde_json::from_str(&body).ok()?; Some(BootResult { boot_prompt: data.get("bootPrompt")?.as_str()?.to_string(), @@ -88,49 +52,27 @@ async fn fetch_boot( savings: data.get("savings").cloned(), }) } - async fn fetch_health(paths: &crate::auth::CortexPaths) -> Option { let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(3)) .timeout(std::time::Duration::from_secs(2)) .build() .ok()?; - let base_url = crate::transport::local_http_base_url(paths); let readiness_url = format!("{base_url}/readiness"); - let (readiness_status, readiness_body) = crate::transport::request_url_with_local_ipc_fallback( - &client, - "GET", - &readiness_url, - paths, - &[], - None, - std::time::Duration::from_secs(2), - ) - .await - .ok()?; - match crate::daemon_lifecycle::readiness_state_from_payload( - readiness_status.as_u16(), - &readiness_body, - Some(paths.port), - None, - ) { + let (readiness_status, readiness_body) = + crate::transport::request_url_with_local_ipc_fallback(&client, "GET", &readiness_url, paths, &[], None, std::time::Duration::from_secs(2)) + .await + .ok()?; + match crate::daemon_lifecycle::readiness_state_from_payload(readiness_status.as_u16(), &readiness_body, Some(paths.port), None) { Some(true) => {} Some(false) | None => return None, } - let health_url = format!("{base_url}/health"); - let (status, body) = crate::transport::request_url_with_local_ipc_fallback( - &client, - "GET", - &health_url, - paths, - &[], - None, - std::time::Duration::from_secs(2), - ) - .await - .ok()?; + let (status, body) = + crate::transport::request_url_with_local_ipc_fallback(&client, "GET", &health_url, paths, &[], None, std::time::Duration::from_secs(2)) + .await + .ok()?; if !status.is_success() { return None; } @@ -139,21 +81,12 @@ async fn fetch_health(paths: &crate::auth::CortexPaths) -> Option Some(HealthResult { memories: stats.get("memories")?.as_i64()?, decisions: stats.get("decisions")?.as_i64()?, - embeddings: stats - .get("embeddings") - .and_then(|v| v.as_i64()) - .unwrap_or(0), + embeddings: stats.get("embeddings").and_then(|v| v.as_i64()).unwrap_or(0), }) } - fn status_path() -> PathBuf { - dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".claude") - .join("brain-status.json") + dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")).join(".claude").join("brain-status.json") } - -/// Register an active session with the daemon so the Agents panel shows it. async fn register_session(agent: &str, paths: &crate::auth::CortexPaths) { let client = match reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(2)) @@ -163,18 +96,9 @@ async fn register_session(agent: &str, paths: &crate::auth::CortexPaths) { Ok(c) => c, Err(_) => return, }; - let base_url = crate::transport::local_http_base_url(paths); - let body = json!({ - "agent": agent, - "ttl": 7200, - "description": "Active coding session" - }) - .to_string(); - let mut headers = vec![ - ("content-type".to_string(), "application/json".to_string()), - ("x-cortex-request".to_string(), "true".to_string()), - ]; + let body = json!({"agent":agent,"ttl":7200,"description":"Active coding session"}).to_string(); + let mut headers = vec![("content-type".to_string(), "application/json".to_string()), ("x-cortex-request".to_string(), "true".to_string())]; if let Some(token) = read_auth_token() { headers.push(("authorization".to_string(), format!("Bearer {token}"))); } @@ -190,123 +114,57 @@ async fn register_session(agent: &str, paths: &crate::auth::CortexPaths) { ) .await; } - -// ---- Public entry points ---------------------------------------------------- - -/// SessionStart hook -- outputs JSON for Claude Code hook system. pub async fn run_boot(agent: &str) { let paths = crate::auth::CortexPaths::resolve(); - let (boot, health) = tokio::join!( - fetch_boot(agent, DEFAULT_BUDGET, &paths), - fetch_health(&paths) - ); - - let (total, memories, decisions) = health - .as_ref() - .map(|h| (h.memories + h.decisions, h.memories, h.decisions)) - .unwrap_or((0, 0, 0)); - + let (boot, health) = tokio::join!(fetch_boot(agent, DEFAULT_BUDGET, &paths), fetch_health(&paths)); + let (total, memories, decisions) = health.as_ref().map(|h| (h.memories + h.decisions, h.memories, h.decisions)).unwrap_or((0, 0, 0)); let cortex_connected = boot.is_some() || health.is_some(); let cortex_booted = boot.is_some(); - - // Register session so the Agents panel shows this agent as online if cortex_booted { register_session(agent, &paths).await; } - let overall = if cortex_connected { - "ONLINE" - } else { - "DEGRADED" - }; - + let overall = if cortex_connected { "ONLINE" } else { "DEGRADED" }; let cortex_part = if cortex_booted { - format!( - "Cortex \u{2713} ({} nodes: {} mem + {} dec)", - total, memories, decisions - ) + format!("Cortex \u{2713} ({} nodes: {} mem + {} dec)", total, memories, decisions) } else if cortex_connected { "Cortex \u{2713} (boot failed)".to_string() } else { "Cortex \u{2717} \u{2014} run cortex-start or cortex service start".to_string() }; - let oneliner = format!("Brain: {} | {}", overall, cortex_part); - - // Write brain-status.json for statusline - let status = json!({ - "timestamp": chrono::Utc::now().to_rfc3339(), - "overall": overall, - "cortex": { - "connected": cortex_connected, - "booted": cortex_booted, - "total": total, - "memories": memories, - "decisions": decisions, - "tokenEstimate": boot.as_ref().and_then(|b| b.token_estimate), - "savings": boot.as_ref().and_then(|b| b.savings.clone()), - }, - "oneliner": oneliner, - }); - - let _ = std::fs::write( - status_path(), - serde_json::to_string_pretty(&status).unwrap_or_default(), - ); - - // Build additionalContext for Claude Code + let status = json!({"timestamp":chrono::Utc::now().to_rfc3339(),"overall":overall,"cortex":{"connected": +cortex_connected,"booted":cortex_booted,"total":total,"memories":memories,"decisions":decisions,"tokenEstimate":boot.as_ref(). +and_then(|b|b.token_estimate),"savings":boot.as_ref().and_then(|b|b.savings.clone()),},"oneliner":oneliner,}); + let _ = std::fs::write(status_path(), serde_json::to_string_pretty(&status).unwrap_or_default()); let mut parts = vec![ "BRAIN BOOT STATUS \u{2014} You MUST include this line at the very start of your first reply:".to_string(), oneliner.clone(), "Do not ask the user about brain status. Do not explain this line. Just print it.".to_string(), ]; - if let Some(b) = &boot { parts.push(String::new()); parts.push("CORTEX BOOT PROMPT (auto-injected by SessionStart hook):".to_string()); parts.push(b.boot_prompt.clone()); parts.push(String::new()); - parts.push( - "cortex_boot() was already called mechanically by the hook. Do NOT call it again." - .to_string(), - ); - parts.push( - "You still use cortex_recall, cortex_store, and cortex_diary as normal MCP tools." - .to_string(), - ); + parts.push("cortex_boot() was already called mechanically by the hook. Do NOT call it again.".to_string()); + parts.push("You still use cortex_recall, cortex_store, and cortex_diary as normal MCP tools.".to_string()); } else if cortex_connected { parts.push(String::new()); - parts.push( - "WARNING: Cortex is running but boot failed. Call cortex_boot() manually as fallback." - .to_string(), - ); + parts.push("WARNING: Cortex is running but boot failed. Call cortex_boot() manually as fallback.".to_string()); } else { parts.push(String::new()); - parts.push( - "WARNING: Cortex daemon is not running. Advise user to run cortex-start.".to_string(), - ); + parts.push("WARNING: Cortex daemon is not running. Advise user to run cortex-start.".to_string()); parts.push("cortex_boot() will fail until the daemon is started.".to_string()); } - - // Output hook JSON to stdout - let output = json!({ - "hookSpecificOutput": { - "hookEventName": "SessionStart", - "additionalContext": parts.join("\n"), - } - }); - + let output = json!({"hookSpecificOutput":{"hookEventName": +"SessionStart","additionalContext":parts.join("\n"),}}); println!("{}", serde_json::to_string(&output).unwrap_or_default()); } - -/// Statusline output -- prints a one-liner to stdout. pub async fn run_status() { let paths = crate::auth::CortexPaths::resolve(); match fetch_health(&paths).await { Some(h) => { - println!( - "ONLINE | {} mem | {} dec | {} emb", - h.memories, h.decisions, h.embeddings - ); + println!("ONLINE | {} mem | {} dec | {} emb", h.memories, h.decisions, h.embeddings); } None => { println!("OFFLINE"); diff --git a/daemon-rs/src/indexer.rs b/daemon-rs/src/indexer.rs deleted file mode 100644 index bf6a9963..00000000 --- a/daemon-rs/src/indexer.rs +++ /dev/null @@ -1,603 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Knowledge indexer: reads filesystem sources and upserts into memories table. -//! -//! **Core indexers** (always run): `~/.claude/state.md` and Claude Code project memory under -//! `~/.claude/projects//memory`. -//! -//! **Custom sources** (opt-in): user-defined paths via `~/.cortex/sources.toml` or -//! `CORTEX_EXTRA_SOURCES` env var. See `config/sources.toml.example`. - -use rusqlite::Connection; -use serde::Deserialize; -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::workspace::claude_project_slug; - -const STATE_SECTIONS: &[&str] = &[ - "## What Was Done", - "## Next Session", - "## Pending", - "## Known Issues", -]; - -/// Run core indexers always; custom sources from config if present. -pub fn index_all(conn: &Connection, home: &Path, owner_id: Option) -> usize { - let mut total = 0; - total += index_state_file(conn, home, owner_id); - total += index_memory_files(conn, home, owner_id); - total += index_custom_sources(conn, home, owner_id); - total -} - -/// Upsert a memory by source. If source exists, update text. Otherwise insert. -fn upsert_memory( - conn: &Connection, - text: &str, - source: &str, - mem_type: &str, - agent: &str, - owner_id: Option, -) -> bool { - let text = text.trim(); - if text.is_empty() { - return false; - } - - let existing: Option = conn - .query_row( - "SELECT id FROM memories WHERE source = ? AND status = 'active'", - [source], - |row| row.get(0), - ) - .ok(); - - if let Some(id) = existing { - let _ = conn.execute( - "UPDATE memories SET text = ?, updated_at = datetime('now') WHERE id = ?", - rusqlite::params![text, id], - ); - let _ = conn.execute( - "DELETE FROM embeddings WHERE target_type = 'memory' AND target_id = ?", - [id], - ); - } else if let Some(oid) = owner_id { - let _ = conn.execute( - "INSERT INTO memories (text, source, type, source_agent, owner_id) VALUES (?, ?, ?, ?, ?)", - rusqlite::params![text, source, mem_type, agent, oid], - ); - } else { - let _ = conn.execute( - "INSERT INTO memories (text, source, type, source_agent) VALUES (?, ?, ?, ?)", - rusqlite::params![text, source, mem_type, agent], - ); - } - - true -} - -// ── Source 1: state.md ────────────────────────────────────────────────────── - -fn index_state_file(conn: &Connection, home: &Path, owner_id: Option) -> usize { - let state_path = home.join(".claude").join("state.md"); - if !state_path.exists() { - return 0; - } - - let content = match fs::read_to_string(&state_path) { - Ok(c) => c, - Err(_) => return 0, - }; - - let mut count = 0; - for section in STATE_SECTIONS { - if let Some(text) = extract_section(&content, section) { - let source = format!("state.md::{}", section.trim_start_matches("## ")); - if upsert_memory(conn, &text, &source, "state", "indexer", owner_id) { - count += 1; - } - } - } - count -} - -fn extract_section(markdown: &str, header: &str) -> Option { - let idx = markdown.find(header)?; - let start = idx + header.len(); - let rest = &markdown[start..]; - let end = rest.find("\n## ").unwrap_or(rest.len()); - let text = rest[..end].trim(); - if text.is_empty() { - None - } else { - Some(text.to_string()) - } -} - -// ── Source 2: Memory files ────────────────────────────────────────────────── - -fn index_memory_files(conn: &Connection, home: &Path, owner_id: Option) -> usize { - let slug = match claude_project_slug() { - Some(s) => s, - None => return 0, - }; - let mem_dir = home - .join(".claude") - .join("projects") - .join(slug) - .join("memory"); - if !mem_dir.exists() { - return 0; - } - - let mut count = 0; - let entries = match fs::read_dir(&mem_dir) { - Ok(e) => e, - Err(_) => return 0, - }; - - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("md") { - continue; - } - if path.file_name().and_then(|f| f.to_str()) == Some("MEMORY.md") { - continue; - } - - if let Ok(raw) = fs::read_to_string(&path) { - let (fm, body) = parse_frontmatter(&raw); - let name = fm.get("name").cloned().unwrap_or_else(|| { - path.file_stem() - .unwrap_or_default() - .to_string_lossy() - .to_string() - }); - let mem_type = fm - .get("type") - .cloned() - .unwrap_or_else(|| "memory".to_string()); - let desc = fm.get("description").cloned().unwrap_or_default(); - - let body_preview: String = body.chars().take(500).collect(); - let text = if !desc.is_empty() { - format!("[{name}] ({mem_type}) {desc}\n{body_preview}") - } else { - format!("[{name}] ({mem_type})\n{body_preview}") - }; - - let source = format!( - "memory::{}", - path.file_name().unwrap_or_default().to_string_lossy() - ); - if upsert_memory(conn, &text, &source, &mem_type, "indexer", owner_id) { - count += 1; - } - } - } - count -} - -fn parse_frontmatter(raw: &str) -> (HashMap, String) { - let mut fm = HashMap::new(); - let body; - - if let Some(rest) = raw.strip_prefix("---") { - if let Some(end) = rest.find("---") { - let yaml_block = &rest[..end]; - body = rest[end + 3..].trim().to_string(); - - for line in yaml_block.lines() { - if let Some(colon) = line.find(':') { - let key = line[..colon].trim().to_string(); - let val = line[colon + 1..].trim().to_string(); - fm.insert(key, val); - } - } - } else { - body = raw.to_string(); - } - } else { - body = raw.to_string(); - } - - (fm, body) -} - -// ── Custom sources from config ───────────────────────────────────────────── - -#[derive(Debug, Deserialize)] -struct SourcesConfig { - #[serde(default)] - source: Vec, -} - -#[derive(Debug, Deserialize)] -struct CustomSource { - name: String, - path: String, - #[serde(default = "default_mem_type")] - mem_type: String, - #[serde(default = "default_glob")] - glob: String, - #[serde(default)] - truncate: usize, - #[serde(default)] - recursive: bool, -} - -fn default_mem_type() -> String { - "custom".to_string() -} -fn default_glob() -> String { - "*.md".to_string() -} - -/// Resolve `~` to the user's home directory. -fn expand_tilde(p: &str) -> PathBuf { - if let Some(rest) = p.strip_prefix("~/") { - if let Some(home) = dirs::home_dir() { - return home.join(rest); - } - } - PathBuf::from(p) -} - -/// Load custom source definitions from `~/.cortex/sources.toml`, falling back -/// to `CORTEX_EXTRA_SOURCES` env var (semicolon-separated directory paths). -fn load_custom_sources(home: &Path) -> Vec { - // Try sources.toml first - let config_path = home.join(".cortex").join("sources.toml"); - if config_path.exists() { - if let Ok(content) = fs::read_to_string(&config_path) { - if let Ok(cfg) = toml::from_str::(&content) { - return cfg.source; - } - eprintln!("[indexer] failed to parse {}", config_path.display()); - } - } - - // Fallback: CORTEX_EXTRA_SOURCES env var (semicolon-separated paths) - if let Ok(val) = std::env::var("CORTEX_EXTRA_SOURCES") { - return val - .split(';') - .filter(|s| !s.is_empty()) - .map(|p| CustomSource { - name: Path::new(p) - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string(), - path: p.to_string(), - mem_type: "custom".to_string(), - glob: "*".to_string(), - truncate: 0, - recursive: false, - }) - .collect(); - } - - Vec::new() -} - -/// Index all user-configured custom sources. -fn index_custom_sources(conn: &Connection, home: &Path, owner_id: Option) -> usize { - let sources = load_custom_sources(home); - let mut total = 0; - let home_root = home.canonicalize().ok(); - - for src in &sources { - let resolved = expand_tilde(&src.path); - if !resolved.exists() { - continue; - } - if let Some(root) = home_root.as_ref() { - let Ok(canonical) = resolved.canonicalize() else { - continue; - }; - if !canonical.starts_with(root) { - eprintln!( - "[cortex] skipping custom source outside Cortex home: {}", - resolved.display() - ); - continue; - } - } - - if resolved.is_dir() { - total += index_directory(conn, &resolved, src, owner_id); - } else if resolved.is_file() { - total += index_single_file(conn, &resolved, src, owner_id); - } - } - total -} - -/// Index all matching files in a directory. -fn index_directory( - conn: &Connection, - dir: &Path, - src: &CustomSource, - owner_id: Option, -) -> usize { - let entries = match fs::read_dir(dir) { - Ok(e) => e, - Err(_) => return 0, - }; - - let mut count = 0; - for entry in entries.flatten() { - let path = entry.path(); - - if path.is_dir() { - if src.recursive { - count += index_directory(conn, &path, src, owner_id); - } - continue; - } - - if !matches_glob(&path, &src.glob) { - continue; - } - - count += index_single_file(conn, &path, src, owner_id); - } - count -} - -/// Index a single file's content as a memory entry. -fn index_single_file( - conn: &Connection, - path: &Path, - src: &CustomSource, - owner_id: Option, -) -> usize { - let content = match fs::read_to_string(path) { - Ok(c) => c, - Err(_) => return 0, - }; - - let file_stem = path - .file_stem() - .unwrap_or_default() - .to_string_lossy() - .to_string(); - - let text = if src.truncate > 0 { - content.chars().take(src.truncate).collect() - } else { - content - }; - - let source = format!("{}::{}", src.name, file_stem); - if upsert_memory(conn, &text, &source, &src.mem_type, "indexer", owner_id) { - 1 - } else { - 0 - } -} - -/// Simple glob matching: supports `*` (any filename) and `*.ext` patterns. -fn matches_glob(path: &Path, pattern: &str) -> bool { - if pattern == "*" { - return true; - } - let name = match path.file_name().and_then(|n| n.to_str()) { - Some(n) => n, - None => return false, - }; - if let Some(ext_pattern) = pattern.strip_prefix("*.") { - return name.ends_with(&format!(".{ext_pattern}")); - } - name == pattern -} - -/// Collect resolved paths from custom sources config. -#[allow(dead_code)] -pub fn custom_source_paths(home: &Path) -> Vec { - load_custom_sources(home) - .iter() - .map(|s| expand_tilde(&s.path)) - .filter(|p| p.exists()) - .collect() -} - -// ── Ebbinghaus decay ──────────────────────────────────────────────────────── - -/// Apply Ebbinghaus-style forgetting curve to all active entries. -/// -/// Formula: score = MAX(floor, score * POWER(decay_rate, days_since_last_touch)) -/// -/// Key improvements over simple 0.95^days: -/// - Uses last_accessed (recall time) not just updated_at (write time) -/// - Retrieval count strengthens durability: decay_rate = 0.95 + 0.005 * min(retrievals, 10) -/// → 0 recalls: 0.950/day (forgets fast) -/// → 5 recalls: 0.975/day (moderate retention) -/// → 10+ recalls: 1.000/day (permanent -- fully reinforced) -/// - Pinned entries are immune to decay -/// - Floor is 0.05 (not 0.1) to better separate stale from semi-stale -/// -/// Also decays decisions table with same formula. -pub fn decay_pass(conn: &Connection) -> usize { - let mem_result = conn.execute( - "UPDATE memories SET score = MAX(0.05, score * POWER( - MIN(1.0, 0.95 + 0.005 * MIN(retrievals, 10)), - CAST((julianday('now') - julianday( - COALESCE(last_accessed, updated_at, created_at) - )) AS REAL) - )) - WHERE status = 'active' AND score > 0.05 AND pinned = 0 - AND (julianday('now') - julianday( - COALESCE(last_accessed, updated_at, created_at) - )) > 1", - [], - ); - - let dec_result = conn.execute( - "UPDATE decisions SET score = MAX(0.05, score * POWER( - MIN(1.0, 0.95 + 0.005 * MIN(retrievals, 10)), - CAST((julianday('now') - julianday( - COALESCE(last_accessed, updated_at, created_at) - )) AS REAL) - )) - WHERE status = 'active' AND score > 0.05 AND pinned = 0 - AND (julianday('now') - julianday( - COALESCE(last_accessed, updated_at, created_at) - )) > 1", - [], - ); - - mem_result.unwrap_or(0) + dec_result.unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use rusqlite::Connection; - - #[test] - fn index_all_empty_home_indexes_nothing() { - let tmp = std::env::temp_dir().join(format!("cortex_ix_{}", std::process::id())); - let _ = std::fs::remove_dir_all(&tmp); - std::fs::create_dir_all(&tmp).unwrap(); - let conn = Connection::open_in_memory().unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - let n = index_all(&conn, tmp.as_path(), None); - assert_eq!(n, 0); - let _ = std::fs::remove_dir_all(&tmp); - } - - #[test] - fn matches_glob_works() { - assert!(super::matches_glob(Path::new("foo.md"), "*.md")); - assert!(!super::matches_glob(Path::new("foo.rs"), "*.md")); - assert!(super::matches_glob(Path::new("anything"), "*")); - assert!(super::matches_glob(Path::new("data.jsonl"), "*.jsonl")); - } - - #[test] - fn expand_tilde_resolves_home() { - let p = super::expand_tilde("~/test/path"); - assert!(p.components().count() > 2); - assert!(!p.to_string_lossy().contains('~')); - } - - #[test] - fn index_custom_sources_from_toml() { - let tmp = std::env::temp_dir().join(format!("cortex_cs_{}", std::process::id())); - let _ = std::fs::remove_dir_all(&tmp); - - // Set up: ~/.cortex/sources.toml pointing to a test directory - let cortex_dir = tmp.join(".cortex"); - std::fs::create_dir_all(&cortex_dir).unwrap(); - - let notes_dir = tmp.join("test-notes"); - std::fs::create_dir_all(¬es_dir).unwrap(); - std::fs::write(notes_dir.join("alpha.md"), "Alpha note content").unwrap(); - std::fs::write(notes_dir.join("beta.md"), "Beta note content").unwrap(); - std::fs::write(notes_dir.join("ignore.txt"), "Should be skipped").unwrap(); - - let single_file = tmp.join("single.json"); - std::fs::write(&single_file, r#"{"key": "value"}"#).unwrap(); - - let toml_content = format!( - r#" -[[source]] -name = "notes" -path = "{}" -mem_type = "note" -glob = "*.md" - -[[source]] -name = "config" -path = "{}" -mem_type = "config" -"#, - notes_dir.to_string_lossy().replace('\\', "/"), - single_file.to_string_lossy().replace('\\', "/"), - ); - std::fs::write(cortex_dir.join("sources.toml"), &toml_content).unwrap(); - - let conn = Connection::open_in_memory().unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - - let n = super::index_custom_sources(&conn, &tmp, None); - // 2 .md files + 1 single json = 3 - assert_eq!(n, 3, "expected 3 indexed entries (2 md + 1 json)"); - - // Verify sources are stored correctly - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE source LIKE 'notes::%'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(count, 2, "expected 2 note memories"); - - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE source LIKE 'config::%'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(count, 1, "expected 1 config memory"); - - // Verify mem_type is set correctly - let mem_type: String = conn - .query_row( - "SELECT type FROM memories WHERE source LIKE 'notes::%' LIMIT 1", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(mem_type, "note"); - - let _ = std::fs::remove_dir_all(&tmp); - } - - #[test] - fn index_custom_sources_truncate() { - let tmp = std::env::temp_dir().join(format!("cortex_tr_{}", std::process::id())); - let _ = std::fs::remove_dir_all(&tmp); - - let cortex_dir = tmp.join(".cortex"); - std::fs::create_dir_all(&cortex_dir).unwrap(); - - let docs_dir = tmp.join("docs"); - std::fs::create_dir_all(&docs_dir).unwrap(); - std::fs::write(docs_dir.join("long.md"), "A".repeat(5000)).unwrap(); - - let toml_content = format!( - r#" -[[source]] -name = "docs" -path = "{}" -mem_type = "doc" -glob = "*.md" -truncate = 100 -"#, - docs_dir.to_string_lossy().replace('\\', "/"), - ); - std::fs::write(cortex_dir.join("sources.toml"), &toml_content).unwrap(); - - let conn = Connection::open_in_memory().unwrap(); - crate::db::initialize_schema(&conn).unwrap(); - - let n = super::index_custom_sources(&conn, &tmp, None); - assert_eq!(n, 1); - - let text: String = conn - .query_row( - "SELECT text FROM memories WHERE source = 'docs::long'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(text.len(), 100, "text should be truncated to 100 chars"); - - let _ = std::fs::remove_dir_all(&tmp); - } -} diff --git a/daemon-rs/src/indexer/mod.rs b/daemon-rs/src/indexer/mod.rs new file mode 100644 index 00000000..616bc5d6 --- /dev/null +++ b/daemon-rs/src/indexer/mod.rs @@ -0,0 +1,288 @@ +use crate::workspace::claude_project_slug; +use rusqlite::Connection; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +const STATE_SECTIONS: &[&str] = &["## What Was Done", "## Next Session", "## Pending", "## Known Issues"]; +pub fn index_all(conn: &Connection, home: &Path, owner_id: Option) -> usize { + let mut total = 0; + total += index_state_file(conn, home, owner_id); + total += index_memory_files(conn, home, owner_id); + total += index_custom_sources(conn, home, owner_id); + total +} +fn upsert_memory(conn: &Connection, text: &str, source: &str, mem_type: &str, agent: &str, owner_id: Option) -> bool { + let text = text.trim(); + if text.is_empty() { + return false; + } + let existing: Option = conn.query_row("SELECT id FROM memories WHERE source = ? AND status = 'active'", [source], |row| row.get(0)).ok(); + if let Some(id) = existing { + let _ = conn.execute("UPDATE memories SET text = ?, updated_at = datetime('now') WHERE id = ?", rusqlite::params![text, id]); + let _ = conn.execute("DELETE FROM embeddings WHERE target_type = 'memory' AND target_id = ?", [id]); + } else if let Some(oid) = owner_id { + let _ = conn.execute( + "INSERT INTO memories (text, source, type, source_agent, owner_id) VALUES (?, ?, ?, ?, ?)", + rusqlite::params![text, source, mem_type, agent, oid], + ); + } else { + let _ = conn.execute("INSERT INTO memories (text, source, type, source_agent) VALUES (?, ?, ?, ?)", rusqlite::params![text, source, mem_type, agent]); + } + true +} +fn index_state_file(conn: &Connection, home: &Path, owner_id: Option) -> usize { + let state_path = home.join(".claude").join("state.md"); + if !state_path.exists() { + return 0; + } + let content = match fs::read_to_string(&state_path) { + Ok(c) => c, + Err(_) => return 0, + }; + let mut count = 0; + for section in STATE_SECTIONS { + if let Some(text) = extract_section(&content, section) { + let source = format!("state.md::{}", section.trim_start_matches("## ")); + if upsert_memory(conn, &text, &source, "state", "indexer", owner_id) { + count += 1; + } + } + } + count +} +fn extract_section(markdown: &str, header: &str) -> Option { + let idx = markdown.find(header)?; + let start = idx + header.len(); + let rest = &markdown[start..]; + let end = rest.find("\n## ").unwrap_or(rest.len()); + let text = rest[..end].trim(); + if text.is_empty() { + None + } else { + Some(text.to_string()) + } +} +fn index_memory_files(conn: &Connection, home: &Path, owner_id: Option) -> usize { + let slug = match claude_project_slug() { + Some(s) => s, + None => return 0, + }; + let mem_dir = home.join(".claude").join("projects").join(slug).join("memory"); + if !mem_dir.exists() { + return 0; + } + let mut count = 0; + let entries = match fs::read_dir(&mem_dir) { + Ok(e) => e, + Err(_) => return 0, + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("md") { + continue; + } + if path.file_name().and_then(|f| f.to_str()) == Some("MEMORY.md") { + continue; + } + if let Ok(raw) = fs::read_to_string(&path) { + let (fm, body) = parse_frontmatter(&raw); + let name = fm.get("name").cloned().unwrap_or_else(|| path.file_stem().unwrap_or_default().to_string_lossy().to_string()); + let mem_type = fm.get("type").cloned().unwrap_or_else(|| "memory".to_string()); + let desc = fm.get("description").cloned().unwrap_or_default(); + let body_preview: String = body.chars().take(500).collect(); + let text = + if !desc.is_empty() { format!("[{name}] ({mem_type}) {desc}\n{body_preview}") } else { format!("[{name}] ({mem_type})\n{body_preview}") }; + let source = format!("memory::{}", path.file_name().unwrap_or_default().to_string_lossy()); + if upsert_memory(conn, &text, &source, &mem_type, "indexer", owner_id) { + count += 1; + } + } + } + count +} +fn parse_frontmatter(raw: &str) -> (HashMap, String) { + let mut fm = HashMap::new(); + let body; + if let Some(rest) = raw.strip_prefix("---") { + if let Some(end) = rest.find("---") { + let yaml_block = &rest[..end]; + body = rest[end + 3..].trim().to_string(); + for line in yaml_block.lines() { + if let Some(colon) = line.find(':') { + let key = line[..colon].trim().to_string(); + let val = line[colon + 1..].trim().to_string(); + fm.insert(key, val); + } + } + } else { + body = raw.to_string(); + } + } else { + body = raw.to_string(); + } + (fm, body) +} +#[derive(Debug, Deserialize)] +struct SourcesConfig { + #[serde(default)] + source: Vec, +} +#[derive(Debug, Deserialize)] +struct CustomSource { + name: String, + path: String, + #[serde(default = "default_mem_type")] + mem_type: String, + #[serde(default = "default_glob")] + glob: String, + #[serde(default)] + truncate: usize, + #[serde(default)] + recursive: bool, +} +fn default_mem_type() -> String { + "custom".to_string() +} +fn default_glob() -> String { + "*.md".to_string() +} +fn expand_tilde(p: &str) -> PathBuf { + if let Some(rest) = p.strip_prefix("~/") { + if let Some(home) = dirs::home_dir() { + return home.join(rest); + } + } + PathBuf::from(p) +} +fn load_custom_sources(home: &Path) -> Vec { + let config_path = home.join(".cortex").join("sources.toml"); + if config_path.exists() { + if let Ok(content) = fs::read_to_string(&config_path) { + if let Ok(cfg) = toml::from_str::(&content) { + return cfg.source; + } + eprintln!("[indexer] failed to parse {}", config_path.display()); + } + } + if let Ok(val) = std::env::var("CORTEX_EXTRA_SOURCES") { + return val + .split(';') + .filter(|s| !s.is_empty()) + .map(|p| CustomSource { + name: Path::new(p).file_name().unwrap_or_default().to_string_lossy().to_string(), + path: p.to_string(), + mem_type: "custom".to_string(), + glob: "*".to_string(), + truncate: 0, + recursive: false, + }) + .collect(); + } + Vec::new() +} +fn index_custom_sources(conn: &Connection, home: &Path, owner_id: Option) -> usize { + let sources = load_custom_sources(home); + let mut total = 0; + let home_root = home.canonicalize().ok(); + for src in &sources { + let resolved = expand_tilde(&src.path); + if !resolved.exists() { + continue; + } + if let Some(root) = home_root.as_ref() { + let Ok(canonical) = resolved.canonicalize() else { + continue; + }; + if !canonical.starts_with(root) { + eprintln!("[cortex] skipping custom source outside Cortex home: {}", resolved.display()); + continue; + } + } + if resolved.is_dir() { + total += index_directory(conn, &resolved, src, owner_id); + } else if resolved.is_file() { + total += index_single_file(conn, &resolved, src, owner_id); + } + } + total +} +fn index_directory(conn: &Connection, dir: &Path, src: &CustomSource, owner_id: Option) -> usize { + let entries = match fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return 0, + }; + let mut count = 0; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if src.recursive { + count += index_directory(conn, &path, src, owner_id); + } + continue; + } + if !matches_glob(&path, &src.glob) { + continue; + } + count += index_single_file(conn, &path, src, owner_id); + } + count +} +fn index_single_file(conn: &Connection, path: &Path, src: &CustomSource, owner_id: Option) -> usize { + let content = match fs::read_to_string(path) { + Ok(c) => c, + Err(_) => return 0, + }; + let file_stem = path.file_stem().unwrap_or_default().to_string_lossy().to_string(); + let text = if src.truncate > 0 { content.chars().take(src.truncate).collect() } else { content }; + let source = format!("{}::{}", src.name, file_stem); + if upsert_memory(conn, &text, &source, &src.mem_type, "indexer", owner_id) { + 1 + } else { + 0 + } +} +fn matches_glob(path: &Path, pattern: &str) -> bool { + if pattern == "*" { + return true; + } + let name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n, + None => return false, + }; + if let Some(ext_pattern) = pattern.strip_prefix("*.") { + return name.ends_with(&format!(".{ext_pattern}")); + } + name == pattern +} +pub fn decay_pass(conn: &Connection) -> usize { + let mem_result = conn.execute( + "UPDATE memories SET score = MAX(0.05, score * POWER( + MIN(1.0, 0.95 + 0.005 * MIN(retrievals, 10)), + CAST((julianday('now') - julianday( + COALESCE(last_accessed, updated_at, created_at) + )) AS REAL) + )) + WHERE status = 'active' AND score > 0.05 AND pinned = 0 + AND (julianday('now') - julianday( + COALESCE(last_accessed, updated_at, created_at) + )) > 1", + [], + ); + let dec_result = conn.execute( + "UPDATE decisions SET score = MAX(0.05, score * POWER( + MIN(1.0, 0.95 + 0.005 * MIN(retrievals, 10)), + CAST((julianday('now') - julianday( + COALESCE(last_accessed, updated_at, created_at) + )) AS REAL) + )) + WHERE status = 'active' AND score > 0.05 AND pinned = 0 + AND (julianday('now') - julianday( + COALESCE(last_accessed, updated_at, created_at) + )) > 1", + [], + ); + mem_result.unwrap_or(0) + dec_result.unwrap_or(0) +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/indexer/tests/mod.rs b/daemon-rs/src/indexer/tests/mod.rs new file mode 100644 index 00000000..80c1b7fe --- /dev/null +++ b/daemon-rs/src/indexer/tests/mod.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use rusqlite::Connection; +#[test] +fn index_all_empty_home_indexes_nothing() { + let tmp = std::env::temp_dir().join(format!("cortex_ix_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&tmp); + std::fs::create_dir_all(&tmp).unwrap(); + let conn = Connection::open_in_memory().unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + let n = index_all(&conn, tmp.as_path(), None); + assert_eq!(n, 0); + let _ = std::fs::remove_dir_all(&tmp); +} +#[test] +fn matches_glob_works() { + assert!(super::matches_glob(Path::new("foo.md"), "*.md")); + assert!(!super::matches_glob(Path::new("foo.rs"), "*.md")); + assert!(super::matches_glob(Path::new("anything"), "*")); + assert!(super::matches_glob(Path::new("data.jsonl"), "*.jsonl")); +} +#[test] +fn expand_tilde_resolves_home() { + let p = super::expand_tilde("~/test/path"); + assert!(p.components().count() > 2); + assert!(!p.to_string_lossy().contains('~')); +} +#[test] +fn index_custom_sources_from_toml() { + let tmp = std::env::temp_dir().join(format!("cortex_cs_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&tmp); + let cortex_dir = tmp.join(".cortex"); + std::fs::create_dir_all(&cortex_dir).unwrap(); + let notes_dir = tmp.join("test-notes"); + std::fs::create_dir_all(¬es_dir).unwrap(); + std::fs::write(notes_dir.join("alpha.md"), "Alpha note content").unwrap(); + std::fs::write(notes_dir.join("beta.md"), "Beta note content").unwrap(); + std::fs::write(notes_dir.join("ignore.txt"), "Should be skipped").unwrap(); + let single_file = tmp.join("single.json"); + std::fs::write(&single_file, r#"{"key": "value"}"#).unwrap(); + let toml_content = format!( + r#" +[[source]] +name = "notes" +path = "{}" +mem_type = "note" +glob = "*.md" +[[source]] +name = "config" +path = "{}" +mem_type = "config" +"#, + notes_dir.to_string_lossy().replace('\\', "/"), + single_file.to_string_lossy().replace('\\', "/"), + ); + std::fs::write(cortex_dir.join("sources.toml"), &toml_content).unwrap(); + let conn = Connection::open_in_memory().unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + let n = super::index_custom_sources(&conn, &tmp, None); + assert_eq!(n, 3, "expected 3 indexed entries (2 md + 1 json)"); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM memories WHERE source LIKE 'notes::%'", [], |r| r.get(0)).unwrap(); + assert_eq!(count, 2, "expected 2 note memories"); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM memories WHERE source LIKE 'config::%'", [], |r| r.get(0)).unwrap(); + assert_eq!(count, 1, "expected 1 config memory"); + let mem_type: String = conn.query_row("SELECT type FROM memories WHERE source LIKE 'notes::%' LIMIT 1", [], |r| r.get(0)).unwrap(); + assert_eq!(mem_type, "note"); + let _ = std::fs::remove_dir_all(&tmp); +} +#[test] +fn index_custom_sources_truncate() { + let tmp = std::env::temp_dir().join(format!("cortex_tr_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&tmp); + let cortex_dir = tmp.join(".cortex"); + std::fs::create_dir_all(&cortex_dir).unwrap(); + let docs_dir = tmp.join("docs"); + std::fs::create_dir_all(&docs_dir).unwrap(); + std::fs::write(docs_dir.join("long.md"), "A".repeat(5000)).unwrap(); + let toml_content = format!( + r#" +[[source]] +name = "docs" +path = "{}" +mem_type = "doc" +glob = "*.md" +truncate = 100 +"#, + docs_dir.to_string_lossy().replace('\\', "/"), + ); + std::fs::write(cortex_dir.join("sources.toml"), &toml_content).unwrap(); + let conn = Connection::open_in_memory().unwrap(); + crate::db::initialize_schema(&conn).unwrap(); + let n = super::index_custom_sources(&conn, &tmp, None); + assert_eq!(n, 1); + let text: String = conn.query_row("SELECT text FROM memories WHERE source = 'docs::long'", [], |r| r.get(0)).unwrap(); + assert_eq!(text.len(), 100, "text should be truncated to 100 chars"); + let _ = std::fs::remove_dir_all(&tmp); +} diff --git a/daemon-rs/src/main.rs b/daemon-rs/src/main.rs index e130ff02..27e01b04 100644 --- a/daemon-rs/src/main.rs +++ b/daemon-rs/src/main.rs @@ -1,16 +1,9 @@ -// SPDX-License-Identifier: MIT - -/// Default TCP port the Cortex daemon binds to when no `--port` flag or -/// `CORTEX_PORT` env var is set. pub const DEFAULT_CORTEX_PORT: u16 = 7437; - -mod admin; mod aging; mod api_types; mod auth; mod budgets; mod cli; -mod co_occurrence; mod compaction; mod compiler; mod conflict; @@ -39,25 +32,17 @@ mod test_support; mod tls; mod transport; mod workspace; - use chrono::Utc; -use std::io::Write as _; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; - +pub(crate) use cli::run_daemon; use cli::{ - apply_path_env, cli_capabilities_payload, cli_capabilities_summary, cli_robot_docs_guide, - cli_service_usage, ensure_daemon, ensure_remote_target_has_api_key, - is_disallowed_startup_binary_path, parse_flag_usize, parse_flag_value, print_usage_and_exit, - resolve_client_target, run_admin_cli, run_backup_cli, run_boot_cli, run_cleanup_cli, - run_doctor_cli, run_embeddings_cli, run_embeddings_drain_cli, run_eval_cli, run_export_cli, - run_import_cli, run_recrystallize_cli, run_reindex_cli, run_restore_cli, run_status_cli, - run_sync_cli, run_team_cli, run_user_cli, unknown_cli_command_message, - unknown_robot_docs_subcommand_message, validate_cli_options_or_exit, + apply_path_env, cli_capabilities_payload, cli_capabilities_summary, cli_robot_docs_guide, cli_service_usage, ensure_daemon, + ensure_remote_target_has_api_key, is_disallowed_startup_binary_path, parse_flag_usize, parse_flag_value, print_usage_and_exit, resolve_client_target, + run_admin_cli, run_backup_cli, run_boot_cli, run_cleanup_cli, run_doctor_cli, run_embeddings_cli, run_embeddings_drain_cli, run_eval_cli, run_export_cli, + run_import_cli, run_recrystallize_cli, run_reindex_cli, run_restore_cli, run_status_cli, run_sync_cli, run_team_cli, run_user_cli, + unknown_cli_command_message, unknown_robot_docs_subcommand_message, validate_cli_options_or_exit, }; - -pub(crate) use cli::run_daemon; - +use std::io::Write as _; +use std::sync::atomic::{AtomicBool, Ordering}; pub(crate) fn install_daemon_panic_hook(paths: &auth::CortexPaths) { static INSTALLED: AtomicBool = AtomicBool::new(false); if INSTALLED.swap(true, Ordering::SeqCst) { @@ -79,25 +64,17 @@ pub(crate) fn install_daemon_panic_hook(paths: &auth::CortexPaths) { .map(|loc| format!("{}:{}:{}", loc.file(), loc.line(), loc.column())) .unwrap_or_else(|| "".to_string()); let backtrace = std::backtrace::Backtrace::force_capture(); - let entry = format!( - "[{ts}] PANIC at {location}: {message}\n{backtrace}\n", - ts = Utc::now().to_rfc3339(), - ); + let entry = format!("[{ts}] PANIC at {location}: {message}\n{backtrace}\n", ts = Utc::now().to_rfc3339(),); eprintln!("[cortex] {entry}"); if let Some(parent) = panic_log_path.parent() { let _ = std::fs::create_dir_all(parent); } - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&panic_log_path) - { + if let Ok(mut file) = std::fs::OpenOptions::new().create(true).append(true).open(&panic_log_path) { let _ = file.write_all(entry.as_bytes()); } previous(info); })); } - #[tokio::main] async fn main() { let args: Vec = std::env::args().collect(); @@ -105,14 +82,10 @@ async fn main() { let paths = auth::CortexPaths::resolve_from_args(&args); if let Ok(current_exe) = std::env::current_exe() { if is_disallowed_startup_binary_path(¤t_exe) { - eprintln!( - "[cortex] Refusing to run from disallowed runtime path: {}", - current_exe.display() - ); + eprintln!("[cortex] Refusing to run from disallowed runtime path: {}", current_exe.display()); std::process::exit(1); } } - match mode { "" | "--help" | "-h" | "help" => print_usage_and_exit(0), "--version" | "-V" | "version" => println!("cortex {}", env!("CARGO_PKG_VERSION")), @@ -161,11 +134,10 @@ async fn main() { std::future::pending::<()>().await; } run_daemon(paths.clone(), async { - tokio::select! { - _ = tokio::signal::ctrl_c() => eprintln!("[cortex] Received Ctrl+C, shutting down..."), - _ = sigterm_future() => eprintln!("[cortex] Received SIGTERM, shutting down..."), - } - }).await; + tokio::select! {_=tokio::signal::ctrl_c()=>eprintln!( + "[cortex] Received Ctrl+C, shutting down..."),_=sigterm_future()=>eprintln!("[cortex] Received SIGTERM, shutting down..."),} + }) + .await; } "mcp" => { let remaining = &args[2..]; @@ -260,12 +232,22 @@ async fn main() { "stop" => u8::from(service::stop()), "status" => u8::from(service::status()), "ensure" => u8::from(service::ensure()), - _ => { eprintln!("{}", cli_service_usage()); 1 } - }).await { + _ => { + eprintln!("{}", cli_service_usage()); + 1 + } + }) + .await + { Ok(code) => code, - Err(err) => { eprintln!("[cortex] Service command task failed: {err}"); 1 } + Err(err) => { + eprintln!("[cortex] Service command task failed: {err}"); + 1 + } }; - if code != 0 { std::process::exit(code as i32); } + if code != 0 { + std::process::exit(code as i32); + } } "service-run" => service::dispatch_service(), "prompt-inject" => prompt_inject::run(&args[2..]).await, @@ -313,7 +295,10 @@ async fn main() { let max_event_passes = match parse_flag_usize(&args[2..], "--max-passes") { Ok(Some(value)) => value.clamp(1, 12), Ok(None) => 3, - Err(err) => { eprintln!("Error: {err}"); std::process::exit(1); } + Err(err) => { + eprintln!("Error: {err}"); + std::process::exit(1); + } }; run_cleanup_cli(&paths, args.iter().any(|a| a == "--dry-run"), args.iter().any(|a| a == "--events"), max_event_passes); } diff --git a/daemon-rs/src/mcp_proxy/mod.rs b/daemon-rs/src/mcp_proxy/mod.rs index 8d0b167e..0032668a 100644 --- a/daemon-rs/src/mcp_proxy/mod.rs +++ b/daemon-rs/src/mcp_proxy/mod.rs @@ -1,14 +1,7 @@ -// SPDX-License-Identifier: MIT -mod session; mod run; - +mod session; #[cfg(test)] #[cfg(test)] -mod tests { - // MCP proxy internals are not release-gated; see Info/testing-philosophy.md. -} - -pub(crate) use session::*; -pub(crate) use run::*; - +mod tests; pub use run::run; +pub(crate) use session::*; diff --git a/daemon-rs/src/mcp_proxy/run.rs b/daemon-rs/src/mcp_proxy/run.rs index 4a11887d..da917007 100644 --- a/daemon-rs/src/mcp_proxy/run.rs +++ b/daemon-rs/src/mcp_proxy/run.rs @@ -1,74 +1,41 @@ -// SPDX-License-Identifier: MIT -use serde_json::Value; -use std::path::{Path, PathBuf}; -use std::sync::{Mutex, OnceLock}; -use sysinfo::{ProcessesToUpdate, System}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; - -use crate::auth::CortexPaths; -use crate::daemon_lifecycle; - - use super::*; -/// Run MCP proxy over stdio -> HTTP. -pub async fn run( - base_url: &str, - api_key: Option<&str>, - agent: Option<&str>, -) -> Result<(), Box> { +use crate::auth::CortexPaths; +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +pub async fn run(base_url: &str, api_key: Option<&str>, agent: Option<&str>) -> Result<(), Box> { let api_key = normalize_api_key(api_key); let base_url = base_url.trim_end_matches('/'); validate_target_base_url(base_url)?; if requires_explicit_api_key(base_url, api_key) { - return Err(format!( - "Remote Cortex target '{base_url}' requires an API key. Pass --api-key or set CORTEX_API_KEY." - ) - .into()); + return Err(format!("Remote Cortex target '{base_url}' requires an API key. Pass --api-key or set CORTEX_API_KEY.").into()); } let mut rpc_base_url = base_url.to_string(); let mut health_url = format!("{rpc_base_url}/readiness"); let (rpc_base_tx, mut rpc_base_rx) = tokio::sync::watch::channel(rpc_base_url.clone()); let (agent_display, agent_model) = resolve_agent_identity(agent); - let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(10)) .connect_timeout(std::time::Duration::from_secs(3)) .build()?; - let team_mode = detect_team_mode(api_key); if team_mode { eprintln!("[cortex-mcp] Team mode proxy -> {base_url} as '{agent_display}'"); } else { eprintln!("[cortex-mcp] Solo mode proxy -> {base_url} as '{agent_display}'"); } - - // Health check with retry (daemon may still be starting) let mut healthy = false; let health_probe_headers = internal_health_probe_headers(); for attempt in 1..=HEALTH_CHECK_ATTEMPTS { - match transport_request_for_url( - &client, - "GET", - &health_url, - &health_probe_headers, - None, - std::time::Duration::from_secs(5), - ) - .await - { + match transport_request_for_url(&client, "GET", &health_url, &health_probe_headers, None, std::time::Duration::from_secs(5)).await { Ok((status, body)) if is_cortex_health_response(status, &body, &health_url) => { healthy = true; break; } Ok((status, _)) => { - eprintln!( - "[cortex-mcp] Health check attempt {attempt}/{HEALTH_CHECK_ATTEMPTS}: HTTP {status} was not a valid Cortex health payload" - ); + eprintln!("[cortex-mcp] Health check attempt {attempt}/{HEALTH_CHECK_ATTEMPTS}: HTTP {status} was not a valid Cortex health payload"); } Err(e) => { - eprintln!( - "[cortex-mcp] Health check attempt {attempt}/{HEALTH_CHECK_ATTEMPTS}: {e}" - ); + eprintln!("[cortex-mcp] Health check attempt {attempt}/{HEALTH_CHECK_ATTEMPTS}: {e}"); } } if attempt < HEALTH_CHECK_ATTEMPTS { @@ -76,44 +43,18 @@ pub async fn run( } } if !healthy { - eprintln!( - "[cortex-mcp] Health check failed after {HEALTH_CHECK_ATTEMPTS} attempts; keeping proxy alive and deferring errors to JSON-RPC responses" - ); + eprintln!("[cortex-mcp] Health check failed after {HEALTH_CHECK_ATTEMPTS} attempts; keeping proxy alive and deferring errors to JSON-RPC responses"); } - - let mut allow_local_token_fallback = - !local_token_fallback_required(&rpc_base_url, api_key) || healthy; + let mut allow_local_token_fallback = !local_token_fallback_required(&rpc_base_url, api_key) || healthy; if local_token_fallback_required(&rpc_base_url, api_key) && !allow_local_token_fallback { - eprintln!( - "[cortex-mcp] Local target is not identity-verified yet; withholding local token auth until health is valid" - ); + eprintln!("[cortex-mcp] Local target is not identity-verified yet; withholding local token auth until health is valid"); } else if healthy { let paths = CortexPaths::resolve(); - drain_write_buffer( - &client, - &rpc_base_url, - api_key, - &agent_display, - agent_model.as_deref(), - &paths, - allow_local_token_fallback, - ) - .await; + drain_write_buffer(&client, &rpc_base_url, api_key, &agent_display, agent_model.as_deref(), &paths, allow_local_token_fallback).await; } - if allow_local_token_fallback || !local_token_fallback_required(&rpc_base_url, api_key) { - let _ = session_start_with_retry( - &client, - &rpc_base_url, - api_key, - &agent_display, - agent_model.as_deref(), - allow_local_token_fallback, - ) - .await; + let _ = session_start_with_retry(&client, &rpc_base_url, api_key, &agent_display, agent_model.as_deref(), allow_local_token_fallback).await; } - - // Spawn background heartbeat to keep sessions visible and recover after daemon restarts. { let heartbeat_base_url = rpc_base_url.clone(); let heartbeat_base_tx = rpc_base_tx.clone(); @@ -122,10 +63,7 @@ pub async fn run( let heartbeat_api_key = api_key.map(String::from); let mut heartbeat_allow_local_token_fallback = allow_local_token_fallback; tokio::spawn(async move { - let hb_client = match reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - .build() - { + let hb_client = match reqwest::Client::builder().timeout(std::time::Duration::from_secs(5)).build() { Ok(client) => client, Err(e) => { eprintln!("[cortex-mcp] Heartbeat client init failed: {e}"); @@ -137,7 +75,6 @@ pub async fn run( let resolved_local_base = local_daemon_base_from_paths(&CortexPaths::resolve()); let heartbeat_can_refresh_local = heartbeat_base_url == resolved_local_base; let mut consecutive_heartbeat_failures = 0u32; - loop { tokio::time::sleep(std::time::Duration::from_secs(SESSION_HEARTBEAT_SECS)).await; match session_heartbeat( @@ -165,10 +102,7 @@ pub async fn run( ) .await; if !restarted - && local_token_fallback_required( - &heartbeat_base_url, - heartbeat_api_key.as_deref(), - ) + && local_token_fallback_required(&heartbeat_base_url, heartbeat_api_key.as_deref()) && !heartbeat_allow_local_token_fallback && health_check_ready(&hb_client, &heartbeat_health_url).await { @@ -184,17 +118,12 @@ pub async fn run( .await; } if !restarted && heartbeat_can_refresh_local { - let refreshed_base = - local_daemon_base_from_paths(&CortexPaths::resolve()); + let refreshed_base = local_daemon_base_from_paths(&CortexPaths::resolve()); if refreshed_base != heartbeat_base_url { heartbeat_base_url = refreshed_base; heartbeat_health_url = format!("{heartbeat_base_url}/readiness"); let _ = heartbeat_base_tx.send(heartbeat_base_url.clone()); - heartbeat_allow_local_token_fallback = - !local_token_fallback_required( - &heartbeat_base_url, - heartbeat_api_key.as_deref(), - ); + heartbeat_allow_local_token_fallback = !local_token_fallback_required(&heartbeat_base_url, heartbeat_api_key.as_deref()); restarted = session_start_with_retry( &hb_client, &heartbeat_base_url, @@ -215,16 +144,13 @@ pub async fn run( if consecutive_heartbeat_failures < HEARTBEAT_RECOVERY_FAILURES { continue; } - consecutive_heartbeat_failures = 0; if !health_check_ready(&hb_client, &heartbeat_health_url).await { if heartbeat_can_refresh_local { - let refreshed_base = - local_daemon_base_from_paths(&CortexPaths::resolve()); + let refreshed_base = local_daemon_base_from_paths(&CortexPaths::resolve()); if refreshed_base != heartbeat_base_url { heartbeat_base_url = refreshed_base; - heartbeat_health_url = - format!("{heartbeat_base_url}/readiness"); + heartbeat_health_url = format!("{heartbeat_base_url}/readiness"); let _ = heartbeat_base_tx.send(heartbeat_base_url.clone()); } } @@ -233,7 +159,6 @@ pub async fn run( } } heartbeat_allow_local_token_fallback = true; - let restarted = session_start_with_retry( &hb_client, &heartbeat_base_url, @@ -244,21 +169,17 @@ pub async fn run( ) .await; if restarted { - eprintln!( - "[cortex-mcp] Recovered heartbeat session for {heartbeat_agent}" - ); + eprintln!("[cortex-mcp] Recovered heartbeat session for {heartbeat_agent}"); } } } } }); } - let stdin = tokio::io::stdin(); let reader = BufReader::new(stdin); let mut stdout = tokio::io::stdout(); - let (stdin_tx, mut stdin_rx) = - tokio::sync::mpsc::channel::, String>>(STDIN_CHANNEL_CAPACITY); + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::, String>>(STDIN_CHANNEL_CAPACITY); tokio::spawn(async move { let mut lines = reader.lines(); loop { @@ -273,120 +194,55 @@ pub async fn run( } } }); - let mut consecutive_failures: u32 = 0; let startup_timeout = startup_idle_timeout(); let parent_process = current_parent_process(); let mut saw_client_message = false; let mut orphan_check = tokio::time::interval(std::time::Duration::from_secs(ORPHAN_CHECK_SECS)); orphan_check.tick().await; - loop { let line = if !saw_client_message { let startup_sleep = tokio::time::sleep(startup_timeout); tokio::pin!(startup_sleep); - tokio::select! { - _ = orphan_check.tick() => { - if let Some(parent_process) = parent_process { - if !process_is_alive(parent_process) { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Proxy session ended (parent process exited before handshake)"); - return Ok(()); - } - } - continue; - } - _ = &mut startup_sleep => { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!( - "[cortex-mcp] Proxy session ended (no client handshake within {}s)", - startup_timeout.as_secs() - ); - return Ok(()); - } - result = stdin_rx.recv() => { - match result { - Some(Ok(Some(line))) => line, - Some(Ok(None)) | None => { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Proxy session ended (stdin closed)"); - return Ok(()); - } - Some(Err(e)) => { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Stdin read error: {e}"); - return Err(std::io::Error::other(e).into()); - } - } - } - } + tokio::select! {_=orphan_check.tick()=>{if let + Some(parent_process)=parent_process{if!process_is_alive(parent_process){finalize_proxy_session(&client,&rpc_base_url,api_key,& + agent_display,allow_local_token_fallback).await;eprintln!( + "[cortex-mcp] Proxy session ended (parent process exited before handshake)");return Ok(());}}continue;}_=&mut startup_sleep=>{ + finalize_proxy_session(&client,&rpc_base_url,api_key,&agent_display,allow_local_token_fallback).await;eprintln!( + "[cortex-mcp] Proxy session ended (no client handshake within {}s)",startup_timeout.as_secs());return Ok(());}result=stdin_rx.recv + ()=>{match result{Some(Ok(Some(line)))=>line,Some(Ok(None))|None=>{finalize_proxy_session(&client,&rpc_base_url,api_key,& + agent_display,allow_local_token_fallback).await;eprintln!("[cortex-mcp] Proxy session ended (stdin closed)");return Ok(());}Some( + Err(e))=>{finalize_proxy_session(&client,&rpc_base_url,api_key,&agent_display,allow_local_token_fallback).await;eprintln!( + "[cortex-mcp] Stdin read error: {e}");return Err(std::io::Error::other(e).into());}}}} } else { - tokio::select! { - _ = orphan_check.tick() => { - if let Some(parent_process) = parent_process { - if !process_is_alive(parent_process) { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Proxy session ended (parent process exited)"); - return Ok(()); - } - } - continue; - } - result = stdin_rx.recv() => { - match result { - Some(Ok(Some(line))) => line, - Some(Ok(None)) | None => { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Proxy session ended (stdin closed)"); - return Ok(()); - } - Some(Err(e)) => { - finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback) - .await; - eprintln!("[cortex-mcp] Stdin read error: {e}"); - return Err(std::io::Error::other(e).into()); - } - } - } - } + tokio::select! {_=orphan_check.tick()=> + {if let Some(parent_process)=parent_process{if!process_is_alive(parent_process){finalize_proxy_session(&client,&rpc_base_url, + api_key,&agent_display,allow_local_token_fallback).await;eprintln!("[cortex-mcp] Proxy session ended (parent process exited)"); + return Ok(());}}continue;}result=stdin_rx.recv()=>{match result{Some(Ok(Some(line)))=>line,Some(Ok(None))|None=>{ + finalize_proxy_session(&client,&rpc_base_url,api_key,&agent_display,allow_local_token_fallback).await;eprintln!( + "[cortex-mcp] Proxy session ended (stdin closed)");return Ok(());}Some(Err(e))=>{finalize_proxy_session(&client,&rpc_base_url, + api_key,&agent_display,allow_local_token_fallback).await;eprintln!("[cortex-mcp] Stdin read error: {e}");return Err(std::io::Error + ::other(e).into());}}}} }; let trimmed = line.trim(); if trimmed.is_empty() { continue; } saw_client_message = true; - let msg: Value = match serde_json::from_str(trimmed) { Ok(v) => v, Err(e) => { eprintln!("[cortex-mcp] Parse error: {e}"); - let err = serde_json::json!({ - "jsonrpc": "2.0", - "error": { "code": -32700, "message": "Parse error" }, - "id": null - }); + let err = serde_json::json!({"jsonrpc": +"2.0","error":{"code":-32700,"message":"Parse error"},"id":null}); if !write_value(&mut stdout, &err).await? { - finalize_proxy_session( - &client, - &rpc_base_url, - api_key, - &agent_display, - allow_local_token_fallback, - ) - .await; + finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback).await; eprintln!("[cortex-mcp] Stdout closed while returning parse error"); return Ok(()); } continue; } }; - let has_id = msg.get("id").is_some(); if rpc_base_rx.has_changed().unwrap_or(false) { let refreshed_base = rpc_base_rx.borrow_and_update().clone(); @@ -396,14 +252,11 @@ pub async fn run( allow_local_token_fallback = !local_token_fallback_required(&rpc_base_url, api_key); } } - - // Retry loop for daemon requests let mut last_err = String::new(); let mut response_body: Option = None; let mut should_count_failure = false; let request_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(15); let mut attempted_auth_recovery = false; - for attempt in 1..=REQUEST_ATTEMPTS { let now = tokio::time::Instant::now(); let remaining = request_deadline.saturating_duration_since(now); @@ -412,7 +265,6 @@ pub async fn run( should_count_failure = true; break; } - let mut headers = vec![ ("content-type".to_string(), "application/json".to_string()), ("x-cortex-request".to_string(), "true".to_string()), @@ -421,7 +273,6 @@ pub async fn run( if let Some(model) = agent_model.as_deref() { headers.push(("x-source-model".to_string(), model.to_string())); } - match transport_request( &client, "POST", @@ -442,7 +293,6 @@ pub async fn run( } else { format!("daemon returned auth HTTP {status}: {}", body.trim()) }; - if attempt < REQUEST_ATTEMPTS { invalidate_auth_token_cache(); if !attempted_auth_recovery { @@ -457,27 +307,20 @@ pub async fn run( ) .await; if recovered { - eprintln!( - "[cortex-mcp] Auth rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); refreshed token and retrying" - ); + eprintln!("[cortex-mcp] Auth rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); refreshed token and retrying"); } else { eprintln!( - "[cortex-mcp] Auth rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); daemon looks live but auth recovery is still settling" - ); +"[cortex-mcp] Auth rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); daemon looks live but auth recovery is still settling" +); } } else { eprintln!( - "[cortex-mcp] Auth still rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); retrying once more before surfacing the error" - ); +"[cortex-mcp] Auth still rejected request (attempt {attempt}/{REQUEST_ATTEMPTS}); retrying once more before surfacing the error"); } - tokio::time::sleep(std::time::Duration::from_millis( - 150 * attempt as u64, - )) - .await; + tokio::time::sleep(std::time::Duration::from_millis(150 * attempt as u64)).await; continue; } } - if is_retryable_status(status) { last_err = if body.trim().is_empty() { format!("daemon returned transient HTTP {status}") @@ -486,18 +329,12 @@ pub async fn run( }; should_count_failure = true; if attempt < REQUEST_ATTEMPTS { - eprintln!( - "[cortex-mcp] Request failed (attempt {attempt}/{REQUEST_ATTEMPTS}): {last_err}" - ); - tokio::time::sleep(std::time::Duration::from_millis( - 500 * attempt as u64, - )) - .await; + eprintln!("[cortex-mcp] Request failed (attempt {attempt}/{REQUEST_ATTEMPTS}): {last_err}"); + tokio::time::sleep(std::time::Duration::from_millis(500 * attempt as u64)).await; continue; } break; } - if status.is_success() && has_id { let body = body.trim(); if body.is_empty() { @@ -521,24 +358,11 @@ pub async fn run( } } } else if !status.is_success() { - eprintln!( - "[cortex-mcp] Notification request returned HTTP {status}: {}", - body.trim() - ); + eprintln!("[cortex-mcp] Notification request returned HTTP {status}: {}", body.trim()); } - if consecutive_failures > 0 && status.is_success() { let paths = CortexPaths::resolve(); - drain_write_buffer( - &client, - &rpc_base_url, - api_key, - &agent_display, - agent_model.as_deref(), - &paths, - allow_local_token_fallback, - ) - .await; + drain_write_buffer(&client, &rpc_base_url, api_key, &agent_display, agent_model.as_deref(), &paths, allow_local_token_fallback).await; } consecutive_failures = 0; break; @@ -547,91 +371,47 @@ pub async fn run( last_err = e.to_string(); should_count_failure = true; if attempt < REQUEST_ATTEMPTS { - eprintln!( - "[cortex-mcp] Request failed (attempt {attempt}/{REQUEST_ATTEMPTS}): {e}" - ); - tokio::time::sleep(std::time::Duration::from_millis(500 * attempt as u64)) - .await; + eprintln!("[cortex-mcp] Request failed (attempt {attempt}/{REQUEST_ATTEMPTS}): {e}"); + tokio::time::sleep(std::time::Duration::from_millis(500 * attempt as u64)).await; } } } } - if response_body.is_none() && should_count_failure { consecutive_failures += 1; - eprintln!( - "[cortex-mcp] Request exhausted after {REQUEST_ATTEMPTS} attempts: {last_err} (consecutive failures: {consecutive_failures})" - ); + eprintln!("[cortex-mcp] Request exhausted after {REQUEST_ATTEMPTS} attempts: {last_err} (consecutive failures: {consecutive_failures})"); } - if response_body.is_none() && !last_err.is_empty() && has_id { let id = msg.get("id").cloned().unwrap_or(Value::Null); - let err_resp = serde_json::json!({ - "jsonrpc": "2.0", - "error": { "code": -32603, "message": format!("Daemon unavailable: {last_err}") }, - "id": id - }); + let err_resp = serde_json +::json!({"jsonrpc":"2.0","error":{"code":-32603,"message":format!("Daemon unavailable: {last_err}")},"id":id}); if !write_value(&mut stdout, &err_resp).await? { - finalize_proxy_session( - &client, - &rpc_base_url, - api_key, - &agent_display, - allow_local_token_fallback, - ) - .await; + finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback).await; eprintln!("[cortex-mcp] Stdout closed while returning daemon error"); return Ok(()); } } - if let Some(body) = response_body { if !write_raw_line(&mut stdout, &body).await? { - finalize_proxy_session( - &client, - &rpc_base_url, - api_key, - &agent_display, - allow_local_token_fallback, - ) - .await; + finalize_proxy_session(&client, &rpc_base_url, api_key, &agent_display, allow_local_token_fallback).await; eprintln!("[cortex-mcp] Stdout closed while returning daemon response"); return Ok(()); } } } } - fn is_retryable_status(status: reqwest::StatusCode) -> bool { - status == reqwest::StatusCode::REQUEST_TIMEOUT - || status == reqwest::StatusCode::TOO_MANY_REQUESTS - || status.is_server_error() + status == reqwest::StatusCode::REQUEST_TIMEOUT || status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error() } - -async fn write_value( - stdout: &mut tokio::io::Stdout, - value: &Value, -) -> Result { +async fn write_value(stdout: &mut tokio::io::Stdout, value: &Value) -> Result { write_raw_line(stdout, &value.to_string()).await } - -async fn write_raw_line( - stdout: &mut tokio::io::Stdout, - line: &str, -) -> Result { +async fn write_raw_line(stdout: &mut tokio::io::Stdout, line: &str) -> Result { if let Err(e) = stdout.write_all(format!("{line}\n").as_bytes()).await { - return if e.kind() == std::io::ErrorKind::BrokenPipe { - Ok(false) - } else { - Err(e) - }; + return if e.kind() == std::io::ErrorKind::BrokenPipe { Ok(false) } else { Err(e) }; } if let Err(e) = stdout.flush().await { - return if e.kind() == std::io::ErrorKind::BrokenPipe { - Ok(false) - } else { - Err(e) - }; + return if e.kind() == std::io::ErrorKind::BrokenPipe { Ok(false) } else { Err(e) }; } Ok(true) } diff --git a/daemon-rs/src/mcp_proxy/session.rs b/daemon-rs/src/mcp_proxy/session.rs index ba2b3f85..c60a4317 100644 --- a/daemon-rs/src/mcp_proxy/session.rs +++ b/daemon-rs/src/mcp_proxy/session.rs @@ -1,22 +1,13 @@ -// SPDX-License-Identifier: MIT -use serde_json::Value; +use crate::auth::CortexPaths; +use crate::daemon_lifecycle; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use sysinfo::{ProcessesToUpdate, System}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; - -use crate::auth::CortexPaths; -use crate::daemon_lifecycle; - - pub(crate) const HEALTH_CHECK_ATTEMPTS: u32 = 5; pub(crate) const REQUEST_ATTEMPTS: u32 = 3; pub(crate) const SESSION_HEARTBEAT_SECS: u64 = 15; pub(crate) const SESSION_RESTART_ATTEMPTS: u32 = 4; pub(crate) const SESSION_RESTART_DELAY_MS: u64 = 250; -// Tolerate ~75s of transient daemon unreachability before triggering recovery. -// At 2, brief supervisor respawns and DB lock spikes (~5–10s) caused the bridge -// to abandon a still-healthy daemon and surface "MCP server exited" to clients. pub(crate) const HEARTBEAT_RECOVERY_FAILURES: u32 = 5; pub(crate) const STARTUP_IDLE_TIMEOUT_SECS: u64 = 60; pub(crate) const ORPHAN_CHECK_SECS: u64 = 15; @@ -24,40 +15,31 @@ pub(crate) const MAX_AGENT_HEADER_LEN: usize = 160; pub(crate) const MAX_MODEL_HEADER_LEN: usize = 160; pub(crate) const AUTH_TOKEN_CACHE_TTL_MS: u64 = 1_000; pub(crate) const STDIN_CHANNEL_CAPACITY: usize = 32; - #[derive(Default)] pub(crate) struct AuthTokenCacheEntry { token_path: Option, token: Option, read_at: Option, } - static AUTH_TOKEN_CACHE: OnceLock> = OnceLock::new(); - pub(crate) fn auth_token_cache() -> &'static Mutex { AUTH_TOKEN_CACHE.get_or_init(|| Mutex::new(AuthTokenCacheEntry::default())) } - #[cfg(test)] static AUTH_TOKEN_CACHE_TEST_LOCK: OnceLock> = OnceLock::new(); - #[cfg(test)] pub(crate) fn auth_token_cache_test_lock() -> &'static Mutex<()> { AUTH_TOKEN_CACHE_TEST_LOCK.get_or_init(|| Mutex::new(())) } - -/// Read the auth token from ~/.cortex/cortex.token. pub(crate) fn read_auth_token() -> Option { let token_path = crate::auth::CortexPaths::resolve().token; read_auth_token_with_cache(&token_path) } - pub(crate) fn read_auth_token_with_cache(token_path: &Path) -> Option { #[cfg(test)] let _guard = auth_token_cache_test_lock().lock().ok(); read_auth_token_with_cache_inner(token_path) } - pub(crate) fn read_auth_token_with_cache_inner(token_path: &Path) -> Option { let now = std::time::Instant::now(); if let Ok(cache) = auth_token_cache().lock() { @@ -69,7 +51,6 @@ pub(crate) fn read_auth_token_with_cache_inner(token_path: &Path) -> Option Option Option { match std::fs::read_to_string(token_path) { Ok(token) => { let trimmed = token.trim(); if trimmed.is_empty() { - eprintln!( - "[cortex-mcp] Auth token file is empty: {}", - token_path.display() - ); + eprintln!("[cortex-mcp] Auth token file is empty: {}", token_path.display()); None } else { Some(trimmed.to_string()) @@ -95,33 +72,24 @@ pub(crate) fn read_auth_token_uncached(token_path: &Path) -> Option { } Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, Err(e) => { - eprintln!( - "[cortex-mcp] Failed to read auth token {}: {e}", - token_path.display() - ); + eprintln!("[cortex-mcp] Failed to read auth token {}: {e}", token_path.display()); None } } } - pub(crate) fn invalidate_auth_token_cache() { #[cfg(test)] let _guard = auth_token_cache_test_lock().lock().ok(); invalidate_auth_token_cache_inner(); } - pub(crate) fn invalidate_auth_token_cache_inner() { if let Ok(mut cache) = auth_token_cache().lock() { *cache = AuthTokenCacheEntry::default(); } } - -/// Detect team mode without a full DB open. -/// Team mode is explicit from CLI options. pub(crate) fn detect_team_mode(api_key: Option<&str>) -> bool { api_key.is_some() } - pub(crate) fn startup_idle_timeout() -> std::time::Duration { let secs = std::env::var("CORTEX_MCP_HANDSHAKE_TIMEOUT_SECS") .ok() @@ -129,14 +97,9 @@ pub(crate) fn startup_idle_timeout() -> std::time::Duration { .unwrap_or(STARTUP_IDLE_TIMEOUT_SECS); std::time::Duration::from_secs(secs.max(1)) } - pub(crate) fn env_trimmed(key: &str) -> Option { - std::env::var(key) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) + std::env::var(key).ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) } - pub(crate) fn normalize_header_value(raw: &str, max_len: usize) -> Option { let trimmed = raw.trim(); if trimmed.is_empty() || trimmed.len() > max_len { @@ -145,20 +108,14 @@ pub(crate) fn normalize_header_value(raw: &str, max_len: usize) -> Option) -> Option<&str> { api_key.map(str::trim).filter(|value| !value.is_empty()) } - pub(crate) fn detect_agent_hint(value: &str) -> Option<&'static str> { let value = value.trim().to_ascii_lowercase(); if value.is_empty() { @@ -181,87 +138,59 @@ pub(crate) fn detect_agent_hint(value: &str) -> Option<&'static str> { } None } - pub(crate) fn infer_agent_from_process_tree() -> Option { let mut system = System::new_all(); system.refresh_processes(ProcessesToUpdate::All, true); let current_pid = sysinfo::get_current_pid().ok()?; let mut next_pid = Some(current_pid); let mut depth = 0usize; - while let Some(pid) = next_pid { let process = system.process(pid)?; let candidates = [ process.name().to_string_lossy().into_owned(), - process - .exe() - .map(|path| path.to_string_lossy().into_owned()) - .unwrap_or_default(), - process - .cmd() - .iter() - .map(|part| part.to_string_lossy()) - .collect::>() - .join(" "), + process.exe().map(|path| path.to_string_lossy().into_owned()).unwrap_or_default(), + process.cmd().iter().map(|part| part.to_string_lossy()).collect::>().join(" "), ]; - for candidate in candidates { if let Some(agent) = detect_agent_hint(&candidate) { return Some(agent.to_string()); } } - next_pid = process.parent(); depth += 1; if depth >= 6 { break; } } - None } - #[derive(Clone, Copy, Debug)] pub(crate) struct ParentProcessRef { pid: sysinfo::Pid, start_time: u64, } - pub(crate) fn current_parent_process() -> Option { let mut system = System::new_all(); system.refresh_processes(ProcessesToUpdate::All, true); let current_pid = sysinfo::get_current_pid().ok()?; let parent_pid = system.process(current_pid)?.parent()?; let parent = system.process(parent_pid)?; - Some(ParentProcessRef { - pid: parent_pid, - start_time: parent.start_time(), - }) + Some(ParentProcessRef { pid: parent_pid, start_time: parent.start_time() }) } - pub(crate) fn process_is_alive(parent: ParentProcessRef) -> bool { let mut system = System::new_all(); system.refresh_processes(ProcessesToUpdate::Some(&[parent.pid]), true); - system - .process(parent.pid) - .is_some_and(|process| process.start_time() == parent.start_time) + system.process(parent.pid).is_some_and(|process| process.start_time() == parent.start_time) } - pub(crate) fn resolve_agent_identity(agent_arg: Option<&str>) -> (String, Option) { let model = env_trimmed("CORTEX_AGENT_MODEL") .or_else(|| env_trimmed("CORTEX_MODEL")) .and_then(|value| normalize_header_value(&value, MAX_MODEL_HEADER_LEN)); - let mut agent = env_trimmed("CORTEX_AGENT_DISPLAY") - .or_else(|| { - agent_arg - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) - }) + .or_else(|| agent_arg.map(|v| v.trim().to_string()).filter(|v| !v.is_empty())) .or_else(|| env_trimmed("CORTEX_AGENT_NAME")) .or_else(infer_agent_from_process_tree) .unwrap_or_else(|| "mcp".to_string()); - if !agent.contains('(') { if let Some(model_name) = model.as_deref() { if agent.eq_ignore_ascii_case("droid") { @@ -271,221 +200,49 @@ pub(crate) fn resolve_agent_identity(agent_arg: Option<&str>) -> (String, Option } } } - let agent = match normalize_header_value(&agent, MAX_AGENT_HEADER_LEN) { Some(agent) => agent, None => { - eprintln!( - "[cortex-mcp] Invalid source agent label after normalization; falling back to 'mcp'" - ); + eprintln!("[cortex-mcp] Invalid source agent label after normalization; falling back to 'mcp'"); "mcp".to_string() } }; - (agent, model) } - pub(crate) fn local_daemon_base_from_paths(paths: &CortexPaths) -> String { crate::transport::local_http_base_url(paths) } - pub(crate) fn is_local_daemon_base(base_url: &str) -> bool { let paths = CortexPaths::resolve(); crate::transport::is_local_http_base_url(base_url, &paths) } - -pub(crate) fn resolve_local_ipc_endpoint(base_url: &str, api_key: Option<&str>) -> Option { - if api_key.is_some() || !is_local_daemon_base(base_url) { - return None; - } - CortexPaths::resolve().ipc_endpoint -} - -pub(crate) fn split_base_and_path(url: &str) -> Option<(String, String)> { - let parsed = reqwest::Url::parse(url).ok()?; - let mut base = parsed.clone(); - base.set_path(""); - base.set_query(None); - base.set_fragment(None); - let mut path = parsed.path().to_string(); - if path.is_empty() { - path.push('/'); - } - if let Some(query) = parsed.query() { - path.push('?'); - path.push_str(query); - } - Some((base.to_string().trim_end_matches('/').to_string(), path)) -} - -pub(crate) fn parse_http_response(raw: &[u8]) -> Result<(reqwest::StatusCode, String), String> { - crate::transport::parse_http_response_bytes(raw, "IPC endpoint") -} - -pub(crate) async fn send_http_over_stream( - stream: &mut S, - method: &str, - path: &str, - headers: &[(String, String)], - body: Option<&str>, -) -> Result<(reqwest::StatusCode, String), String> -where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, -{ - let body = body.unwrap_or(""); - let mut request = String::new(); - request.push_str(method); - request.push(' '); - request.push_str(path); - request.push_str(" HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n"); - for (name, value) in headers { - request.push_str(name); - request.push_str(": "); - request.push_str(value); - request.push_str("\r\n"); - } - request.push_str("Content-Length: "); - request.push_str(&body.len().to_string()); - request.push_str("\r\n\r\n"); - request.push_str(body); - - stream - .write_all(request.as_bytes()) - .await - .map_err(|e| format!("IPC write failed: {e}"))?; - stream - .flush() - .await - .map_err(|e| format!("IPC flush failed: {e}"))?; - - let mut response = Vec::new(); - stream - .read_to_end(&mut response) - .await - .map_err(|e| format!("IPC read failed: {e}"))?; - parse_http_response(&response) -} - -pub(crate) async fn ipc_http_request( - endpoint: &str, - method: &str, - path: &str, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, -) -> Result<(reqwest::StatusCode, String), String> { - let fut = async { - #[cfg(unix)] - { - let mut stream = tokio::net::UnixStream::connect(endpoint) - .await - .map_err(|e| format!("IPC connect failed: {e}"))?; - return send_http_over_stream(&mut stream, method, path, headers, body).await; - } - #[cfg(windows)] - { - let mut stream = tokio::net::windows::named_pipe::ClientOptions::new() - .open(endpoint) - .map_err(|e| format!("IPC connect failed: {e}"))?; - return send_http_over_stream(&mut stream, method, path, headers, body).await; - } - #[allow(unreachable_code)] - Err("IPC transport is unsupported on this platform".to_string()) - }; - tokio::time::timeout(timeout, fut) - .await - .map_err(|_| "IPC request timed out".to_string())? -} - #[allow(clippy::too_many_arguments)] pub(crate) async fn transport_request( - client: &reqwest::Client, - method: &str, - base_url: &str, - path: &str, - api_key: Option<&str>, - allow_local_token_fallback: bool, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, + client: &reqwest::Client, method: &str, base_url: &str, path: &str, api_key: Option<&str>, allow_local_token_fallback: bool, headers: &[(String, String)], + body: Option<&str>, timeout: std::time::Duration, ) -> Result<(reqwest::StatusCode, String), String> { let mut all_headers = Vec::with_capacity(headers.len() + 1); all_headers.extend_from_slice(headers); if let Some(auth) = build_auth_header(base_url, api_key, allow_local_token_fallback) { all_headers.push(("authorization".to_string(), auth)); } - - if let Some(endpoint) = resolve_local_ipc_endpoint(base_url, api_key) { - match ipc_http_request(&endpoint, method, path, &all_headers, body, timeout).await { - Ok(response) => return Ok(response), - Err(err) => { - eprintln!( - "[cortex-mcp] IPC request failed for {method} {path} ({endpoint}): {err}; falling back to HTTP" - ); - } - } - } - - let url = format!("{base_url}{path}"); - let mut req = match method { - "GET" => client.get(&url), - "POST" => client.post(&url), - other => return Err(format!("Unsupported request method '{other}'")), - }; - req = req.timeout(timeout); - for (name, value) in &all_headers { - req = req.header(name, value); - } - if let Some(payload) = body { - req = req.body(payload.to_string()); + if api_key.is_none() && is_local_daemon_base(base_url) { + return crate::transport::request_with_local_ipc_fallback(client, method, base_url, path, &CortexPaths::resolve(), &all_headers, body, timeout).await; } - let response = req.send().await.map_err(|e| e.to_string())?; - let status = response.status(); - let body = response.text().await.map_err(|e| e.to_string())?; - Ok((status, body)) + crate::transport::send_http_request(client, method, &format!("{base_url}{path}"), &all_headers, body, timeout).await } - pub(crate) async fn transport_request_for_url( - client: &reqwest::Client, - method: &str, - url: &str, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, + client: &reqwest::Client, method: &str, url: &str, headers: &[(String, String)], body: Option<&str>, timeout: std::time::Duration, ) -> Result<(reqwest::StatusCode, String), String> { - let Some((base_url, path)) = split_base_and_path(url) else { - let mut req = match method { - "GET" => client.get(url), - "POST" => client.post(url), - other => return Err(format!("Unsupported request method '{other}'")), - }; - req = req.timeout(timeout); - for (name, value) in headers { - req = req.header(name, value); - } - if let Some(payload) = body { - req = req.body(payload.to_string()); - } - let response = req.send().await.map_err(|e| e.to_string())?; - let status = response.status(); - let body = response.text().await.map_err(|e| e.to_string())?; - return Ok((status, body)); + let Some((base_url, path)) = crate::transport::split_base_and_path(url) else { + return crate::transport::send_http_request(client, method, url, headers, body, timeout).await; }; - transport_request( - client, method, &base_url, &path, None, false, headers, body, timeout, - ) - .await + transport_request(client, method, &base_url, &path, None, false, headers, body, timeout).await } - pub(crate) fn local_token_fallback_required(base_url: &str, api_key: Option<&str>) -> bool { api_key.is_none() && is_local_daemon_base(base_url) } - -pub(crate) fn build_auth_header( - base_url: &str, - api_key: Option<&str>, - allow_local_token_fallback: bool, -) -> Option { +pub(crate) fn build_auth_header(base_url: &str, api_key: Option<&str>, allow_local_token_fallback: bool) -> Option { if let Some(key) = api_key { return Some(format!("Bearer {key}")); } @@ -494,199 +251,93 @@ pub(crate) fn build_auth_header( } None } - pub(crate) fn requires_explicit_api_key(base_url: &str, api_key: Option<&str>) -> bool { api_key.is_none() && !is_local_daemon_base(base_url) } - pub(crate) fn validate_target_base_url(base_url: &str) -> Result<(), String> { - let parsed = reqwest::Url::parse(base_url).map_err(|_| { - format!( - "Invalid Cortex target URL '{base_url}'. Use an absolute http:// or https:// base URL." - ) - })?; + let parsed = reqwest::Url::parse(base_url).map_err(|_| format!("Invalid Cortex target URL '{base_url}'. Use an absolute http:// or https:// base URL."))?; if !matches!(parsed.scheme(), "http" | "https") { - return Err(format!( - "Unsupported Cortex target URL scheme '{}' in '{base_url}'. Use http or https.", - parsed.scheme() - )); + return Err(format!("Unsupported Cortex target URL scheme '{}' in '{base_url}'. Use http or https.", parsed.scheme())); } if parsed.host_str().is_none() { - return Err(format!( - "Invalid Cortex target URL '{base_url}': missing host." - )); + return Err(format!("Invalid Cortex target URL '{base_url}': missing host.")); } if !parsed.username().is_empty() || parsed.password().is_some() { - return Err( - "Cortex target URL must not include embedded credentials; pass --api-key instead." - .to_string(), - ); + return Err("Cortex target URL must not include embedded credentials; pass --api-key instead.".to_string()); } if parsed.query().is_some() || parsed.fragment().is_some() { - return Err( - "Cortex target URL must not include query parameters or fragments.".to_string(), - ); + return Err("Cortex target URL must not include query parameters or fragments.".to_string()); } Ok(()) } - pub(crate) fn expected_port_from_url(url: &str) -> Option { - reqwest::Url::parse(url) - .ok() - .and_then(|parsed| parsed.port_or_known_default()) + reqwest::Url::parse(url).ok().and_then(|parsed| parsed.port_or_known_default()) } - pub(crate) fn fallback_health_probe_url(probe_url: &str) -> Option { - probe_url - .strip_suffix("/readiness") - .map(|base| format!("{base}/health")) + probe_url.strip_suffix("/readiness").map(|base| format!("{base}/health")) } - pub(crate) fn internal_health_probe_headers() -> [(String, String); 1] { [(String::from("X-Cortex-Request"), String::from("true"))] } - pub(crate) fn is_cortex_health_response(status: reqwest::StatusCode, body: &str, probe_url: &str) -> bool { - let local_paths = if is_local_daemon_base(probe_url) { - Some(CortexPaths::resolve()) - } else { - None - }; - if let Some(ready) = daemon_lifecycle::readiness_state_from_payload( - status.as_u16(), - body, - expected_port_from_url(probe_url), - local_paths.as_ref(), - ) { + let local_paths = if is_local_daemon_base(probe_url) { Some(CortexPaths::resolve()) } else { None }; + if let Some(ready) = daemon_lifecycle::readiness_state_from_payload(status.as_u16(), body, expected_port_from_url(probe_url), local_paths.as_ref()) { return ready; } - daemon_lifecycle::is_cortex_health_payload( - status.as_u16(), - body, - expected_port_from_url(probe_url), - local_paths.as_ref(), - ) + daemon_lifecycle::is_cortex_health_payload(status.as_u16(), body, expected_port_from_url(probe_url), local_paths.as_ref()) } - pub(crate) async fn health_check_ready(client: &reqwest::Client, probe_url: &str) -> bool { let probe_headers = internal_health_probe_headers(); - let (status, body) = match transport_request_for_url( - client, - "GET", - probe_url, - &probe_headers, - None, - std::time::Duration::from_secs(5), - ) - .await - { + let (status, body) = match transport_request_for_url(client, "GET", probe_url, &probe_headers, None, std::time::Duration::from_secs(5)).await { Ok(response) => response, Err(_) => return false, }; - if is_cortex_health_response(status, &body, probe_url) { return true; } - let Some(health_url) = fallback_health_probe_url(probe_url) else { return false; }; - let (status, body) = match transport_request_for_url( - client, - "GET", - &health_url, - &probe_headers, - None, - std::time::Duration::from_secs(5), - ) - .await - { + let (status, body) = match transport_request_for_url(client, "GET", &health_url, &probe_headers, None, std::time::Duration::from_secs(5)).await { Ok(response) => response, Err(_) => return false, }; is_cortex_health_response(status, &body, &health_url) } - pub(crate) fn is_auth_recovery_status(status: reqwest::StatusCode) -> bool { status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN } - pub(crate) async fn recover_solo_auth( - client: &reqwest::Client, - health_url: &str, - base_url: &str, - agent: &str, - model: Option<&str>, - allow_local_token_fallback: &mut bool, + client: &reqwest::Client, health_url: &str, base_url: &str, agent: &str, model: Option<&str>, allow_local_token_fallback: &mut bool, ) -> bool { if !health_check_ready(client, health_url).await { *allow_local_token_fallback = false; return false; } *allow_local_token_fallback = true; - - if !session_start_with_retry( - client, - base_url, - None, - agent, - model, - *allow_local_token_fallback, - ) - .await - { + if !session_start_with_retry(client, base_url, None, agent, model, *allow_local_token_fallback).await { eprintln!("[cortex-mcp] Auth recovered but session re-registration did not succeed yet"); return false; } - true } - pub(crate) async fn session_start_with_retry( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - model: Option<&str>, - allow_local_token_fallback: bool, + client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, model: Option<&str>, allow_local_token_fallback: bool, ) -> bool { for attempt in 1..=SESSION_RESTART_ATTEMPTS.max(1) { - if session_start( - client, - base_url, - api_key, - agent, - model, - allow_local_token_fallback, - ) - .await - { + if session_start(client, base_url, api_key, agent, model, allow_local_token_fallback).await { return true; } - if attempt < SESSION_RESTART_ATTEMPTS { - tokio::time::sleep(std::time::Duration::from_millis( - SESSION_RESTART_DELAY_MS * attempt as u64, - )) - .await; + tokio::time::sleep(std::time::Duration::from_millis(SESSION_RESTART_DELAY_MS * attempt as u64)).await; } } - false } - -pub(crate) fn persist_write_buffer( - buffer_path: &std::path::Path, - remaining: &[String], -) -> Result<(), std::io::Error> { +pub(crate) fn persist_write_buffer(buffer_path: &std::path::Path, remaining: &[String]) -> Result<(), std::io::Error> { use std::io::{BufWriter, Write}; - - let parent = buffer_path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| std::path::Path::new(".")); + let parent = buffer_path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or_else(|| std::path::Path::new(".")); std::fs::create_dir_all(parent)?; - let mut tmp = tempfile::NamedTempFile::new_in(parent)?; { let mut writer = BufWriter::new(tmp.as_file_mut()); @@ -700,45 +351,28 @@ pub(crate) fn persist_write_buffer( sync_parent_dir(parent)?; Ok(()) } - #[cfg(unix)] pub(crate) fn sync_parent_dir(parent: &std::path::Path) -> std::io::Result<()> { std::fs::File::open(parent)?.sync_all() } - #[cfg(not(unix))] pub(crate) fn sync_parent_dir(_parent: &std::path::Path) -> std::io::Result<()> { Ok(()) } - pub(crate) async fn drain_write_buffer( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - model: Option<&str>, - paths: &CortexPaths, - allow_local_token_fallback: bool, + client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, model: Option<&str>, paths: &CortexPaths, allow_local_token_fallback: bool, ) { let buffer_path = &paths.write_buffer; let content = match std::fs::read_to_string(buffer_path) { Ok(content) if !content.trim().is_empty() => content, _ => return, }; - - let lines: Vec = content - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(|line| line.to_string()) - .collect(); + let lines: Vec = content.lines().map(str::trim).filter(|line| !line.is_empty()).map(|line| line.to_string()).collect(); if lines.is_empty() { return; } - let mut remaining = Vec::new(); let mut drained = 0usize; - for line in lines { let mut headers = vec![ ("content-type".to_string(), "application/json".to_string()), @@ -748,7 +382,6 @@ pub(crate) async fn drain_write_buffer( if let Some(model_name) = model { headers.push(("x-source-model".to_string(), model_name.to_string())); } - match transport_request( client, "POST", @@ -768,43 +401,21 @@ pub(crate) async fn drain_write_buffer( _ => remaining.push(line), } } - if let Err(e) = persist_write_buffer(buffer_path, &remaining) { - eprintln!( - "[cortex-mcp] Failed to compact write buffer {}: {e}", - buffer_path.display() - ); + eprintln!("[cortex-mcp] Failed to compact write buffer {}: {e}", buffer_path.display()); return; } - if drained > 0 { - eprintln!( - "[cortex-mcp] Drained {drained} buffered writes and compacted {}", - buffer_path.display() - ); + eprintln!("[cortex-mcp] Drained {drained} buffered writes and compacted {}", buffer_path.display()); } } - pub(crate) async fn session_start( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - model: Option<&str>, - allow_local_token_fallback: bool, + client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, model: Option<&str>, allow_local_token_fallback: bool, ) -> bool { - let payload = serde_json::json!({ - "agent": agent, - "ttl": 7200, - "description": model - .map(|m| format!("MCP session - {m}")) - .unwrap_or_else(|| "MCP session".to_string()) - }) + let payload = serde_json::json!({"agent":agent,"ttl":7200,"description":model.map(|m|format!("MCP session - {m}")).unwrap_or_else(|| +"MCP session".to_string())}) .to_string(); - let headers = vec![ - ("content-type".to_string(), "application/json".to_string()), - ("x-cortex-request".to_string(), "true".to_string()), - ]; + let headers = vec![("content-type".to_string(), "application/json".to_string()), ("x-cortex-request".to_string(), "true".to_string())]; match transport_request( client, "POST", @@ -822,33 +433,18 @@ pub(crate) async fn session_start( Err(_) => false, } } - pub(crate) enum SessionHeartbeatOutcome { Renewed, MissingSession, Failed, } - pub(crate) async fn session_heartbeat( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - model: Option<&str>, - allow_local_token_fallback: bool, + client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, model: Option<&str>, allow_local_token_fallback: bool, ) -> SessionHeartbeatOutcome { - let payload = serde_json::json!({ - "agent": agent, - "description": model - .map(|m| format!("MCP session - {m}")) - .unwrap_or_else(|| "MCP session".to_string()) - }) + let payload = serde_json::json!({"agent":agent,"description":model.map(|m|format!("MCP session - {m}")). +unwrap_or_else(||"MCP session".to_string())}) .to_string(); - let headers = vec![ - ("content-type".to_string(), "application/json".to_string()), - ("x-cortex-request".to_string(), "true".to_string()), - ]; - + let headers = vec![("content-type".to_string(), "application/json".to_string()), ("x-cortex-request".to_string(), "true".to_string())]; match transport_request( client, "POST", @@ -863,25 +459,15 @@ pub(crate) async fn session_heartbeat( .await { Ok((status, _)) if status.is_success() => SessionHeartbeatOutcome::Renewed, - Ok((status, _)) if status == reqwest::StatusCode::NOT_FOUND => { - SessionHeartbeatOutcome::MissingSession - } + Ok((status, _)) if status == reqwest::StatusCode::NOT_FOUND => SessionHeartbeatOutcome::MissingSession, Ok(_) | Err(_) => SessionHeartbeatOutcome::Failed, } } - -pub(crate) async fn session_end( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - allow_local_token_fallback: bool, -) -> bool { - let payload = serde_json::json!({ "agent": agent }).to_string(); - let headers = vec![ - ("content-type".to_string(), "application/json".to_string()), - ("x-cortex-request".to_string(), "true".to_string()), - ]; +pub(crate) async fn session_end(client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, allow_local_token_fallback: bool) -> bool { + let payload = serde_json::json! +({"agent":agent}) + .to_string(); + let headers = vec![("content-type".to_string(), "application/json".to_string()), ("x-cortex-request".to_string(), "true".to_string())]; match transport_request( client, "POST", @@ -899,14 +485,6 @@ pub(crate) async fn session_end( Err(_) => false, } } - -pub(crate) async fn finalize_proxy_session( - client: &reqwest::Client, - base_url: &str, - api_key: Option<&str>, - agent: &str, - allow_local_token_fallback: bool, -) { +pub(crate) async fn finalize_proxy_session(client: &reqwest::Client, base_url: &str, api_key: Option<&str>, agent: &str, allow_local_token_fallback: bool) { let _ = session_end(client, base_url, api_key, agent, allow_local_token_fallback).await; } - diff --git a/daemon-rs/src/mcp_proxy/tests/mod.rs b/daemon-rs/src/mcp_proxy/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/mcp_proxy/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/prompt_inject.rs b/daemon-rs/src/prompt_inject/mod.rs similarity index 51% rename from daemon-rs/src/prompt_inject.rs rename to daemon-rs/src/prompt_inject/mod.rs index 3aed7fd6..8e388784 100644 --- a/daemon-rs/src/prompt_inject.rs +++ b/daemon-rs/src/prompt_inject/mod.rs @@ -1,19 +1,7 @@ -// SPDX-License-Identifier: MIT -//! System prompt injector CLI. -//! -//! `cortex prompt-inject --file [--agent NAME] [--budget N] [--watch]` -//! -//! Reads a base system prompt file, appends Cortex context (boot data), -//! and writes the result to `.injected`. With `--watch`, re-injects -//! whenever the source file changes (file-based refresh). - use std::ffi::OsString; use std::path::{Path, PathBuf}; - const DEFAULT_BUDGET: u32 = 400; -const USAGE: &str = - "Usage: cortex prompt-inject --file [--agent NAME] [--budget N] [--watch]"; - +const USAGE: &str = "Usage: cortex prompt-inject --file [--agent NAME] [--budget N] [--watch]"; #[derive(Clone, Debug, PartialEq, Eq)] struct PromptInjectConfig { file_path: PathBuf, @@ -21,13 +9,11 @@ struct PromptInjectConfig { budget: u32, watch: bool, } - fn parse_args(args: &[String]) -> Result { let mut file_path: Option = None; let mut agent = "prompt-inject".to_string(); let mut budget = DEFAULT_BUDGET; let mut watch = false; - let mut i = 0; while i < args.len() { match args[i].as_str() { @@ -50,9 +36,7 @@ fn parse_args(args: &[String]) -> Result { if i >= args.len() { return Err(format!("{USAGE}\nMissing value for --budget")); } - budget = args[i] - .parse() - .map_err(|_| format!("{USAGE}\nInvalid --budget '{}'", args[i]))?; + budget = args[i].parse().map_err(|_| format!("{USAGE}\nInvalid --budget '{}'", args[i]))?; } "--watch" | "-w" => { watch = true; @@ -63,38 +47,24 @@ fn parse_args(args: &[String]) -> Result { } i += 1; } - let Some(file_path) = file_path else { return Err(format!("{USAGE}\nMissing required --file ")); }; - - Ok(PromptInjectConfig { - file_path, - agent, - budget, - watch, - }) + Ok(PromptInjectConfig { file_path, agent, budget, watch }) } - fn compose_injected_prompt(base_prompt: &str, cortex_context: &str) -> String { format!("{base_prompt}\n\n{cortex_context}") } - fn output_path_for(file_path: &Path) -> PathBuf { let mut out: OsString = file_path.as_os_str().to_os_string(); out.push(".injected"); PathBuf::from(out) } - pub async fn run(args: &[String]) { - if args - .iter() - .any(|arg| matches!(arg.as_str(), "--help" | "-h" | "help")) - { + if args.iter().any(|arg| matches!(arg.as_str(), "--help" | "-h" | "help")) { println!("{USAGE}"); return; } - let config = match parse_args(args) { Ok(config) => config, Err(usage) => { @@ -102,7 +72,6 @@ pub async fn run(args: &[String]) { std::process::exit(1); } }; - if config.watch { run_watch_loop(&config.file_path, &config.agent, config.budget).await; } else { @@ -112,40 +81,21 @@ pub async fn run(args: &[String]) { } } } - async fn inject_once(file_path: &Path, agent: &str, budget: u32) -> Result<(), String> { - let base_prompt = std::fs::read_to_string(file_path) - .map_err(|e| format!("Failed to read {}: {e}", file_path.display()))?; - + let base_prompt = std::fs::read_to_string(file_path).map_err(|e| format!("Failed to read {}: {e}", file_path.display()))?; let cortex_context = fetch_boot_context(agent, budget).await; - let output = compose_injected_prompt(&base_prompt, &cortex_context); - let out_path = output_path_for(file_path); - std::fs::write(&out_path, &output) - .map_err(|e| format!("Failed to write {}: {e}", out_path.display()))?; - - eprintln!( - "[prompt-inject] Wrote {} ({} bytes)", - out_path.display(), - output.len() - ); + std::fs::write(&out_path, &output).map_err(|e| format!("Failed to write {}: {e}", out_path.display()))?; + eprintln!("[prompt-inject] Wrote {} ({} bytes)", out_path.display(), output.len()); Ok(()) } - async fn run_watch_loop(file_path: &Path, agent: &str, budget: u32) { - eprintln!( - "[prompt-inject] Watching {} for changes (Ctrl+C to stop)", - file_path.display() - ); - + eprintln!("[prompt-inject] Watching {} for changes (Ctrl+C to stop)", file_path.display()); let mut last_modified = file_modified(file_path); - - // Initial injection if let Err(e) = inject_once(file_path, agent, budget).await { eprintln!("[prompt-inject] Initial inject error: {e}"); } - loop { tokio::time::sleep(std::time::Duration::from_secs(2)).await; let current = file_modified(file_path); @@ -158,7 +108,6 @@ async fn run_watch_loop(file_path: &Path, agent: &str, budget: u32) { } } } - fn file_modified(path: &Path) -> u128 { std::fs::metadata(path) .and_then(|m| m.modified()) @@ -167,7 +116,6 @@ fn file_modified(path: &Path) -> u128 { .map(|d| d.as_nanos()) .unwrap_or(0) } - async fn fetch_boot_context(agent: &str, budget: u32) -> String { let token = read_auth_token(); let client = match reqwest::Client::builder() @@ -178,7 +126,6 @@ async fn fetch_boot_context(agent: &str, budget: u32) -> String { Ok(c) => c, Err(e) => return format!(""), }; - let port = crate::auth::CortexPaths::resolve().port; let mut url = match reqwest::Url::parse(&format!("http://127.0.0.1:{port}/boot")) { Ok(u) => u, @@ -193,17 +140,11 @@ async fn fetch_boot_context(agent: &str, budget: u32) -> String { if let Some(t) = &token { req = req.header("Authorization", format!("Bearer {t}")); } - match req.send().await { Ok(resp) if resp.status().is_success() => match resp.json::().await { Ok(data) => { - let boot = data - .get("bootPrompt") - .and_then(|v| v.as_str()) - .unwrap_or("(no boot prompt)"); - format!( - "\n{boot}\n" - ) + let boot = data.get("bootPrompt").and_then(|v| v.as_str()).unwrap_or("(no boot prompt)"); + format!("\n{boot}\n") } Err(_) => "".to_string(), }, @@ -211,7 +152,6 @@ async fn fetch_boot_context(agent: &str, budget: u32) -> String { Err(e) => format!(""), } } - fn read_auth_token_from_path(path: &Path) -> Option { match std::fs::read_to_string(path) { Ok(token) => { @@ -225,120 +165,9 @@ fn read_auth_token_from_path(path: &Path) -> Option { Err(_) => None, } } - fn read_auth_token() -> Option { let path = crate::auth::CortexPaths::resolve().token; read_auth_token_from_path(&path) } - #[cfg(test)] -mod tests { - use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn unique_temp_dir(name: &str) -> PathBuf { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - std::env::temp_dir().join(format!("cortex_prompt_inject_{name}_{unique}")) - } - - #[test] - fn parse_args_supports_short_and_long_flags() { - let args = vec![ - "--file".to_string(), - "C:/tmp/system.txt".to_string(), - "-a".to_string(), - "codex".to_string(), - "--budget".to_string(), - "512".to_string(), - "-w".to_string(), - ]; - let parsed = parse_args(&args).expect("args should parse"); - assert_eq!(parsed.file_path, PathBuf::from("C:/tmp/system.txt")); - assert_eq!(parsed.agent, "codex"); - assert_eq!(parsed.budget, 512); - assert!(parsed.watch); - } - - #[test] - fn parse_args_requires_file() { - let args = vec!["--agent".to_string(), "codex".to_string()]; - let err = parse_args(&args).expect_err("missing file should error"); - assert!(err.contains("Missing required --file ")); - } - - #[test] - fn parse_args_missing_agent_value_errors() { - let args = vec![ - "--file".to_string(), - "prompt.txt".to_string(), - "--agent".to_string(), - ]; - let err = parse_args(&args).expect_err("missing agent value should error"); - assert!(err.contains("Missing value for --agent")); - } - - #[test] - fn parse_args_invalid_budget_errors() { - let args = vec![ - "--file".to_string(), - "prompt.txt".to_string(), - "--budget".to_string(), - "not-a-number".to_string(), - ]; - let err = parse_args(&args).expect_err("invalid budget should error"); - assert!(err.contains("Invalid --budget")); - } - - #[test] - fn parse_args_rejects_unknown_flags() { - let args = vec![ - "--file".to_string(), - "prompt.txt".to_string(), - "--budegt".to_string(), - "512".to_string(), - ]; - - let err = parse_args(&args).expect_err("unknown flag should error"); - assert!(err.contains("Unknown option: --budegt")); - assert!(err.contains(USAGE)); - } - - #[test] - fn compose_injected_prompt_appends_cortex_context() { - let output = compose_injected_prompt("base prompt", ""); - assert_eq!(output, "base prompt\n\n"); - } - - #[test] - fn file_modified_returns_zero_for_missing_path() { - let path = PathBuf::from("__missing_prompt_inject_file__.txt"); - assert_eq!(file_modified(&path), 0); - } - - #[test] - fn output_path_appends_injected_suffix() { - let path = PathBuf::from("C:/tmp/system.txt"); - let out = output_path_for(&path); - assert_eq!(out, PathBuf::from("C:/tmp/system.txt.injected")); - - let dotfile = PathBuf::from("C:/tmp/.env"); - let dot_out = output_path_for(&dotfile); - assert_eq!(dot_out, PathBuf::from("C:/tmp/.env.injected")); - } - - #[test] - fn read_auth_token_from_path_reads_trimmed_token() { - let temp_home = unique_temp_dir("token"); - std::fs::create_dir_all(&temp_home).expect("create temp home"); - let token_path = temp_home.join("cortex.token"); - std::fs::write(&token_path, "ctx_prompt_token\n").expect("write token file"); - - let token = read_auth_token_from_path(&token_path); - assert_eq!(token.as_deref(), Some("ctx_prompt_token")); - - let _ = std::fs::remove_dir_all(&temp_home); - } -} +mod tests; diff --git a/daemon-rs/src/prompt_inject/tests/mod.rs b/daemon-rs/src/prompt_inject/tests/mod.rs new file mode 100644 index 00000000..90ba8479 --- /dev/null +++ b/daemon-rs/src/prompt_inject/tests/mod.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use std::time::{SystemTime, UNIX_EPOCH}; +fn unique_temp_dir(name: &str) -> PathBuf { + let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos(); + std::env::temp_dir().join(format!("cortex_prompt_inject_{name}_{unique}")) +} +#[test] +fn parse_args_supports_short_and_long_flags() { + let args = vec![ + "--file".to_string(), + "C:/tmp/system.txt".to_string(), + "-a".to_string(), + "codex".to_string(), + "--budget".to_string(), + "512".to_string(), + "-w".to_string(), + ]; + let parsed = parse_args(&args).expect("args should parse"); + assert_eq!(parsed.file_path, PathBuf::from("C:/tmp/system.txt")); + assert_eq!(parsed.agent, "codex"); + assert_eq!(parsed.budget, 512); + assert!(parsed.watch); +} +#[test] +fn parse_args_requires_file() { + let args = vec!["--agent".to_string(), "codex".to_string()]; + let err = parse_args(&args).expect_err("missing file should error"); + assert!(err.contains("Missing required --file ")); +} +#[test] +fn parse_args_missing_agent_value_errors() { + let args = vec!["--file".to_string(), "prompt.txt".to_string(), "--agent".to_string()]; + let err = parse_args(&args).expect_err("missing agent value should error"); + assert!(err.contains("Missing value for --agent")); +} +#[test] +fn parse_args_invalid_budget_errors() { + let args = vec!["--file".to_string(), "prompt.txt".to_string(), "--budget".to_string(), "not-a-number".to_string()]; + let err = parse_args(&args).expect_err("invalid budget should error"); + assert!(err.contains("Invalid --budget")); +} +#[test] +fn parse_args_rejects_unknown_flags() { + let args = vec!["--file".to_string(), "prompt.txt".to_string(), "--budegt".to_string(), "512".to_string()]; + let err = parse_args(&args).expect_err("unknown flag should error"); + assert!(err.contains("Unknown option: --budegt")); + assert!(err.contains(USAGE)); +} +#[test] +fn compose_injected_prompt_appends_cortex_context() { + let output = compose_injected_prompt("base prompt", ""); + assert_eq!(output, "base prompt\n\n"); +} +#[test] +fn file_modified_returns_zero_for_missing_path() { + let path = PathBuf::from("__missing_prompt_inject_file__.txt"); + assert_eq!(file_modified(&path), 0); +} +#[test] +fn output_path_appends_injected_suffix() { + let path = PathBuf::from("C:/tmp/system.txt"); + let out = output_path_for(&path); + assert_eq!(out, PathBuf::from("C:/tmp/system.txt.injected")); + let dotfile = PathBuf::from("C:/tmp/.env"); + let dot_out = output_path_for(&dotfile); + assert_eq!(dot_out, PathBuf::from("C:/tmp/.env.injected")); +} +#[test] +fn read_auth_token_from_path_reads_trimmed_token() { + let temp_home = unique_temp_dir("token"); + std::fs::create_dir_all(&temp_home).expect("create temp home"); + let token_path = temp_home.join("cortex.token"); + std::fs::write(&token_path, "ctx_prompt_token\n").expect("write token file"); + let token = read_auth_token_from_path(&token_path); + assert_eq!(token.as_deref(), Some("ctx_prompt_token")); + let _ = std::fs::remove_dir_all(&temp_home); +} diff --git a/daemon-rs/src/rate_limit.rs b/daemon-rs/src/rate_limit.rs deleted file mode 100644 index df80a11d..00000000 --- a/daemon-rs/src/rate_limit.rs +++ /dev/null @@ -1,560 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Sliding-window rate limiter. -//! -//! Two independent buckets per source IP: -//! - Auth failures: 10 per minute (brute-force protection) -//! - Request volume: 100 per minute for non-loopback callers -//! - Request volume: 10,000 per minute for loopback callers (desktop/plugin local workloads) -//! -//! Responses include `Retry-After`, `X-RateLimit-Remaining`, and -//! `X-RateLimit-Reset` headers when the limit is hit. -#![allow(dead_code)] - -use std::collections::{HashMap, VecDeque}; -use std::net::IpAddr; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; - -use crate::budgets::{BudgetConfigStatus, BudgetDecision, BudgetEndpoint, EndpointBudget}; - -const AUTH_FAIL_LIMIT: usize = 10; -const REQUEST_LIMIT_NON_LOOPBACK: usize = 100; -const REQUEST_LIMIT_LOOPBACK: usize = 10_000; -const WINDOW: Duration = Duration::from_secs(60); -const BUDGET_DENIAL_RECENT_WINDOW: Duration = Duration::from_secs(60 * 60); -const LIMIT_MAX: usize = 1_000_000; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum RequestClass { - Default, - Recall, - Store, - Boot, -} - -fn read_limit_env(key: &str, default: usize) -> usize { - std::env::var(key) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|value| value.clamp(1, LIMIT_MAX)) - .unwrap_or(default) -} - -#[derive(Clone)] -struct SlidingWindow { - timestamps: VecDeque, -} - -impl SlidingWindow { - fn new() -> Self { - Self { - timestamps: VecDeque::new(), - } - } - - fn prune(&mut self, now: Instant, window: Duration) { - while let Some(oldest) = self.timestamps.front().copied() { - if now.duration_since(oldest) < window { - break; - } - self.timestamps.pop_front(); - } - } - - fn seconds_until_slot_pruned(&self, now: Instant, limit: usize, window: Duration) -> u64 { - if self.timestamps.len() < limit { - return 0; - } - let oldest = self.timestamps.front().copied().unwrap_or(now); - let elapsed = now.duration_since(oldest); - window.as_secs().saturating_sub(elapsed.as_secs()).max(1) - } - - fn try_record(&mut self, now: Instant, limit: usize, window: Duration) -> Result { - self.prune(now, window); - let current = self.timestamps.len(); - if current >= limit { - return Err(self.seconds_until_slot_pruned(now, limit, window)); - } - self.timestamps.push_back(now); - Ok(limit - current - 1) - } - - fn record_unbounded(&mut self, now: Instant, window: Duration) { - self.prune(now, window); - self.timestamps.push_back(now); - } - - fn len_after_prune(&mut self, now: Instant, window: Duration) -> usize { - self.prune(now, window); - self.timestamps.len() - } -} - -/// Shared rate limiter state, added to `RuntimeState`. -#[derive(Clone)] -pub struct RateLimiter { - auth_failures: Arc>>, - requests: Arc>>, - budget_requests: Arc>>, - budget_denials: Arc>, - total_budget_denials: Arc, - budget_config_status: Arc, - auth_fail_limit: usize, - request_limit_non_loopback: usize, - request_limit_loopback: usize, - recall_request_limit_non_loopback: usize, - recall_request_limit_loopback: usize, - store_request_limit_non_loopback: usize, - store_request_limit_loopback: usize, -} - -impl RateLimiter { - pub fn new() -> Self { - Self::new_with_budget_status(BudgetConfigStatus::missing_for_tests()) - } - - pub fn new_with_budget_status(budget_config_status: BudgetConfigStatus) -> Self { - let auth_fail_limit = - read_limit_env("CORTEX_RATE_LIMIT_AUTH_FAILS_PER_MIN", AUTH_FAIL_LIMIT); - let request_limit_non_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_REQUESTS_PER_MIN", - REQUEST_LIMIT_NON_LOOPBACK, - ); - let request_limit_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_LOOPBACK_REQUESTS_PER_MIN", - REQUEST_LIMIT_LOOPBACK, - ); - let recall_request_limit_non_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_RECALL_REQUESTS_PER_MIN", - request_limit_non_loopback, - ); - let recall_request_limit_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_RECALL_LOOPBACK_REQUESTS_PER_MIN", - request_limit_loopback, - ); - let store_request_limit_non_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_STORE_REQUESTS_PER_MIN", - request_limit_non_loopback, - ); - let store_request_limit_loopback = read_limit_env( - "CORTEX_RATE_LIMIT_STORE_LOOPBACK_REQUESTS_PER_MIN", - request_limit_loopback, - ); - if auth_fail_limit != AUTH_FAIL_LIMIT - || request_limit_non_loopback != REQUEST_LIMIT_NON_LOOPBACK - || request_limit_loopback != REQUEST_LIMIT_LOOPBACK - || recall_request_limit_non_loopback != request_limit_non_loopback - || recall_request_limit_loopback != request_limit_loopback - || store_request_limit_non_loopback != request_limit_non_loopback - || store_request_limit_loopback != request_limit_loopback - { - eprintln!( - "[cortex] Rate limiter configured: auth_fails/min={auth_fail_limit}, default_requests/min(non-loopback)={request_limit_non_loopback}, default_requests/min(loopback)={request_limit_loopback}, recall_requests/min(non-loopback)={recall_request_limit_non_loopback}, recall_requests/min(loopback)={recall_request_limit_loopback}, store_requests/min(non-loopback)={store_request_limit_non_loopback}, store_requests/min(loopback)={store_request_limit_loopback}" - ); - } - Self { - auth_failures: Arc::new(Mutex::new(HashMap::new())), - requests: Arc::new(Mutex::new(HashMap::new())), - budget_requests: Arc::new(Mutex::new(HashMap::new())), - budget_denials: Arc::new(Mutex::new(SlidingWindow::new())), - total_budget_denials: Arc::new(AtomicUsize::new(0)), - budget_config_status: Arc::new(budget_config_status), - auth_fail_limit, - request_limit_non_loopback, - request_limit_loopback, - recall_request_limit_non_loopback, - recall_request_limit_loopback, - store_request_limit_non_loopback, - store_request_limit_loopback, - } - } - - fn request_limit_for_ip_class(&self, ip: IpAddr, class: RequestClass) -> usize { - let loopback = ip.is_loopback(); - match class { - RequestClass::Default | RequestClass::Boot => { - if loopback { - self.request_limit_loopback - } else { - self.request_limit_non_loopback - } - } - RequestClass::Recall => { - if loopback { - self.recall_request_limit_loopback - } else { - self.recall_request_limit_non_loopback - } - } - RequestClass::Store => { - if loopback { - self.store_request_limit_loopback - } else { - self.store_request_limit_non_loopback - } - } - } - } - - /// Record a failed auth attempt. Returns `Err(retry_after_secs)` if blocked. - pub async fn record_auth_failure(&self, ip: IpAddr) -> Result<(), u64> { - let mut map = self.auth_failures.lock().await; - let window = map.entry(ip).or_insert_with(SlidingWindow::new); - let now = Instant::now(); - window - .try_record(now, self.auth_fail_limit, WINDOW) - .map(|_| ()) - } - - /// Check if an IP is currently blocked due to auth failures. - pub async fn is_auth_blocked(&self, ip: &IpAddr) -> Option { - let mut map = self.auth_failures.lock().await; - if let Some(window) = map.get_mut(ip) { - let now = Instant::now(); - window.prune(now, WINDOW); - if window.timestamps.len() >= self.auth_fail_limit { - return Some(window.seconds_until_slot_pruned(now, self.auth_fail_limit, WINDOW)); - } - } - None - } - - /// Check and record a request. Returns `Ok(remaining)` or `Err(retry_after)`. - pub async fn check_request(&self, ip: IpAddr) -> Result { - self.check_request_for_class(ip, RequestClass::Default) - .await - } - - /// Check and record a request for a route class. - /// Returns `Ok(remaining)` or `Err(retry_after)`. - pub async fn check_request_for_class( - &self, - ip: IpAddr, - class: RequestClass, - ) -> Result { - let mut map = self.requests.lock().await; - let window = map.entry((ip, class)).or_insert_with(SlidingWindow::new); - let request_limit = self.request_limit_for_ip_class(ip, class); - let now = Instant::now(); - window.try_record(now, request_limit, WINDOW) - } - - pub fn budget_status(&self) -> BudgetConfigStatus { - (*self.budget_config_status).clone() - } - - pub fn budget_for_endpoint(&self, endpoint: BudgetEndpoint) -> Option { - self.budget_config_status.budget_for(endpoint) - } - - pub async fn check_budget_for_endpoint( - &self, - ip: IpAddr, - endpoint: BudgetEndpoint, - ) -> Option { - let budget = self.budget_for_endpoint(endpoint)?; - let window_duration = Duration::from_secs(budget.window_seconds); - let mut map = self.budget_requests.lock().await; - let window = map.entry((ip, endpoint)).or_insert_with(SlidingWindow::new); - let now = Instant::now(); - match window.try_record(now, budget.limit, window_duration) { - Ok(remaining) => Some(BudgetDecision::allowed(endpoint, budget, remaining)), - Err(retry_after) => { - drop(map); - self.record_budget_denial().await; - Some(BudgetDecision::denied(endpoint, budget, retry_after)) - } - } - } - - async fn record_budget_denial(&self) { - self.total_budget_denials.fetch_add(1, Ordering::Relaxed); - let mut denials = self.budget_denials.lock().await; - denials.record_unbounded(Instant::now(), BUDGET_DENIAL_RECENT_WINDOW); - } - - pub async fn recent_budget_denials(&self) -> usize { - let mut denials = self.budget_denials.lock().await; - denials.len_after_prune(Instant::now(), BUDGET_DENIAL_RECENT_WINDOW) - } - - #[allow(dead_code)] - pub fn total_budget_denials(&self) -> usize { - self.total_budget_denials.load(Ordering::Relaxed) - } - - /// Periodic cleanup of stale entries (call from background task). - pub async fn cleanup(&self) { - let now = Instant::now(); - { - let mut map = self.auth_failures.lock().await; - map.retain(|_, w| { - w.prune(now, WINDOW); - !w.timestamps.is_empty() - }); - } - { - let mut map = self.requests.lock().await; - map.retain(|_, w| { - w.prune(now, WINDOW); - !w.timestamps.is_empty() - }); - } - { - let budget_status = self.budget_status(); - let mut map = self.budget_requests.lock().await; - map.retain(|(_, endpoint), w| { - let window = budget_status - .budget_for(*endpoint) - .map(|budget| Duration::from_secs(budget.window_seconds)) - .unwrap_or(WINDOW); - w.prune(now, window); - !w.timestamps.is_empty() - }); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::net::Ipv4Addr; - - #[tokio::test] - async fn test_request_limit_allows_under_limit() { - let rl = RateLimiter::new(); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); - for _ in 0..99 { - assert!(rl.check_request(ip).await.is_ok()); - } - } - - #[tokio::test] - async fn test_request_limit_blocks_at_limit() { - let rl = RateLimiter::new(); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 7)); - let limit = rl.request_limit_for_ip_class(ip, RequestClass::Default); - for _ in 0..limit { - let _ = rl.check_request(ip).await; - } - assert!(rl.check_request(ip).await.is_err()); - } - - #[tokio::test] - async fn test_loopback_has_higher_request_limit_than_non_loopback() { - let rl = RateLimiter::new(); - let loopback = IpAddr::V4(Ipv4Addr::LOCALHOST); - let remote = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 9)); - assert!( - rl.request_limit_for_ip_class(loopback, RequestClass::Default) - > rl.request_limit_for_ip_class(remote, RequestClass::Default) - ); - } - - #[tokio::test] - async fn test_auth_failure_blocks_after_limit() { - let rl = RateLimiter::new(); - let ip = IpAddr::V4(Ipv4Addr::LOCALHOST); - for _ in 0..AUTH_FAIL_LIMIT { - let _ = rl.record_auth_failure(ip).await; - } - assert!(rl.is_auth_blocked(&ip).await.is_some()); - } - - #[tokio::test] - async fn test_different_ips_independent() { - let rl = RateLimiter::new(); - let ip1 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); - let ip2 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)); - let limit = rl.request_limit_for_ip_class(ip1, RequestClass::Default); - for _ in 0..limit { - let _ = rl.check_request(ip1).await; - } - assert!(rl.check_request(ip1).await.is_err()); - assert!(rl.check_request(ip2).await.is_ok()); - } - - #[tokio::test] - async fn test_route_class_buckets_are_independent() { - let rl = RateLimiter::new(); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 42)); - let store_limit = rl.request_limit_for_ip_class(ip, RequestClass::Store); - for _ in 0..store_limit { - let _ = rl - .check_request_for_class(ip, RequestClass::Store) - .await - .expect("store class should allow requests below class limit"); - } - assert!( - rl.check_request_for_class(ip, RequestClass::Store) - .await - .is_err(), - "store class should rate limit once its own bucket is exhausted" - ); - assert!( - rl.check_request_for_class(ip, RequestClass::Recall) - .await - .is_ok(), - "recall class should remain available after store bucket is saturated" - ); - } - - #[tokio::test] - async fn test_cleanup_removes_stale() { - let rl = RateLimiter::new(); - let ip = IpAddr::V4(Ipv4Addr::LOCALHOST); - let _ = rl.check_request(ip).await; - rl.cleanup().await; - // Entry still there (not expired) - let map = rl.requests.lock().await; - assert!(map.contains_key(&(ip, RequestClass::Default))); - } - - #[test] - fn sliding_window_try_record_prunes_expired_front_entries() { - let mut window = SlidingWindow::new(); - let now = Instant::now(); - window.timestamps.push_back(now - Duration::from_secs(61)); - window.timestamps.push_back(now - Duration::from_secs(59)); - - let remaining = window - .try_record(now, 2, WINDOW) - .expect("expired entries should be pruned before limit check"); - assert_eq!(remaining, 0); - assert_eq!(window.timestamps.len(), 2); - assert!(window - .timestamps - .iter() - .all(|ts| now.duration_since(*ts) < WINDOW)); - - let retry = window - .try_record(now, 2, WINDOW) - .expect_err("window should be full at limit"); - assert_eq!(retry, 1); - - let later = now + Duration::from_secs(2); - assert!( - window.try_record(later, 2, WINDOW).is_ok(), - "oldest non-expired entry should age out and free a slot" - ); - } - - #[tokio::test] - async fn budget_allows_exactly_limit_then_rejects() { - let status = BudgetConfigStatus::load_from_path(write_budget_file( - r#" -[endpoints.recall] -limit = 2 -window_seconds = 60 -"#, - )); - let rl = RateLimiter::new_with_budget_status(status); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 88)); - - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall) - .await - .unwrap() - .allowed - ); - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall) - .await - .unwrap() - .allowed - ); - let denied = rl - .check_budget_for_endpoint(ip, BudgetEndpoint::Recall) - .await - .unwrap(); - assert!(!denied.allowed); - assert_eq!(denied.endpoint, BudgetEndpoint::Recall); - assert_eq!(denied.limit, 2); - assert_eq!(denied.window_seconds, 60); - assert_eq!( - denied.http_body_json()["source"], - crate::budgets::BUDGET_SOURCE - ); - } - - #[tokio::test] - async fn budget_resets_after_window() { - let status = BudgetConfigStatus::load_from_path(write_budget_file( - r#" -[endpoints.store] -limit = 1 -window_seconds = 1 -"#, - )); - let rl = RateLimiter::new_with_budget_status(status); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 89)); - - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store) - .await - .unwrap() - .allowed - ); - assert!( - !rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store) - .await - .unwrap() - .allowed - ); - tokio::time::sleep(Duration::from_millis(1100)).await; - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store) - .await - .unwrap() - .allowed - ); - } - - #[tokio::test] - async fn budget_endpoint_buckets_are_independent() { - let status = BudgetConfigStatus::load_from_path(write_budget_file( - r#" -[endpoints.store] -limit = 1 -window_seconds = 60 - -[endpoints.recall] -limit = 1 -window_seconds = 60 -"#, - )); - let rl = RateLimiter::new_with_budget_status(status); - let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 90)); - - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store) - .await - .unwrap() - .allowed - ); - assert!( - !rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store) - .await - .unwrap() - .allowed - ); - assert!( - rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall) - .await - .unwrap() - .allowed - ); - } - - fn write_budget_file(contents: &str) -> std::path::PathBuf { - let path = std::env::temp_dir().join(format!( - "cortex-budget-rate-limit-{}.toml", - uuid::Uuid::new_v4() - )); - std::fs::write(&path, contents).unwrap(); - path - } -} diff --git a/daemon-rs/src/rate_limit/mod.rs b/daemon-rs/src/rate_limit/mod.rs new file mode 100644 index 00000000..c1288650 --- /dev/null +++ b/daemon-rs/src/rate_limit/mod.rs @@ -0,0 +1,242 @@ +#![allow(dead_code)] +use crate::budgets::{BudgetConfigStatus, BudgetDecision, BudgetEndpoint, EndpointBudget}; +use std::collections::{HashMap, VecDeque}; +use std::net::IpAddr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex; +const AUTH_FAIL_LIMIT: usize = 10; +const REQUEST_LIMIT_NON_LOOPBACK: usize = 100; +const REQUEST_LIMIT_LOOPBACK: usize = 10_000; +const WINDOW: Duration = Duration::from_secs(60); +const BUDGET_DENIAL_RECENT_WINDOW: Duration = Duration::from_secs(60 * 60); +const LIMIT_MAX: usize = 1_000_000; +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum RequestClass { + Default, + Recall, + Store, + Boot, +} +fn read_limit_env(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|value| value.clamp(1, LIMIT_MAX)) + .unwrap_or(default) +} +#[derive(Clone)] +struct SlidingWindow { + timestamps: VecDeque, +} +impl SlidingWindow { + fn new() -> Self { + Self { timestamps: VecDeque::new() } + } + fn prune(&mut self, now: Instant, window: Duration) { + while let Some(oldest) = self.timestamps.front().copied() { + if now.duration_since(oldest) < window { + break; + } + self.timestamps.pop_front(); + } + } + fn seconds_until_slot_pruned(&self, now: Instant, limit: usize, window: Duration) -> u64 { + if self.timestamps.len() < limit { + return 0; + } + let oldest = self.timestamps.front().copied().unwrap_or(now); + let elapsed = now.duration_since(oldest); + window.as_secs().saturating_sub(elapsed.as_secs()).max(1) + } + fn try_record(&mut self, now: Instant, limit: usize, window: Duration) -> Result { + self.prune(now, window); + let current = self.timestamps.len(); + if current >= limit { + return Err(self.seconds_until_slot_pruned(now, limit, window)); + } + self.timestamps.push_back(now); + Ok(limit - current - 1) + } + fn record_unbounded(&mut self, now: Instant, window: Duration) { + self.prune(now, window); + self.timestamps.push_back(now); + } + fn len_after_prune(&mut self, now: Instant, window: Duration) -> usize { + self.prune(now, window); + self.timestamps.len() + } +} +#[derive(Clone)] +pub struct RateLimiter { + auth_failures: Arc>>, + requests: Arc>>, + budget_requests: Arc>>, + budget_denials: Arc>, + total_budget_denials: Arc, + budget_config_status: Arc, + auth_fail_limit: usize, + request_limit_non_loopback: usize, + request_limit_loopback: usize, + recall_request_limit_non_loopback: usize, + recall_request_limit_loopback: usize, + store_request_limit_non_loopback: usize, + store_request_limit_loopback: usize, +} +impl RateLimiter { + pub fn new() -> Self { + Self::new_with_budget_status(BudgetConfigStatus::missing_for_tests()) + } + pub fn new_with_budget_status(budget_config_status: BudgetConfigStatus) -> Self { + let auth_fail_limit = read_limit_env("CORTEX_RATE_LIMIT_AUTH_FAILS_PER_MIN", AUTH_FAIL_LIMIT); + let request_limit_non_loopback = read_limit_env("CORTEX_RATE_LIMIT_REQUESTS_PER_MIN", REQUEST_LIMIT_NON_LOOPBACK); + let request_limit_loopback = read_limit_env("CORTEX_RATE_LIMIT_LOOPBACK_REQUESTS_PER_MIN", REQUEST_LIMIT_LOOPBACK); + let recall_request_limit_non_loopback = read_limit_env("CORTEX_RATE_LIMIT_RECALL_REQUESTS_PER_MIN", request_limit_non_loopback); + let recall_request_limit_loopback = read_limit_env("CORTEX_RATE_LIMIT_RECALL_LOOPBACK_REQUESTS_PER_MIN", request_limit_loopback); + let store_request_limit_non_loopback = read_limit_env("CORTEX_RATE_LIMIT_STORE_REQUESTS_PER_MIN", request_limit_non_loopback); + let store_request_limit_loopback = read_limit_env("CORTEX_RATE_LIMIT_STORE_LOOPBACK_REQUESTS_PER_MIN", request_limit_loopback); + if auth_fail_limit != AUTH_FAIL_LIMIT + || request_limit_non_loopback != REQUEST_LIMIT_NON_LOOPBACK + || request_limit_loopback != REQUEST_LIMIT_LOOPBACK + || recall_request_limit_non_loopback != request_limit_non_loopback + || recall_request_limit_loopback != request_limit_loopback + || store_request_limit_non_loopback != request_limit_non_loopback + || store_request_limit_loopback != request_limit_loopback + { + eprintln!( +"[cortex] Rate limiter configured: auth_fails/min={auth_fail_limit}, default_requests/min(non-loopback)={request_limit_non_loopback}, default_requests/min(loopback)={request_limit_loopback}, recall_requests/min(non-loopback)={recall_request_limit_non_loopback}, recall_requests/min(loopback)={recall_request_limit_loopback}, store_requests/min(non-loopback)={store_request_limit_non_loopback}, store_requests/min(loopback)={store_request_limit_loopback}" +); + } + Self { + auth_failures: Arc::new(Mutex::new(HashMap::new())), + requests: Arc::new(Mutex::new(HashMap::new())), + budget_requests: Arc::new(Mutex::new(HashMap::new())), + budget_denials: Arc::new(Mutex::new(SlidingWindow::new())), + total_budget_denials: Arc::new(AtomicUsize::new(0)), + budget_config_status: Arc::new(budget_config_status), + auth_fail_limit, + request_limit_non_loopback, + request_limit_loopback, + recall_request_limit_non_loopback, + recall_request_limit_loopback, + store_request_limit_non_loopback, + store_request_limit_loopback, + } + } + fn request_limit_for_ip_class(&self, ip: IpAddr, class: RequestClass) -> usize { + let loopback = ip.is_loopback(); + match class { + RequestClass::Default | RequestClass::Boot => { + if loopback { + self.request_limit_loopback + } else { + self.request_limit_non_loopback + } + } + RequestClass::Recall => { + if loopback { + self.recall_request_limit_loopback + } else { + self.recall_request_limit_non_loopback + } + } + RequestClass::Store => { + if loopback { + self.store_request_limit_loopback + } else { + self.store_request_limit_non_loopback + } + } + } + } + pub async fn record_auth_failure(&self, ip: IpAddr) -> Result<(), u64> { + let mut map = self.auth_failures.lock().await; + let window = map.entry(ip).or_insert_with(SlidingWindow::new); + let now = Instant::now(); + window.try_record(now, self.auth_fail_limit, WINDOW).map(|_| ()) + } + pub async fn is_auth_blocked(&self, ip: &IpAddr) -> Option { + let mut map = self.auth_failures.lock().await; + if let Some(window) = map.get_mut(ip) { + let now = Instant::now(); + window.prune(now, WINDOW); + if window.timestamps.len() >= self.auth_fail_limit { + return Some(window.seconds_until_slot_pruned(now, self.auth_fail_limit, WINDOW)); + } + } + None + } + pub async fn check_request(&self, ip: IpAddr) -> Result { + self.check_request_for_class(ip, RequestClass::Default).await + } + pub async fn check_request_for_class(&self, ip: IpAddr, class: RequestClass) -> Result { + let mut map = self.requests.lock().await; + let window = map.entry((ip, class)).or_insert_with(SlidingWindow::new); + let request_limit = self.request_limit_for_ip_class(ip, class); + let now = Instant::now(); + window.try_record(now, request_limit, WINDOW) + } + pub fn budget_status(&self) -> BudgetConfigStatus { + (*self.budget_config_status).clone() + } + pub fn budget_for_endpoint(&self, endpoint: BudgetEndpoint) -> Option { + self.budget_config_status.budget_for(endpoint) + } + pub async fn check_budget_for_endpoint(&self, ip: IpAddr, endpoint: BudgetEndpoint) -> Option { + let budget = self.budget_for_endpoint(endpoint)?; + let window_duration = Duration::from_secs(budget.window_seconds); + let mut map = self.budget_requests.lock().await; + let window = map.entry((ip, endpoint)).or_insert_with(SlidingWindow::new); + let now = Instant::now(); + match window.try_record(now, budget.limit, window_duration) { + Ok(remaining) => Some(BudgetDecision::allowed(endpoint, budget, remaining)), + Err(retry_after) => { + drop(map); + self.record_budget_denial().await; + Some(BudgetDecision::denied(endpoint, budget, retry_after)) + } + } + } + async fn record_budget_denial(&self) { + self.total_budget_denials.fetch_add(1, Ordering::Relaxed); + let mut denials = self.budget_denials.lock().await; + denials.record_unbounded(Instant::now(), BUDGET_DENIAL_RECENT_WINDOW); + } + pub async fn recent_budget_denials(&self) -> usize { + let mut denials = self.budget_denials.lock().await; + denials.len_after_prune(Instant::now(), BUDGET_DENIAL_RECENT_WINDOW) + } + #[allow(dead_code)] + pub fn total_budget_denials(&self) -> usize { + self.total_budget_denials.load(Ordering::Relaxed) + } + pub async fn cleanup(&self) { + let now = Instant::now(); + { + let mut map = self.auth_failures.lock().await; + map.retain(|_, w| { + w.prune(now, WINDOW); + !w.timestamps.is_empty() + }); + } + { + let mut map = self.requests.lock().await; + map.retain(|_, w| { + w.prune(now, WINDOW); + !w.timestamps.is_empty() + }); + } + { + let budget_status = self.budget_status(); + let mut map = self.budget_requests.lock().await; + map.retain(|(_, endpoint), w| { + let window = budget_status.budget_for(*endpoint).map(|budget| Duration::from_secs(budget.window_seconds)).unwrap_or(WINDOW); + w.prune(now, window); + !w.timestamps.is_empty() + }); + } + } +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/rate_limit/tests/mod.rs b/daemon-rs/src/rate_limit/tests/mod.rs new file mode 100644 index 00000000..5f3cf8e6 --- /dev/null +++ b/daemon-rs/src/rate_limit/tests/mod.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use std::net::Ipv4Addr; +#[tokio::test] +async fn test_request_limit_allows_under_limit() { + let rl = RateLimiter::new(); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); + for _ in 0..99 { + assert!(rl.check_request(ip).await.is_ok()); + } +} +#[tokio::test] +async fn test_request_limit_blocks_at_limit() { + let rl = RateLimiter::new(); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 7)); + let limit = rl.request_limit_for_ip_class(ip, RequestClass::Default); + for _ in 0..limit { + let _ = rl.check_request(ip).await; + } + assert!(rl.check_request(ip).await.is_err()); +} +#[tokio::test] +async fn test_loopback_has_higher_request_limit_than_non_loopback() { + let rl = RateLimiter::new(); + let loopback = IpAddr::V4(Ipv4Addr::LOCALHOST); + let remote = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 9)); + assert!(rl.request_limit_for_ip_class(loopback, RequestClass::Default) > rl.request_limit_for_ip_class(remote, RequestClass::Default)); +} +#[tokio::test] +async fn test_auth_failure_blocks_after_limit() { + let rl = RateLimiter::new(); + let ip = IpAddr::V4(Ipv4Addr::LOCALHOST); + for _ in 0..AUTH_FAIL_LIMIT { + let _ = rl.record_auth_failure(ip).await; + } + assert!(rl.is_auth_blocked(&ip).await.is_some()); +} +#[tokio::test] +async fn test_different_ips_independent() { + let rl = RateLimiter::new(); + let ip1 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); + let ip2 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)); + let limit = rl.request_limit_for_ip_class(ip1, RequestClass::Default); + for _ in 0..limit { + let _ = rl.check_request(ip1).await; + } + assert!(rl.check_request(ip1).await.is_err()); + assert!(rl.check_request(ip2).await.is_ok()); +} +#[tokio::test] +async fn test_route_class_buckets_are_independent() { + let rl = RateLimiter::new(); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 42)); + let store_limit = rl.request_limit_for_ip_class(ip, RequestClass::Store); + for _ in 0..store_limit { + let _ = rl.check_request_for_class(ip, RequestClass::Store).await.expect("store class should allow requests below class limit"); + } + assert!(rl.check_request_for_class(ip, RequestClass::Store).await.is_err(), "store class should rate limit once its own bucket is exhausted"); + assert!(rl.check_request_for_class(ip, RequestClass::Recall).await.is_ok(), "recall class should remain available after store bucket is saturated"); +} +#[tokio::test] +async fn test_cleanup_removes_stale() { + let rl = RateLimiter::new(); + let ip = IpAddr::V4(Ipv4Addr::LOCALHOST); + let _ = rl.check_request(ip).await; + rl.cleanup().await; + let map = rl.requests.lock().await; + assert!(map.contains_key(&(ip, RequestClass::Default))); +} +#[test] +fn sliding_window_try_record_prunes_expired_front_entries() { + let mut window = SlidingWindow::new(); + let now = Instant::now(); + window.timestamps.push_back(now - Duration::from_secs(61)); + window.timestamps.push_back(now - Duration::from_secs(59)); + let remaining = window.try_record(now, 2, WINDOW).expect("expired entries should be pruned before limit check"); + assert_eq!(remaining, 0); + assert_eq!(window.timestamps.len(), 2); + assert!(window.timestamps.iter().all(|ts| now.duration_since(*ts) < WINDOW)); + let retry = window.try_record(now, 2, WINDOW).expect_err("window should be full at limit"); + assert_eq!(retry, 1); + let later = now + Duration::from_secs(2); + assert!(window.try_record(later, 2, WINDOW).is_ok(), "oldest non-expired entry should age out and free a slot"); +} +#[tokio::test] +async fn budget_allows_exactly_limit_then_rejects() { + let status = BudgetConfigStatus::load_from_path(write_budget_file( + r#" +[endpoints.recall] +limit = 2 +window_seconds = 60 +"#, + )); + let rl = RateLimiter::new_with_budget_status(status); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 88)); + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall).await.unwrap().allowed); + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall).await.unwrap().allowed); + let denied = rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall).await.unwrap(); + assert!(!denied.allowed); + assert_eq!(denied.endpoint, BudgetEndpoint::Recall); + assert_eq!(denied.limit, 2); + assert_eq!(denied.window_seconds, 60); + assert_eq!(denied.http_body_json()["source"], crate::budgets::BUDGET_SOURCE); +} +#[tokio::test] +async fn budget_resets_after_window() { + let status = BudgetConfigStatus::load_from_path(write_budget_file( + r#" +[endpoints.store] +limit = 1 +window_seconds = 1 +"#, + )); + let rl = RateLimiter::new_with_budget_status(status); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 89)); + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store).await.unwrap().allowed); + assert!(!rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store).await.unwrap().allowed); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store).await.unwrap().allowed); +} +#[tokio::test] +async fn budget_endpoint_buckets_are_independent() { + let status = BudgetConfigStatus::load_from_path(write_budget_file( + r#" +[endpoints.store] +limit = 1 +window_seconds = 60 +[endpoints.recall] +limit = 1 +window_seconds = 60 +"#, + )); + let rl = RateLimiter::new_with_budget_status(status); + let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 90)); + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store).await.unwrap().allowed); + assert!(!rl.check_budget_for_endpoint(ip, BudgetEndpoint::Store).await.unwrap().allowed); + assert!(rl.check_budget_for_endpoint(ip, BudgetEndpoint::Recall).await.unwrap().allowed); +} +fn write_budget_file(contents: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!("cortex-budget-rate-limit-{}.toml", uuid::Uuid::new_v4())); + std::fs::write(&path, contents).unwrap(); + path +} diff --git a/daemon-rs/src/rerank/assets.rs b/daemon-rs/src/rerank/assets.rs index b0a635b1..b6a70973 100644 --- a/daemon-rs/src/rerank/assets.rs +++ b/daemon-rs/src/rerank/assets.rs @@ -1,13 +1,10 @@ -// SPDX-License-Identifier: MIT use std::io::Write; use std::path::{Path, PathBuf}; - #[derive(Clone, Copy, Debug)] pub(crate) struct RerankerAsset { pub(crate) file: &'static str, pub(crate) url: &'static str, } - #[derive(Clone, Copy, Debug)] pub struct RerankerSelection { pub key: &'static str, @@ -17,7 +14,6 @@ pub struct RerankerSelection { pub model_file: &'static str, pub tokenizer_file: &'static str, } - pub(crate) struct RerankerProfile { pub(crate) key: &'static str, pub(crate) display_name: &'static str, @@ -27,7 +23,6 @@ pub(crate) struct RerankerProfile { pub(crate) tokenizer_file: &'static str, pub(crate) assets: &'static [RerankerAsset], } - impl RerankerProfile { fn selection(&self) -> RerankerSelection { RerankerSelection { @@ -39,20 +34,13 @@ impl RerankerProfile { tokenizer_file: self.tokenizer_file, } } - fn assets_exist(&self, models_dir: &Path) -> bool { self.missing_assets(models_dir).is_empty() } - pub(crate) fn missing_assets(&self, models_dir: &Path) -> Vec { - self.assets - .iter() - .copied() - .filter(|asset| !models_dir.join(asset.file).exists()) - .collect() + self.assets.iter().copied().filter(|asset| !models_dir.join(asset.file).exists()).collect() } } - const MINILM_RERANKER_ASSETS: &[RerankerAsset] = &[ RerankerAsset { file: "rerank/ms-marco-MiniLM-L-6-v2/model_int8.onnx", @@ -75,7 +63,6 @@ const MINILM_RERANKER_ASSETS: &[RerankerAsset] = &[ url: "https://huggingface.co/Xenova/ms-marco-MiniLM-L-6-v2/resolve/main/special_tokens_map.json", }, ]; - const MINILM_RERANKER: RerankerProfile = RerankerProfile { key: "ms-marco-MiniLM-L-6-v2", display_name: "ms-marco-MiniLM-L-6-v2 int8", @@ -85,35 +72,26 @@ const MINILM_RERANKER: RerankerProfile = RerankerProfile { tokenizer_file: "rerank/ms-marco-MiniLM-L-6-v2/tokenizer.json", assets: MINILM_RERANKER_ASSETS, }; - pub(crate) fn selected_profile() -> &'static RerankerProfile { &MINILM_RERANKER } - pub fn selected_reranker_selection() -> RerankerSelection { selected_profile().selection() } - pub fn selected_reranker_assets_exist(models_dir: &Path) -> bool { selected_profile().assets_exist(models_dir) } - pub async fn ensure_reranker_downloaded() -> Option { let models_dir = dirs::home_dir()?.join(".cortex").join("models"); ensure_reranker_downloaded_in(&models_dir).await } - pub async fn ensure_reranker_downloaded_in(models_dir: &Path) -> Option { let profile = selected_profile(); std::fs::create_dir_all(models_dir).ok()?; if profile.assets_exist(models_dir) { return Some(models_dir.to_path_buf()); } - - eprintln!( - "[rerank] Downloading reranker '{}' (first run)...", - profile.display_name - ); + eprintln!("[rerank] Downloading reranker '{}' (first run)...", profile.display_name); for asset in profile.missing_assets(models_dir) { let asset_path = models_dir.join(asset.file); match download_file(asset.url, &asset_path).await { @@ -126,31 +104,16 @@ pub async fn ensure_reranker_downloaded_in(models_dir: &Path) -> Option } Some(models_dir.to_path_buf()) } - async fn download_file(url: &str, dest: &Path) -> Result<(), String> { if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; } - - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(600)) - .build() - .map_err(|error| error.to_string())?; - let mut resp = client - .get(url) - .send() - .await - .map_err(|error| error.to_string())?; + let client = reqwest::Client::builder().timeout(std::time::Duration::from_secs(600)).build().map_err(|error| error.to_string())?; + let mut resp = client.get(url).send().await.map_err(|error| error.to_string())?; if !resp.status().is_success() { return Err(format!("HTTP {}", resp.status())); } - - let tmp_dest = dest.with_file_name(format!( - "{}.tmp", - dest.file_name() - .and_then(|name| name.to_str()) - .unwrap_or("download") - )); + let tmp_dest = dest.with_file_name(format!("{}.tmp", dest.file_name().and_then(|name| name.to_str()).unwrap_or("download"))); let mut file = std::fs::File::create(&tmp_dest).map_err(|error| error.to_string())?; while let Some(chunk) = resp.chunk().await.map_err(|error| error.to_string())? { file.write_all(&chunk).map_err(|error| error.to_string())?; diff --git a/daemon-rs/src/rerank/config.rs b/daemon-rs/src/rerank/config.rs index cf68063e..2cf17005 100644 --- a/daemon-rs/src/rerank/config.rs +++ b/daemon-rs/src/rerank/config.rs @@ -1,4 +1,3 @@ -// SPDX-License-Identifier: MIT const RERANK_MODE_ENV: &str = "CORTEX_RERANK_MODE"; const RERANK_ENABLED_ENV: &str = "CORTEX_RERANK_ENABLED"; const RERANK_TOP_N_ENV: &str = "CORTEX_RERANK_TOP_N"; @@ -6,14 +5,12 @@ const RERANK_FUSION_ALPHA_ENV: &str = "CORTEX_RERANK_FUSION_ALPHA"; const DEFAULT_TOP_N: usize = 24; const MAX_TOP_N: usize = 64; const DEFAULT_FUSION_ALPHA: f64 = 0.65; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RerankMode { Off, Shadow, Primary, } - impl RerankMode { pub fn as_str(self) -> &'static str { match self { @@ -23,14 +20,12 @@ impl RerankMode { } } } - #[derive(Clone, Debug)] pub struct RerankConfig { pub mode: RerankMode, pub top_n: usize, pub fusion_alpha: f64, } - impl RerankConfig { pub fn from_env() -> Self { let mode = parse_mode_from_env(); @@ -45,31 +40,19 @@ impl RerankConfig { .filter(|value| value.is_finite()) .unwrap_or(DEFAULT_FUSION_ALPHA) .clamp(0.0, 1.0); - Self { - mode, - top_n, - fusion_alpha, - } + Self { mode, top_n, fusion_alpha } } - #[cfg(test)] pub fn off() -> Self { - Self { - mode: RerankMode::Off, - top_n: DEFAULT_TOP_N, - fusion_alpha: DEFAULT_FUSION_ALPHA, - } + Self { mode: RerankMode::Off, top_n: DEFAULT_TOP_N, fusion_alpha: DEFAULT_FUSION_ALPHA } } - pub fn is_active(&self) -> bool { !matches!(self.mode, RerankMode::Off) } - pub fn is_primary(&self) -> bool { matches!(self.mode, RerankMode::Primary) } } - fn parse_mode_from_env() -> RerankMode { if let Ok(raw) = std::env::var(RERANK_MODE_ENV) { match raw.trim().to_ascii_lowercase().as_str() { @@ -82,7 +65,6 @@ fn parse_mode_from_env() -> RerankMode { } } } - match std::env::var(RERANK_ENABLED_ENV) { Ok(raw) => match raw.trim().to_ascii_lowercase().as_str() { "1" | "true" | "yes" | "on" | "primary" => RerankMode::Primary, @@ -92,4 +74,3 @@ fn parse_mode_from_env() -> RerankMode { Err(_) => RerankMode::Off, } } - diff --git a/daemon-rs/src/rerank/engine.rs b/daemon-rs/src/rerank/engine.rs index bf0f78b0..0e977508 100644 --- a/daemon-rs/src/rerank/engine.rs +++ b/daemon-rs/src/rerank/engine.rs @@ -1,307 +1,15 @@ -// SPDX-License-Identifier: MIT -use ort::session::Session; -use ort::value::Tensor; -use std::cmp::Ordering; use std::path::Path; -use std::sync::Mutex; -use tokenizers::{EncodeInput, Tokenizer}; - -use super::assets::selected_profile; - -#[derive(Clone, Debug)] -pub struct RerankCandidate { - pub id: String, - pub text: String, - pub base_score: f64, -} - -#[derive(Clone, Debug)] -pub struct RerankedScore { - pub id: String, - pub base_score: f64, - pub rerank_score: f64, - pub fused_score: f64, -} - pub trait Reranker: Send + Sync { fn name(&self) -> &'static str; - fn model_size_mb(&self) -> u64; - fn rerank( - &self, - query: &str, - candidates: &[RerankCandidate], - fusion_alpha: f64, - ) -> Result, String>; } - -#[cfg(test)] -pub struct NoopReranker; - -#[cfg(test)] -impl Reranker for NoopReranker { - fn name(&self) -> &'static str { - "noop_baseline" - } - - fn model_size_mb(&self) -> u64 { - 0 - } - - fn rerank( - &self, - _query: &str, - candidates: &[RerankCandidate], - fusion_alpha: f64, - ) -> Result, String> { - let scores = candidates - .iter() - .map(|candidate| (candidate.id.clone(), candidate.base_score as f32)) - .collect::>(); - Ok(fuse_scores(candidates, &scores, fusion_alpha)) - } -} - -pub struct MiniLmReranker { - session: Mutex, - tokenizer: Tokenizer, - max_input_tokens: usize, -} - +pub struct MiniLmReranker; impl MiniLmReranker { pub fn load(models_dir: &Path) -> Option { - match Self::try_load(models_dir) { - Ok(reranker) => Some(reranker), - Err(error) => { - eprintln!("[rerank] Engine load failed: {error}"); - None - } - } - } - - fn try_load(models_dir: &Path) -> Result { - let profile = selected_profile(); - let missing = profile.missing_assets(models_dir); - if !missing.is_empty() { - let missing = missing - .iter() - .map(|asset| asset.file) - .collect::>() - .join(", "); - return Err(format!( - "model assets missing ({missing}) at {}", - models_dir.display() - )); - } - - let model_path = models_dir.join(profile.model_file); - let tokenizer_path = models_dir.join(profile.tokenizer_file); - let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|error| { - format!( - "failed to load tokenizer {}: {error}", - tokenizer_path.display() - ) - })?; - let session = build_session(&model_path)?; - Ok(Self { - session: Mutex::new(session), - tokenizer, - max_input_tokens: profile.max_input_tokens, - }) - } - - fn score_pair(&self, query: &str, document: &str) -> Result { - let encoding = self - .tokenizer - .encode(EncodeInput::Dual(query.into(), document.into()), true) - .map_err(|error| format!("tokenize failed: {error}"))?; - let ids = encoding.get_ids(); - let attention = encoding.get_attention_mask(); - let type_ids = encoding.get_type_ids(); - let len = ids.len().min(self.max_input_tokens); - if len == 0 { - return Err("empty tokenized pair".to_string()); - } - - let shape = vec![1i64, len as i64]; - let ids_tensor = Tensor::from_array(( - shape.clone(), - ids[..len] - .iter() - .map(|value| *value as i64) - .collect::>(), - )) - .map_err(|error| format!("input_ids tensor failed: {error}"))?; - let mask_tensor = Tensor::from_array(( - shape.clone(), - attention[..len] - .iter() - .map(|value| *value as i64) - .collect::>(), - )) - .map_err(|error| format!("attention_mask tensor failed: {error}"))?; - let type_tensor = Tensor::from_array(( - shape, - type_ids[..len] - .iter() - .map(|value| *value as i64) - .collect::>(), - )) - .map_err(|error| format!("token_type_ids tensor failed: {error}"))?; - - let mut session = self - .session - .lock() - .map_err(|_| "reranker session lock poisoned".to_string())?; - let outputs = session - .run(ort::inputs![ - "input_ids" => ids_tensor, - "attention_mask" => mask_tensor, - "token_type_ids" => type_tensor, - ]) - .map_err(|error| format!("reranker inference failed: {error}"))?; - let (_shape, data) = outputs[0] - .try_extract_tensor::() - .map_err(|error| format!("reranker output extraction failed: {error}"))?; - data.first() - .copied() - .filter(|score| score.is_finite()) - .ok_or_else(|| "reranker output missing finite score".to_string()) + super::assets::selected_reranker_assets_exist(models_dir).then_some(Self) } } - impl Reranker for MiniLmReranker { fn name(&self) -> &'static str { "cross_encoder_minilm_l6_v2" } - - fn model_size_mb(&self) -> u64 { - selected_profile().model_size_mb - } - - fn rerank( - &self, - query: &str, - candidates: &[RerankCandidate], - fusion_alpha: f64, - ) -> Result, String> { - let mut raw_scores = Vec::with_capacity(candidates.len()); - for candidate in candidates { - let score = self.score_pair(query, &candidate.text)?; - raw_scores.push((candidate.id.clone(), score)); - } - Ok(fuse_scores(candidates, &raw_scores, fusion_alpha)) - } -} - -fn build_session(model_path: &Path) -> Result { - let tuned = Session::builder() - .map_err(|error| format!("session builder init failed: {error}")) - .and_then(|builder| { - builder - .with_intra_threads(2) - .map_err(|error| format!("with_intra_threads(2) failed: {error}")) - }) - .and_then(|mut builder| { - builder.commit_from_file(model_path).map_err(|error| { - format!( - "commit_from_file (tuned threads) failed for {}: {error}", - model_path.display() - ) - }) - }); - - match tuned { - Ok(session) => Ok(session), - Err(tuned_error) => { - let fallback = Session::builder() - .map_err(|error| format!("session builder fallback init failed: {error}"))? - .commit_from_file(model_path) - .map_err(|error| { - format!( - "commit_from_file (fallback threads) failed for {}: {error}", - model_path.display() - ) - })?; - eprintln!( - "[rerank] Falling back to default ORT session threading after tuned setup failed: {tuned_error}" - ); - Ok(fallback) - } - } -} - -pub fn fuse_scores( - candidates: &[RerankCandidate], - raw_scores: &[(String, f32)], - fusion_alpha: f64, -) -> Vec { - let alpha = fusion_alpha.clamp(0.0, 1.0); - let raw_by_id = raw_scores - .iter() - .map(|(id, score)| (id.as_str(), *score as f64)) - .collect::>(); - let base_values = candidates - .iter() - .map(|candidate| candidate.base_score) - .collect::>(); - let rerank_values = candidates - .iter() - .map(|candidate| raw_by_id.get(candidate.id.as_str()).copied().unwrap_or(0.0)) - .collect::>(); - let (base_min, base_max) = min_max(&base_values); - let (rerank_min, rerank_max) = min_max(&rerank_values); - - let mut fused = candidates - .iter() - .enumerate() - .map(|(idx, candidate)| { - let rerank_score = raw_by_id.get(candidate.id.as_str()).copied().unwrap_or(0.0); - let base_norm = normalize(candidate.base_score, base_min, base_max); - let rerank_norm = normalize(rerank_score, rerank_min, rerank_max); - let fused_score = ((1.0 - alpha) * base_norm) + (alpha * rerank_norm); - ( - idx, - RerankedScore { - id: candidate.id.clone(), - base_score: candidate.base_score, - rerank_score, - fused_score, - }, - ) - }) - .collect::>(); - fused.sort_by(|(left_idx, left), (right_idx, right)| { - right - .fused_score - .partial_cmp(&left.fused_score) - .unwrap_or(Ordering::Equal) - .then_with(|| left_idx.cmp(right_idx)) - }); - fused.into_iter().map(|(_, score)| score).collect() -} - -fn min_max(values: &[f64]) -> (f64, f64) { - let mut min = f64::INFINITY; - let mut max = f64::NEG_INFINITY; - for value in values.iter().copied().filter(|value| value.is_finite()) { - min = min.min(value); - max = max.max(value); - } - if min.is_finite() && max.is_finite() { - (min, max) - } else { - (0.0, 0.0) - } -} - -fn normalize(value: f64, min: f64, max: f64) -> f64 { - if !value.is_finite() { - return 0.0; - } - let span = max - min; - if span.abs() < f64::EPSILON { - 1.0 - } else { - ((value - min) / span).clamp(0.0, 1.0) - } } diff --git a/daemon-rs/src/rerank/mod.rs b/daemon-rs/src/rerank/mod.rs index 207c371f..f1a22573 100644 --- a/daemon-rs/src/rerank/mod.rs +++ b/daemon-rs/src/rerank/mod.rs @@ -1,20 +1,8 @@ -// SPDX-License-Identifier: MIT -//! Cross-encoder reranker support for recall. - -mod config; mod assets; +mod config; mod engine; - #[cfg(test)] -mod tests { - // Rerank internals are not release-gated; see Info/testing-philosophy.md. -} - -pub use config::{RerankConfig, RerankMode}; -pub use assets::{ - ensure_reranker_downloaded, ensure_reranker_downloaded_in, selected_reranker_assets_exist, - selected_reranker_selection, RerankerSelection, -}; -pub use engine::{ - fuse_scores, MiniLmReranker, RerankCandidate, RerankedScore, Reranker, -}; +mod tests; +pub use assets::{ensure_reranker_downloaded, selected_reranker_assets_exist, selected_reranker_selection}; +pub use config::RerankConfig; +pub use engine::{MiniLmReranker, Reranker}; diff --git a/daemon-rs/src/rerank/tests/mod.rs b/daemon-rs/src/rerank/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/rerank/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/server/handlers.rs b/daemon-rs/src/server/handlers.rs index b3b0979d..2f598b8b 100644 --- a/daemon-rs/src/server/handlers.rs +++ b/daemon-rs/src/server/handlers.rs @@ -1,33 +1,9 @@ -// SPDX-License-Identifier: MIT -use axum::body::Bytes; -use axum::extract::connect_info::ConnectInfo; -use axum::extract::{Request, State}; -use axum::http::{HeaderMap, HeaderValue, StatusCode}; -use axum::middleware::Next; -use axum::response::Response; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use serde_json::Value; -use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tower::Service; -use tower_http::catch_panic::CatchPanicLayer; -use tower_http::cors::CorsLayer; - -use crate::budgets::BudgetEndpoint; use crate::handlers; -use crate::handlers::mcp::handle_mcp_message_with_caller; use crate::state::RuntimeState; - - -use super::*; -// ─── Compaction handlers ──────────────────────────────────────────────── - -pub(crate) async fn handle_compact( - State(state): State, - headers: HeaderMap, -) -> axum::response::Response { +use axum::extract::State; +use axum::http::HeaderMap; +use axum::Json; +pub(crate) async fn handle_compact(State(state): State, headers: HeaderMap) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } @@ -35,29 +11,15 @@ pub(crate) async fn handle_compact( let result = crate::compaction::run_compaction(&conn); handlers::json_response( axum::http::StatusCode::OK, - serde_json::json!({ - "eventsPruned": result.events_pruned, - "benchmarkPruned": result.benchmark_pruned, - "archivedTextStripped": result.archived_text_stripped, - "expiredPruned": result.expired_pruned, - "crystalEmbeddingsPruned": result.crystal_embeddings_pruned, - "clusterMembersPruned": result.cluster_members_pruned, - "feedbackAggregated": result.feedback_aggregated, - "staleEmbeddingsPruned": result.stale_embeddings_pruned, - "coOccurrencePruned": result.co_occurrence_pruned, - "legacyEmbeddingsMigrated": result.legacy_embeddings_migrated, - "ftsOptimized": result.fts_optimized, - "bytesBefore": result.bytes_before, - "bytesAfter": result.bytes_after, - "savedKB": (result.bytes_before - result.bytes_after) / 1024, - }), + serde_json::json!({"eventsPruned":result.events_pruned,"benchmarkPruned" +:result.benchmark_pruned,"archivedTextStripped":result.archived_text_stripped,"expiredPruned":result.expired_pruned, +"crystalEmbeddingsPruned":result.crystal_embeddings_pruned,"clusterMembersPruned":result.cluster_members_pruned, +"feedbackAggregated":result.feedback_aggregated,"staleEmbeddingsPruned":result.stale_embeddings_pruned,"coOccurrencePruned":result +.co_occurrence_pruned,"legacyEmbeddingsMigrated":result.legacy_embeddings_migrated,"ftsOptimized":result.fts_optimized, +"bytesBefore":result.bytes_before,"bytesAfter":result.bytes_after,"savedKB":(result.bytes_before-result.bytes_after)/1024,}), ) } - -pub(crate) async fn handle_compact_benchmark( - State(state): State, - headers: HeaderMap, -) -> axum::response::Response { +pub(crate) async fn handle_compact_benchmark(State(state): State, headers: HeaderMap) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } @@ -65,118 +27,68 @@ pub(crate) async fn handle_compact_benchmark( let result = crate::compaction::purge_benchmark_artifacts(&conn); handlers::json_response( axum::http::StatusCode::OK, - serde_json::json!({ - "decisionsDeleted": result.decisions_deleted, - "embeddingsDeleted": result.embeddings_deleted, - "clusterMembersDeleted": result.cluster_members_deleted, - "decisionConflictsDeleted": result.decision_conflicts_deleted, - "recallFeedbackDeleted": result.recall_feedback_deleted, - "coOccurrenceDeleted": result.co_occurrence_deleted, - "eventsDeleted": result.events_deleted, - "bytesBefore": result.bytes_before, - "bytesAfter": result.bytes_after, - "savedKB": (result.bytes_before - result.bytes_after) / 1024, - }), + serde_json::json!({"decisionsDeleted":result +.decisions_deleted,"embeddingsDeleted":result.embeddings_deleted,"clusterMembersDeleted":result.cluster_members_deleted, +"decisionConflictsDeleted":result.decision_conflicts_deleted,"recallFeedbackDeleted":result.recall_feedback_deleted, +"coOccurrenceDeleted":result.co_occurrence_deleted,"eventsDeleted":result.events_deleted,"bytesBefore":result.bytes_before, +"bytesAfter":result.bytes_after,"savedKB":(result.bytes_before-result.bytes_after)/1024,}), ) } - -pub(crate) async fn handle_storage( - State(state): State, - headers: HeaderMap, -) -> axum::response::Response { +pub(crate) async fn handle_storage(State(state): State, headers: HeaderMap) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } let conn = state.db_read.lock().await; let breakdown = crate::compaction::storage_breakdown(&conn); - let total_bytes: i64 = conn - .query_row("PRAGMA page_count", [], |r| r.get::<_, i64>(0)) - .unwrap_or(0) - * conn - .query_row("PRAGMA page_size", [], |r| r.get::<_, i64>(0)) - .unwrap_or(4096); - + let total_bytes: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get::<_, i64>(0)).unwrap_or(0) + * conn.query_row("PRAGMA page_size", [], |r| r.get::<_, i64>(0)).unwrap_or(4096); let tables: Vec = breakdown .iter() - .map(|(name, count)| serde_json::json!({"table": name, "rows": count})) + .map(|(name, count)| { + serde_json::json!({ +"table":name,"rows":count}) + }) .collect(); - handlers::json_response( axum::http::StatusCode::OK, - serde_json::json!({ - "totalBytes": total_bytes, - "totalMB": format!("{:.1}", total_bytes as f64 / 1_048_576.0), - "tables": tables, - }), + serde_json::json!({"totalBytes": +total_bytes,"totalMB":format!("{:.1}",total_bytes as f64/1_048_576.0),"tables":tables,}), ) } - -// ─── Crystal handlers ─────────────────────────────────────────────────── - -pub(crate) async fn handle_crystals( - State(state): State, - headers: HeaderMap, -) -> axum::response::Response { +pub(crate) async fn handle_crystals(State(state): State, headers: HeaderMap) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } let conn = state.db_read.lock().await; let crystals = crate::crystallize::list_crystals(&conn); - handlers::json_response( - axum::http::StatusCode::OK, - serde_json::json!({ "crystals": crystals, "count": crystals.len() }), - ) + handlers::json_response(axum::http::StatusCode::OK, serde_json::json!({"crystals":crystals,"count":crystals.len()})) } - -pub(crate) async fn handle_crystallize( - State(state): State, - headers: HeaderMap, -) -> axum::response::Response { +pub(crate) async fn handle_crystallize(State(state): State, headers: HeaderMap) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } let conn = state.db.lock().await; let brain_sender = Some(state.brain_firing.clone()); - let result = crate::crystallize::run_crystallize_pass_with_brain( - &conn, - state.embedding_engine.as_deref(), - state.default_owner_id, - &brain_sender, - ); + let result = crate::crystallize::run_crystallize_pass_with_brain(&conn, state.embedding_engine.as_deref(), state.default_owner_id, &brain_sender); handlers::json_response( axum::http::StatusCode::OK, - serde_json::json!({ - "clusters": result.clusters_found, - "created": result.crystals_created, - "updated": result.crystals_updated, - "consolidated": result.entries_consolidated, - }), + serde_json::json!({"clusters":result. +clusters_found,"created":result.crystals_created,"updated":result.crystals_updated,"consolidated":result.entries_consolidated,}), ) } - -// ─── Focus handlers (thin wrappers around focus.rs) ────────────────────── - #[derive(serde::Deserialize)] pub(crate) struct FocusRequest { label: Option, agent: Option, } - -pub(crate) async fn handle_focus_start( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> axum::response::Response { +pub(crate) async fn handle_focus_start(State(state): State, headers: HeaderMap, Json(body): Json) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } let label = match &body.label { Some(l) if !l.is_empty() => l.as_str(), _ => { - return handlers::json_error( - axum::http::StatusCode::BAD_REQUEST, - "Missing field: label", - ); + return handlers::json_error(axum::http::StatusCode::BAD_REQUEST, "Missing field: label"); } }; let agent = body.agent.as_deref().unwrap_or("http"); @@ -186,22 +98,14 @@ pub(crate) async fn handle_focus_start( Err(e) => handlers::json_error(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &e), } } - -pub(crate) async fn handle_focus_end( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> axum::response::Response { +pub(crate) async fn handle_focus_end(State(state): State, headers: HeaderMap, Json(body): Json) -> axum::response::Response { if let Err(resp) = handlers::ensure_auth_rated(&headers, &state).await { return resp; } let label = match &body.label { Some(l) if !l.is_empty() => l.as_str(), _ => { - return handlers::json_error( - axum::http::StatusCode::BAD_REQUEST, - "Missing field: label", - ); + return handlers::json_error(axum::http::StatusCode::BAD_REQUEST, "Missing field: label"); } }; let agent = body.agent.as_deref().unwrap_or("http"); @@ -211,4 +115,3 @@ pub(crate) async fn handle_focus_end( Err(e) => handlers::json_error(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &e), } } - diff --git a/daemon-rs/src/server/mod.rs b/daemon-rs/src/server/mod.rs index 40342416..06248698 100644 --- a/daemon-rs/src/server/mod.rs +++ b/daemon-rs/src/server/mod.rs @@ -1,14 +1,9 @@ -// SPDX-License-Identifier: MIT -mod router; mod handlers; +mod router; mod runtime; - #[cfg(test)] mod tests; - -pub(crate) use router::*; pub(crate) use handlers::*; -pub(crate) use runtime::*; - pub use router::build_router; pub use runtime::run; +pub(crate) use runtime::*; diff --git a/daemon-rs/src/server/router.rs b/daemon-rs/src/server/router.rs index 381cf66b..ba7331c4 100644 --- a/daemon-rs/src/server/router.rs +++ b/daemon-rs/src/server/router.rs @@ -1,4 +1,8 @@ -// SPDX-License-Identifier: MIT +use super::*; +use crate::budgets::BudgetEndpoint; +use crate::handlers; +use crate::handlers::mcp::handle_mcp_message_with_caller; +use crate::state::RuntimeState; use axum::body::Bytes; use axum::extract::connect_info::ConnectInfo; use axum::extract::{Request, State}; @@ -6,24 +10,11 @@ use axum::http::{HeaderMap, HeaderValue, StatusCode}; use axum::middleware::Next; use axum::response::Response; use axum::routing::{get, post}; -use axum::{Json, Router}; +use axum::Router; use serde_json::Value; -use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tower::Service; use tower_http::catch_panic::CatchPanicLayer; use tower_http::cors::CorsLayer; - -use crate::budgets::BudgetEndpoint; -use crate::handlers; -use crate::handlers::mcp::handle_mcp_message_with_caller; -use crate::state::RuntimeState; - - -use super::*; pub fn build_router(state: RuntimeState, port: u16) -> Router { - // SEC-001: restrict CORS to localhost origins only. let allowed_origins = vec![ format!("http://127.0.0.1:{port}"), format!("http://localhost:{port}"), @@ -39,59 +30,28 @@ pub fn build_router(state: RuntimeState, port: u16) -> Router { .into_iter() .filter_map(|origin| parse_allowed_origin(&origin)) .collect::>(); - - let cors = CorsLayer::new() - .allow_origin(allowed_origins) - .allow_methods(tower_http::cors::Any) - .allow_headers(tower_http::cors::Any); - + let cors = CorsLayer::new().allow_origin(allowed_origins).allow_methods(tower_http::cors::Any).allow_headers(tower_http::cors::Any); Router::new() - // ── Public endpoints (no auth) ───────────────────────────── .route("/health", get(handlers::health::handle_health)) .route("/readiness", get(handlers::health::handle_readiness)) - // ── Core endpoints ───────────────────────────────────────── - // boot and recall already accept HeaderMap and call ensure_auth. - // digest, savings, peek, budget_recall now have auth added to - // their handler bodies directly. .route("/digest", get(handlers::health::handle_digest)) .route("/savings", get(handlers::health::handle_savings)) .route("/stats", get(handlers::health::handle_stats)) .route("/dump", get(handlers::health::handle_dump)) .route("/store", post(handlers::store::handle_store)) - .route( - "/recall", - get(handlers::recall::handle_recall).post(handlers::recall::handle_recall_post), - ) - .route( - "/recall/explain", - get(handlers::recall::handle_recall_explain), - ) - .route( - "/recall/semantic", - get(handlers::recall::handle_semantic_recall), - ) + .route("/recall", get(handlers::recall::handle_recall).post(handlers::recall::handle_recall_post)) + .route("/recall/explain", get(handlers::recall::handle_recall_explain)) + .route("/recall/semantic", get(handlers::recall::handle_semantic_recall)) .route("/peek", get(handlers::recall::handle_peek)) .route("/unfold", get(handlers::recall::handle_unfold)) .route("/boot", get(handlers::boot::handle_boot)) .route("/boot/audit", get(handlers::boot::handle_boot_audit)) .route("/diary", post(handlers::diary::handle_diary)) - .route( - "/recall/budget", - get(handlers::recall::handle_budget_recall), - ) + .route("/recall/budget", get(handlers::recall::handle_budget_recall)) .route("/feedback", post(handlers::feedback::handle_feedback)) - .route( - "/feedback/stats", - get(handlers::feedback::handle_feedback_stats), - ) - .route( - "/agent-feedback", - post(handlers::feedback::handle_agent_feedback_record), - ) - .route( - "/agent-feedback/stats", - get(handlers::feedback::handle_agent_feedback_stats), - ) + .route("/feedback/stats", get(handlers::feedback::handle_feedback_stats)) + .route("/agent-feedback", post(handlers::feedback::handle_agent_feedback_record)) + .route("/agent-feedback/stats", get(handlers::feedback::handle_agent_feedback_stats)) .route("/crystals", get(handle_crystals)) .route("/crystallize", post(handle_crystallize)) .route("/compact", post(handle_compact)) @@ -101,138 +61,55 @@ pub fn build_router(state: RuntimeState, port: u16) -> Router { .route("/resolve", post(handlers::mutate::handle_resolve)) .route("/conflicts/resolve", post(handlers::mutate::handle_resolve)) .route("/conflicts", get(handlers::mutate::handle_conflicts)) - .route( - "/permissions", - get(handlers::mutate::handle_permissions_list), - ) - .route( - "/permissions/grant", - post(handlers::mutate::handle_permissions_grant), - ) - .route( - "/permissions/revoke", - post(handlers::mutate::handle_permissions_revoke), - ) + .route("/permissions", get(handlers::mutate::handle_permissions_list)) + .route("/permissions/grant", post(handlers::mutate::handle_permissions_grant)) + .route("/permissions/revoke", post(handlers::mutate::handle_permissions_revoke)) .route("/archive", post(handlers::mutate::handle_archive)) .route("/focus/start", post(handle_focus_start)) .route("/focus/end", post(handle_focus_end)) .route("/shutdown", post(handlers::mutate::handle_shutdown)) - // ── Conductor (locks, activity, messages, sessions, tasks) ── .route("/lock", post(handlers::conductor::handle_lock)) .route("/unlock", post(handlers::conductor::handle_unlock)) .route("/locks", get(handlers::conductor::handle_locks)) - .route( - "/activity", - post(handlers::conductor::handle_post_activity) - .get(handlers::conductor::handle_get_activity), - ) + .route("/activity", post(handlers::conductor::handle_post_activity).get(handlers::conductor::handle_get_activity)) .route("/message", post(handlers::conductor::handle_post_message)) .route("/messages", get(handlers::conductor::handle_get_messages)) - .route( - "/session/start", - post(handlers::conductor::handle_session_start), - ) - .route( - "/session/heartbeat", - post(handlers::conductor::handle_session_heartbeat), - ) - .route( - "/session/end", - post(handlers::conductor::handle_session_end), - ) + .route("/session/start", post(handlers::conductor::handle_session_start)) + .route("/session/heartbeat", post(handlers::conductor::handle_session_heartbeat)) + .route("/session/end", post(handlers::conductor::handle_session_end)) .route("/sessions", get(handlers::conductor::handle_sessions)) - .route( - "/tasks", - post(handlers::conductor::handle_create_task) - .get(handlers::conductor::handle_get_tasks), - ) + .route("/tasks", post(handlers::conductor::handle_create_task).get(handlers::conductor::handle_get_tasks)) .route("/tasks/next", get(handlers::conductor::handle_next_task)) .route("/tasks/claim", post(handlers::conductor::handle_claim_task)) - .route( - "/tasks/complete", - post(handlers::conductor::handle_complete_task), - ) - .route( - "/tasks/abandon", - post(handlers::conductor::handle_abandon_task), - ) - .route( - "/tasks/delete", - post(handlers::conductor::handle_delete_task), - ) - // ── Feed ──────────────────────────────────────────────────── - .route( - "/feed", - post(handlers::feed::handle_post_feed).get(handlers::feed::handle_get_feed), - ) + .route("/tasks/complete", post(handlers::conductor::handle_complete_task)) + .route("/tasks/abandon", post(handlers::conductor::handle_abandon_task)) + .route("/tasks/delete", post(handlers::conductor::handle_delete_task)) + .route("/feed", post(handlers::feed::handle_post_feed).get(handlers::feed::handle_get_feed)) .route("/feed/ack", post(handlers::feed::handle_feed_ack)) .route("/feed/{id}", get(handlers::feed::handle_get_feed_by_id)) - // ── Export / Import ──────────────────────────────────────── .route("/export", get(handlers::export::handle_export)) .route("/import", post(handlers::export::handle_import)) - // ── Admin (team-mode only, owner/admin role required) ── .route("/admin/user/add", post(handlers::admin::handle_user_add)) - .route( - "/admin/user/rotate-key", - post(handlers::admin::handle_user_rotate_key), - ) - .route( - "/admin/user/remove", - post(handlers::admin::handle_user_remove), - ) + .route("/admin/user/rotate-key", post(handlers::admin::handle_user_rotate_key)) + .route("/admin/user/remove", post(handlers::admin::handle_user_remove)) .route("/admin/users", get(handlers::admin::handle_user_list)) - .route( - "/admin/team/create", - post(handlers::admin::handle_team_create), - ) - .route( - "/admin/team/add-member", - post(handlers::admin::handle_team_add_member), - ) - .route( - "/admin/team/remove-member", - post(handlers::admin::handle_team_remove_member), - ) + .route("/admin/team/create", post(handlers::admin::handle_team_create)) + .route("/admin/team/add-member", post(handlers::admin::handle_team_add_member)) + .route("/admin/team/remove-member", post(handlers::admin::handle_team_remove_member)) .route("/admin/teams", get(handlers::admin::handle_team_list)) .route("/admin/unowned", get(handlers::admin::handle_unowned)) - .route( - "/admin/assign-owner", - post(handlers::admin::handle_assign_owner), - ) - .route( - "/admin/set-visibility", - post(handlers::admin::handle_set_visibility), - ) + .route("/admin/assign-owner", post(handlers::admin::handle_assign_owner)) + .route("/admin/set-visibility", post(handlers::admin::handle_set_visibility)) .route("/admin/archive", post(handlers::admin::handle_archive)) .route("/admin/stats", get(handlers::admin::handle_stats)) - // ── SSE events ────────────────────────────────────────────── - .route( - "/events/stream", - get(handlers::events::handle_events_stream), - ) - // ── Brain firing telemetry (owner-scoped, full payload) ───── - .route( - "/brain/firing", - get(handlers::events::handle_brain_firing_stream), - ) - // ── MCP-RPC proxy endpoint ───────────────────────────────── - // Accepts raw JSON-RPC messages (same format as MCP stdio). - // This lets `cortex mcp` run as a thin proxy -- no separate - // ONNX engine, no separate caches, zero duplication. + .route("/events/stream", get(handlers::events::handle_events_stream)) + .route("/brain/firing", get(handlers::events::handle_brain_firing_stream)) .route("/mcp-rpc", post(handle_mcp_rpc)) - .layer(axum::middleware::from_fn_with_state( - state.clone(), - activity_tracking_middleware, - )) - // Convert handler panics into HTTP 500 responses instead of letting - // them tear down the request task and surface as "MCP server exited" - // / "daemon went down" to clients. The daemon stays up; the bad call - // gets a structured error. + .layer(axum::middleware::from_fn_with_state(state.clone(), activity_tracking_middleware)) .layer(CatchPanicLayer::custom(handle_handler_panic)) .layer(cors) .with_state(state) } - pub(crate) fn handle_handler_panic(err: Box) -> Response { let message = if let Some(s) = err.downcast_ref::<&str>() { (*s).to_string() @@ -244,114 +121,64 @@ pub(crate) fn handle_handler_panic(err: Box) eprintln!("[cortex] HTTP handler panic: {message}"); handlers::json_response( StatusCode::INTERNAL_SERVER_ERROR, - serde_json::json!({ - "error": "internal server error", - }), + serde_json::json!({"error": +"internal server error",}), ) } - -pub(crate) async fn activity_tracking_middleware( - State(state): State, - mut request: Request, - next: Next, -) -> Response { +pub(crate) async fn activity_tracking_middleware(State(state): State, mut request: Request, next: Next) -> Response { state.mark_activity_now(); - request - .headers_mut() - .remove(handlers::CORTEX_PEER_IP_HEADER); - let peer_ip = request - .extensions() - .get::>() - .map(|ConnectInfo(addr)| addr.ip()); + request.headers_mut().remove(handlers::CORTEX_PEER_IP_HEADER); + let peer_ip = request.extensions().get::>().map(|ConnectInfo(addr)| addr.ip()); if let Some(ip) = peer_ip { if let Ok(value) = HeaderValue::from_str(&ip.to_string()) { - request - .headers_mut() - .insert(handlers::CORTEX_PEER_IP_HEADER, value); + request.headers_mut().insert(handlers::CORTEX_PEER_IP_HEADER, value); } } next.run(request).await } - -/// HTTP endpoint for MCP proxy -- accepts JSON-RPC, returns JSON-RPC. -/// SEC-001 fix: requires Bearer auth like all other POST mutation endpoints. -pub(crate) async fn handle_mcp_rpc( - State(state): State, - headers: HeaderMap, - body: Bytes, -) -> axum::response::Response { +pub(crate) async fn handle_mcp_rpc(State(state): State, headers: HeaderMap, body: Bytes) -> axum::response::Response { let caller_id = match handlers::ensure_auth_with_caller_rated(&headers, &state).await { Ok(caller_id) => caller_id, Err(resp) => { let (message, hint) = match resp.status() { - StatusCode::FORBIDDEN => ( - "Missing X-Cortex-Request header", - Some("Include header X-Cortex-Request: true"), - ), + StatusCode::FORBIDDEN => ("Missing X-Cortex-Request header", Some("Include header X-Cortex-Request: true")), StatusCode::UNAUTHORIZED => ("Unauthorized", None), _ => ("Auth failed", None), }; let status = resp.status(); return handlers::json_response( status, - serde_json::json!({ - "jsonrpc": "2.0", - "error": { - "code": -32600, - "message": message, - "hint": hint - }, - "id": serde_json::Value::Null - }), + serde_json::json!({"jsonrpc":"2.0","error":{"code":-32600,"message":message, +"hint":hint},"id":serde_json::Value::Null}), ); } }; - let msg: Value = match serde_json::from_slice(&body) { Ok(msg) => msg, Err(_) => { return handlers::json_response( StatusCode::BAD_REQUEST, - serde_json::json!({ - "jsonrpc": "2.0", - "error": { - "code": -32700, - "message": "Parse error" - }, - "id": serde_json::Value::Null - }), + serde_json::json!({"jsonrpc":"2.0","error":{"code":-32700,"message":"Parse error"} +,"id":serde_json::Value::Null}), ); } }; let source = handlers::resolve_source_identity(&headers, "mcp"); let ip = handlers::client_ip(&headers); - if let Some(decision) = state - .rate_limiter - .check_budget_for_endpoint(ip, BudgetEndpoint::Mcp) - .await - { + if let Some(decision) = state.rate_limiter.check_budget_for_endpoint(ip, BudgetEndpoint::Mcp).await { if !decision.allowed { handlers::log_budget_rejection(&state, &decision, &source.agent, &ip).await; let id = msg.get("id").cloned().unwrap_or(serde_json::Value::Null); return handlers::json_response( StatusCode::OK, - serde_json::json!({ - "jsonrpc": "2.0", - "error": { - "code": -32029, - "message": "budget_exceeded", - "data": decision.http_body_json() - }, - "id": id - }), + serde_json::json!({"jsonrpc":"2.0","error":{"code":-32029,"message": +"budget_exceeded","data":decision.http_body_json()},"id":id}), ); } } handlers::register_agent_presence_from_headers(&state, &headers, caller_id).await; - match handle_mcp_message_with_caller(&state, &msg, caller_id, Some(&source)).await { Some(resp) => handlers::json_response(StatusCode::OK, resp), None => handlers::json_response(StatusCode::OK, serde_json::json!({})), } } - diff --git a/daemon-rs/src/server/runtime.rs b/daemon-rs/src/server/runtime.rs index 98fba014..fbe9155f 100644 --- a/daemon-rs/src/server/runtime.rs +++ b/daemon-rs/src/server/runtime.rs @@ -1,34 +1,11 @@ -// SPDX-License-Identifier: MIT -use axum::body::Bytes; -use axum::extract::connect_info::ConnectInfo; -use axum::extract::{Request, State}; -use axum::http::{HeaderMap, HeaderValue, StatusCode}; -use axum::middleware::Next; -use axum::response::Response; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use serde_json::Value; +use axum::http::HeaderValue; +use axum::Router; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use tower::Service; -use tower_http::catch_panic::CatchPanicLayer; -use tower_http::cors::CorsLayer; - -use crate::budgets::BudgetEndpoint; -use crate::handlers; -use crate::handlers::mcp::handle_mcp_message_with_caller; -use crate::state::RuntimeState; - - -use super::*; pub async fn run( - router: Router, - bind_addr: &str, - port: u16, - ipc_endpoint: Option, - db_path: &Path, - readiness_signal: Option>, + router: Router, bind_addr: &str, port: u16, ipc_endpoint: Option, db_path: &Path, readiness_signal: Option>, shutdown: impl std::future::Future + Send + 'static, ) { if let Some(endpoint) = ipc_endpoint { @@ -42,78 +19,42 @@ pub async fn run( } }; let policy_bind_addr = effective_bind_addr_for_policy(bind_addr, activated_listener.as_ref()); - match crate::tls::try_load_tls() { Ok(Some(acceptor)) => { - run_tls( - router, - bind_addr, - port, - acceptor, - activated_listener.take(), - readiness_signal, - shutdown, - ) - .await; + run_tls(router, bind_addr, port, acceptor, activated_listener.take(), readiness_signal, shutdown).await; } Ok(None) => { let team_mode = detect_team_mode_for_tls(db_path); let allow_insecure_remote = allow_insecure_remote_http(); - if let Some(reason) = - plain_http_rejection_reason(&policy_bind_addr, team_mode, allow_insecure_remote) - { + if let Some(reason) = plain_http_rejection_reason(&policy_bind_addr, team_mode, allow_insecure_remote) { match reason { PlainHttpRejectionReason::TeamMode => { eprintln!("[cortex] TLS certificate not configured"); - eprintln!( - "[cortex] Team mode requires valid TLS -- add certs at ~/.cortex/tls/ or set CORTEX_TLS_CERT/CORTEX_TLS_KEY" - ); + eprintln!("[cortex] Team mode requires valid TLS -- add certs at ~/.cortex/tls/ or set CORTEX_TLS_CERT/CORTEX_TLS_KEY"); } PlainHttpRejectionReason::NonLocalBind => { eprintln!("[cortex] TLS certificate not configured"); - eprintln!( - "[cortex] Refusing plain HTTP for non-local bind '{policy_bind_addr}'." - ); - eprintln!( - "[cortex] Add TLS certs, bind to localhost, or set CORTEX_ALLOW_INSECURE_REMOTE=1 for explicit temporary override." - ); + eprintln!("[cortex] Refusing plain HTTP for non-local bind '{policy_bind_addr}'."); + eprintln!("[cortex] Add TLS certs, bind to localhost, or set CORTEX_ALLOW_INSECURE_REMOTE=1 for explicit temporary override."); } } std::process::exit(1); } - run_plain( - router, - bind_addr, - port, - activated_listener.take(), - readiness_signal, - shutdown, - ) - .await; + run_plain(router, bind_addr, port, activated_listener.take(), readiness_signal, shutdown).await; } Err(e) => { - // Team mode: refuse to start with broken TLS (auth integrity requires it) - // Solo mode: allow plain fallback only for localhost binds (or explicit override). let team_mode = detect_team_mode_for_tls(db_path); let allow_insecure_remote = allow_insecure_remote_http(); - if let Some(reason) = - plain_http_rejection_reason(&policy_bind_addr, team_mode, allow_insecure_remote) - { + if let Some(reason) = plain_http_rejection_reason(&policy_bind_addr, team_mode, allow_insecure_remote) { match reason { PlainHttpRejectionReason::TeamMode => { eprintln!("[cortex] TLS configuration error: {e}"); - eprintln!( - "[cortex] Team mode requires valid TLS -- fix certs at ~/.cortex/tls/ or set CORTEX_TLS_CERT/CORTEX_TLS_KEY" - ); + eprintln!("[cortex] Team mode requires valid TLS -- fix certs at ~/.cortex/tls/ or set CORTEX_TLS_CERT/CORTEX_TLS_KEY"); } PlainHttpRejectionReason::NonLocalBind => { eprintln!("[cortex] TLS configuration error: {e}"); - eprintln!( - "[cortex] Refusing insecure HTTP fallback for non-local bind '{policy_bind_addr}'." - ); - eprintln!( - "[cortex] Fix TLS certs, bind to localhost, or set CORTEX_ALLOW_INSECURE_REMOTE=1 for explicit temporary override." - ); + eprintln!("[cortex] Refusing insecure HTTP fallback for non-local bind '{policy_bind_addr}'."); + eprintln!("[cortex] Fix TLS certs, bind to localhost, or set CORTEX_ALLOW_INSECURE_REMOTE=1 for explicit temporary override."); } } std::process::exit(1); @@ -122,56 +63,30 @@ pub async fn run( if is_local_bind_addr(&policy_bind_addr) { eprintln!("[cortex] Starting without TLS (solo mode -- localhost bind)"); } else { - eprintln!( - "[cortex] Starting without TLS on non-local bind due to CORTEX_ALLOW_INSECURE_REMOTE=1" - ); + eprintln!("[cortex] Starting without TLS on non-local bind due to CORTEX_ALLOW_INSECURE_REMOTE=1"); } - run_plain( - router, - bind_addr, - port, - activated_listener.take(), - readiness_signal, - shutdown, - ) - .await; + run_plain(router, bind_addr, port, activated_listener.take(), readiness_signal, shutdown).await; } } } } - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum PlainHttpRejectionReason { TeamMode, NonLocalBind, } - pub(crate) fn allow_insecure_remote_http() -> bool { std::env::var("CORTEX_ALLOW_INSECURE_REMOTE") .ok() - .is_some_and(|value| { - matches!( - value.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ) - }) + .is_some_and(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) } - -pub(crate) fn effective_bind_addr_for_policy( - configured_bind: &str, - activated_listener: Option<&tokio::net::TcpListener>, -) -> String { +pub(crate) fn effective_bind_addr_for_policy(configured_bind: &str, activated_listener: Option<&tokio::net::TcpListener>) -> String { activated_listener .and_then(|listener| listener.local_addr().ok()) .map(|addr| addr.ip().to_string()) .unwrap_or_else(|| configured_bind.to_string()) } - -pub(crate) fn plain_http_rejection_reason( - bind_addr: &str, - team_mode: bool, - allow_insecure_remote: bool, -) -> Option { +pub(crate) fn plain_http_rejection_reason(bind_addr: &str, team_mode: bool, allow_insecure_remote: bool) -> Option { if team_mode { Some(PlainHttpRejectionReason::TeamMode) } else if !is_local_bind_addr(bind_addr) && !allow_insecure_remote { @@ -180,55 +95,29 @@ pub(crate) fn plain_http_rejection_reason( None } } - #[cfg(unix)] -pub(crate) fn resolve_socket_activation_listener( - expected_port: u16, -) -> Result, String> { +pub(crate) fn resolve_socket_activation_listener(expected_port: u16) -> Result, String> { use std::os::fd::FromRawFd; - pub(crate) const SYSTEMD_FIRST_SOCKET_FD: libc::c_int = 3; - - let listen_fds = std::env::var("LISTEN_FDS") - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - .unwrap_or(0); + let listen_fds = std::env::var("LISTEN_FDS").ok().and_then(|raw| raw.trim().parse::().ok()).unwrap_or(0); if listen_fds == 0 { return Ok(None); } - - let Some(listen_pid) = std::env::var("LISTEN_PID") - .ok() - .and_then(|raw| raw.trim().parse::().ok()) - else { + let Some(listen_pid) = std::env::var("LISTEN_PID").ok().and_then(|raw| raw.trim().parse::().ok()) else { return Err("LISTEN_FDS is set but LISTEN_PID is missing or invalid".to_string()); }; if listen_pid != std::process::id() { return Ok(None); } if listen_fds != 1 { - return Err(format!( - "Expected exactly one activated socket (LISTEN_FDS=1), got {listen_fds}" - )); + return Err(format!("Expected exactly one activated socket (LISTEN_FDS=1), got {listen_fds}")); } - validate_socket_activation_fd(SYSTEMD_FIRST_SOCKET_FD)?; - - // SAFETY: systemd-compatible socket activation passes the first owned - // listening socket at fd 3 when LISTEN_PID matches this process and - // LISTEN_FDS is exactly 1. The descriptor was also validated as open and - // stream-socket-shaped immediately before ownership is adopted here. let std_listener = unsafe { std::net::TcpListener::from_raw_fd(SYSTEMD_FIRST_SOCKET_FD) }; - std_listener - .set_nonblocking(true) - .map_err(|e| format!("configure activated socket nonblocking: {e}"))?; + std_listener.set_nonblocking(true).map_err(|e| format!("configure activated socket nonblocking: {e}"))?; if let Ok(addr) = std_listener.local_addr() { if expected_port > 0 && addr.port() != expected_port { - eprintln!( - "[cortex] Warning: activated socket port {} does not match configured port {}", - addr.port(), - expected_port - ); + eprintln!("[cortex] Warning: activated socket port {} does not match configured port {}", addr.port(), expected_port); } eprintln!("[cortex] Using socket-activated listener on {addr}"); } else { @@ -236,60 +125,27 @@ pub(crate) fn resolve_socket_activation_listener( } std::env::remove_var("LISTEN_FDS"); std::env::remove_var("LISTEN_PID"); - tokio::net::TcpListener::from_std(std_listener) - .map(Some) - .map_err(|e| format!("adopt socket-activated listener: {e}")) + tokio::net::TcpListener::from_std(std_listener).map(Some).map_err(|e| format!("adopt socket-activated listener: {e}")) } - #[cfg(unix)] pub(crate) fn validate_socket_activation_fd(fd: libc::c_int) -> Result<(), String> { - // SAFETY: F_GETFD only probes the integer descriptor. It does not borrow, - // duplicate, or take ownership of the descriptor, and invalid descriptors - // are reported as EBADF by the OS. if unsafe { libc::fcntl(fd, libc::F_GETFD) } < 0 { - return Err(format!( - "activated socket fd {fd} is not open: {}", - std::io::Error::last_os_error() - )); + return Err(format!("activated socket fd {fd} is not open: {}", std::io::Error::last_os_error())); } - let mut socket_type: libc::c_int = 0; let mut socket_type_len = std::mem::size_of::() as libc::socklen_t; - // SAFETY: `socket_type` and `socket_type_len` are valid out-pointers for - // the duration of the call. `getsockopt` only observes descriptor metadata - // and returns an error instead of taking ownership when `fd` is not a socket. - if unsafe { - libc::getsockopt( - fd, - libc::SOL_SOCKET, - libc::SO_TYPE, - (&mut socket_type as *mut libc::c_int).cast(), - &mut socket_type_len, - ) - } < 0 - { - return Err(format!( - "activated socket fd {fd} is not a socket: {}", - std::io::Error::last_os_error() - )); + if unsafe { libc::getsockopt(fd, libc::SOL_SOCKET, libc::SO_TYPE, (&mut socket_type as *mut libc::c_int).cast(), &mut socket_type_len) } < 0 { + return Err(format!("activated socket fd {fd} is not a socket: {}", std::io::Error::last_os_error())); } - if socket_type != libc::SOCK_STREAM { - return Err(format!( - "activated socket fd {fd} has unsupported socket type {socket_type}; expected SOCK_STREAM" - )); + return Err(format!("activated socket fd {fd} has unsupported socket type {socket_type}; expected SOCK_STREAM")); } - Ok(()) } - #[cfg(not(unix))] -pub(crate) fn resolve_socket_activation_listener( - _expected_port: u16, -) -> Result, String> { +pub(crate) fn resolve_socket_activation_listener(_expected_port: u16) -> Result, String> { Ok(None) } - pub(crate) fn mark_runtime_ready(readiness_signal: Option<&Arc>) { if let Some(readiness) = readiness_signal { let was_ready = readiness.swap(true, Ordering::SeqCst); @@ -298,7 +154,6 @@ pub(crate) fn mark_runtime_ready(readiness_signal: Option<&Arc>) { } } } - pub(crate) fn spawn_ipc_listener(router: Router, endpoint: String) { tokio::spawn(async move { if let Err(err) = run_ipc_listener(router, endpoint.clone()).await { @@ -306,12 +161,10 @@ pub(crate) fn spawn_ipc_listener(router: Router, endpoint: String) { } }); } - #[cfg(unix)] pub(crate) async fn run_ipc_listener(router: Router, endpoint: String) -> Result<(), String> { use std::os::unix::fs::PermissionsExt; use tokio::net::UnixListener; - let path = std::path::PathBuf::from(&endpoint); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).map_err(|e| format!("create IPC dir: {e}"))?; @@ -319,23 +172,16 @@ pub(crate) async fn run_ipc_listener(router: Router, endpoint: String) -> Result if path.exists() { std::fs::remove_file(&path).map_err(|e| format!("remove stale IPC socket: {e}"))?; } - let listener = UnixListener::bind(&path).map_err(|e| format!("bind IPC socket: {e}"))?; let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); eprintln!("[cortex] Listening on unix://{}", path.display()); - loop { match listener.accept().await { Ok((stream, _)) => { let hyper_svc = hyper_util::service::TowerToHyperService::new(router.clone()); tokio::spawn(async move { let io = hyper_util::rt::TokioIo::new(stream); - if let Err(err) = hyper_util::server::conn::auto::Builder::new( - hyper_util::rt::TokioExecutor::new(), - ) - .serve_connection(io, hyper_svc) - .await - { + if let Err(err) = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()).serve_connection(io, hyper_svc).await { eprintln!("[cortex] IPC unix connection error: {err}"); } }); @@ -344,11 +190,9 @@ pub(crate) async fn run_ipc_listener(router: Router, endpoint: String) -> Result } } } - #[cfg(windows)] pub(crate) async fn run_ipc_listener(router: Router, endpoint: String) -> Result<(), String> { use tokio::net::windows::named_pipe::ServerOptions; - let mut first_instance = true; eprintln!("[cortex] Listening on pipe://{endpoint}"); loop { @@ -368,35 +212,22 @@ pub(crate) async fn run_ipc_listener(router: Router, endpoint: String) -> Result } }; first_instance = false; - if let Err(err) = server.connect().await { eprintln!("[cortex] IPC pipe connect error: {err}"); continue; } - let hyper_svc = hyper_util::service::TowerToHyperService::new(router.clone()); tokio::spawn(async move { let io = hyper_util::rt::TokioIo::new(server); - if let Err(err) = - hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()) - .serve_connection(io, hyper_svc) - .await - { + if let Err(err) = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()).serve_connection(io, hyper_svc).await { eprintln!("[cortex] IPC pipe connection error: {err}"); } }); } } - pub(crate) fn is_local_bind_addr(bind_addr: &str) -> bool { - matches!( - bind_addr.trim().to_ascii_lowercase().as_str(), - "127.0.0.1" | "localhost" | "::1" | "[::1]" - ) + matches!(bind_addr.trim().to_ascii_lowercase().as_str(), "127.0.0.1" | "localhost" | "::1" | "[::1]") } - -/// Lightweight team-mode detection for TLS decisions (before full state init). -/// Opens the DB briefly to read the config table. pub(crate) fn detect_team_mode_for_tls(db_path: &Path) -> bool { if let Ok(conn) = crate::db::open(db_path) { crate::db::is_team_mode(&conn) @@ -404,13 +235,8 @@ pub(crate) fn detect_team_mode_for_tls(db_path: &Path) -> bool { false } } - pub(crate) async fn run_plain( - router: Router, - bind_addr: &str, - port: u16, - activated_listener: Option, - readiness_signal: Option>, + router: Router, bind_addr: &str, port: u16, activated_listener: Option, readiness_signal: Option>, shutdown: impl std::future::Future + Send + 'static, ) { let listener = match activated_listener { @@ -430,25 +256,16 @@ pub(crate) async fn run_plain( } else { eprintln!("[cortex] Listening on http://{bind_addr}:{port}"); } - if let Err(e) = axum::serve( - listener, - router.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(shutdown) - .await + if let Err(e) = axum::serve(listener, router.into_make_service_with_connect_info::()) + .with_graceful_shutdown(shutdown) + .await { eprintln!("[cortex] HTTP server exited with error: {e}"); } } - pub(crate) async fn run_tls( - router: Router, - bind_addr: &str, - port: u16, - acceptor: tokio_rustls::TlsAcceptor, - activated_listener: Option, - readiness_signal: Option>, - shutdown: impl std::future::Future + Send + 'static, + router: Router, bind_addr: &str, port: u16, acceptor: tokio_rustls::TlsAcceptor, activated_listener: Option, + readiness_signal: Option>, shutdown: impl std::future::Future + Send + 'static, ) { let listener = match activated_listener { Some(listener) => listener, @@ -467,57 +284,19 @@ pub(crate) async fn run_tls( } else { eprintln!("[cortex] Listening on https://{bind_addr}:{port} (TLS via rustls)"); } - let mut make_svc = router.into_make_service_with_connect_info::(); - tokio::pin!(shutdown); - loop { - tokio::select! { - _ = &mut shutdown => { - eprintln!("[cortex] TLS server shutting down"); - break; - } - accept = listener.accept() => { - match accept { - Ok((stream, _addr)) => { - let acceptor = acceptor.clone(); - let tower_svc = match make_svc.call(_addr).await { - Ok(tower_svc) => tower_svc, - Err(e) => { - eprintln!("[cortex] Failed to build TLS service for {_addr}: {e}"); - continue; - } - }; - tokio::spawn(async move { - match acceptor.accept(stream).await { - Ok(tls_stream) => { - let hyper_svc = hyper_util::service::TowerToHyperService::new(tower_svc); - let io = hyper_util::rt::TokioIo::new(tls_stream); - if let Err(e) = hyper_util::server::conn::auto::Builder::new( - hyper_util::rt::TokioExecutor::new(), - ) - .serve_connection(io, hyper_svc) - .await - { - eprintln!("[cortex] TLS connection error for {_addr}: {e}"); - } - } - Err(e) => { - eprintln!("[cortex] TLS handshake failed: {e}"); - } - } - }); - } - Err(e) => { - eprintln!("[cortex] TCP accept error: {e}"); - } - } - } - } + tokio::select! {_=&mut shutdown=>{eprintln + !("[cortex] TLS server shutting down");break;}accept=listener.accept()=>{match accept{Ok((stream,_addr))=>{let acceptor=acceptor. + clone();let tower_svc=match make_svc.call(_addr).await{Ok(tower_svc)=>tower_svc,Err(e)=>{eprintln!( + "[cortex] Failed to build TLS service for {_addr}: {e}");continue;}};tokio::spawn(async move{match acceptor.accept(stream).await{ + Ok(tls_stream)=>{let hyper_svc=hyper_util::service::TowerToHyperService::new(tower_svc);let io=hyper_util::rt::TokioIo::new( + tls_stream);if let Err(e)=hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new(),).serve_connection(io, + hyper_svc).await{eprintln!("[cortex] TLS connection error for {_addr}: {e}");}}Err(e)=>{eprintln!( + "[cortex] TLS handshake failed: {e}");}}});}Err(e)=>{eprintln!("[cortex] TCP accept error: {e}");}}}} } } - pub(crate) fn parse_allowed_origin(origin: &str) -> Option { match origin.parse::() { Ok(value) => Some(value), diff --git a/daemon-rs/src/server/tests.rs b/daemon-rs/src/server/tests/mod.rs similarity index 70% rename from daemon-rs/src/server/tests.rs rename to daemon-rs/src/server/tests/mod.rs index 878e5207..854c0a88 100644 --- a/daemon-rs/src/server/tests.rs +++ b/daemon-rs/src/server/tests/mod.rs @@ -1,8 +1,5 @@ // SPDX-License-Identifier: MIT -//! HTTP bind/TLS policy boundaries only. - use super::*; - #[test] fn local_bind_detection_is_strict() { assert!(is_local_bind_addr("127.0.0.1")); @@ -11,25 +8,16 @@ fn local_bind_detection_is_strict() { assert!(!is_local_bind_addr("0.0.0.0")); assert!(!is_local_bind_addr("100.84.247.96")); } - #[test] fn plain_http_policy_rejects_team_mode_and_non_local_binds() { - assert_eq!( - plain_http_rejection_reason("127.0.0.1", true, false), - Some(PlainHttpRejectionReason::TeamMode) - ); - assert_eq!( - plain_http_rejection_reason("0.0.0.0", false, false), - Some(PlainHttpRejectionReason::NonLocalBind) - ); + assert_eq!(plain_http_rejection_reason("127.0.0.1", true, false), Some(PlainHttpRejectionReason::TeamMode)); + assert_eq!(plain_http_rejection_reason("0.0.0.0", false, false), Some(PlainHttpRejectionReason::NonLocalBind)); assert_eq!(plain_http_rejection_reason("127.0.0.1", false, false), None); } - #[cfg(unix)] #[test] fn socket_activation_fd_validation_rejects_regular_file() { use std::os::fd::AsRawFd; - let file = tempfile::tempfile().unwrap(); let err = validate_socket_activation_fd(file.as_raw_fd()).unwrap_err(); assert!(err.contains("not a socket"), "{err}"); diff --git a/daemon-rs/src/service.rs b/daemon-rs/src/service.rs deleted file mode 100644 index 22fddbcb..00000000 --- a/daemon-rs/src/service.rs +++ /dev/null @@ -1,962 +0,0 @@ -// SPDX-License-Identifier: MIT -//! Windows Service support for Cortex daemon. -//! -//! Subcommands: -//! `cortex service install` -- Register as Windows Service (requires Admin) -//! `cortex service uninstall` -- Remove Windows Service -//! `cortex service start` -- Start the service -//! `cortex service stop` -- Stop the service -//! `cortex service status` -- Check service status -//! `cortex service ensure` -- Ensure installed + running + healthy -//! `cortex service-run` -- Internal: SCM entry point -//! -//! The service runs the same daemon as `cortex serve` but under the Windows -//! Service Control Manager with manual start by default, auto-restart on -//! failure, and proper lifecycle management. - -const SERVICE_NAME: &str = "CortexDaemon"; -const DISPLAY_NAME: &str = "Cortex Memory Daemon"; -const DESCRIPTION: &str = "Always-on AI memory daemon -- serves Claude, Gemini, Codex, Cursor, and local LLMs via HTTP (:7437) and MCP."; -const DEFAULT_START_MODE: &str = "demand"; -const ENSURE_HEALTH_TIMEOUT_SECS: u64 = 12; -const ENSURE_POLL_MILLIS: u64 = 250; -const HEALTH_PROBE_TIMEOUT_SECS: u64 = 2; - -#[cfg(windows)] -const CREATE_NO_WINDOW_FLAG: u32 = 0x0800_0000; - -#[cfg(windows)] -fn apply_hidden_process_flags(command: &mut std::process::Command) { - use std::os::windows::process::CommandExt; - command.creation_flags(CREATE_NO_WINDOW_FLAG); -} - -#[cfg(not(windows))] -fn apply_hidden_process_flags(_command: &mut std::process::Command) {} - -fn daemon_base_url() -> String { - let port = crate::auth::CortexPaths::resolve().port; - format!("http://127.0.0.1:{port}") -} - -fn daemon_health_url() -> String { - format!("{}/health", daemon_base_url()) -} - -fn daemon_ready_from_payload( - status: u16, - body: &str, - paths: &crate::auth::CortexPaths, -) -> Option { - if let Some(ready) = crate::daemon_lifecycle::readiness_state_from_payload( - status, - body, - Some(paths.port), - Some(paths), - ) { - return Some(ready); - } - if crate::daemon_lifecycle::is_cortex_health_payload( - status, - body, - Some(paths.port), - Some(paths), - ) { - return Some(true); - } - None -} - -fn escape_cmd_quoted_fragment(value: &str) -> String { - value.replace('^', "^^").replace('%', "^%") -} - -fn build_sc_create_command(exe_path: &str, username: &str) -> String { - let exe_path = escape_cmd_quoted_fragment(exe_path); - let username = escape_cmd_quoted_fragment(username); - format!( - "sc.exe create {} binPath= \"\\\"{}\\\" service-run\" start= {} DisplayName= \"{}\" obj= \".\\{}\"", - SERVICE_NAME, exe_path, DEFAULT_START_MODE, DISPLAY_NAME, username - ) -} - -fn username_is_safe_for_cmd_fragment(value: &str) -> bool { - !value.trim().is_empty() - && value - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, ' ' | '.' | '_' | '-')) -} - -fn resolve_service_username_from_env() -> String { - match std::env::var("USERNAME") { - Ok(raw) => { - let trimmed = raw.trim(); - if username_is_safe_for_cmd_fragment(trimmed) { - trimmed.to_string() - } else { - "cortex-user".to_string() - } - } - Err(_) => "cortex-user".to_string(), - } -} - -fn service_exe_path_from_result( - result: std::io::Result, -) -> Result { - result - .map(|exe| exe.to_string_lossy().to_string()) - .map_err(|err| format!("Failed to get exe path: {err}")) -} - -fn service_exe_path() -> Result { - service_exe_path_from_result(std::env::current_exe()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ServiceState { - NotInstalled, - Running, - Stopped, - StartPending, - StopPending, - Unknown, -} - -impl ServiceState { - fn as_str(self) -> &'static str { - match self { - ServiceState::NotInstalled => "NOT_INSTALLED", - ServiceState::Running => "RUNNING", - ServiceState::Stopped => "STOPPED", - ServiceState::StartPending => "START_PENDING", - ServiceState::StopPending => "STOP_PENDING", - ServiceState::Unknown => "UNKNOWN", - } - } -} - -fn output_text(output: &std::process::Output) -> String { - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - match (stdout.is_empty(), stderr.is_empty()) { - (false, false) => format!("{stdout}\n{stderr}"), - (false, true) => stdout, - (true, false) => stderr, - (true, true) => "".to_string(), - } -} - -fn parse_service_state(output_text: &str) -> ServiceState { - if output_text.contains("RUNNING") { - ServiceState::Running - } else if output_text.contains("STOPPED") { - ServiceState::Stopped - } else if output_text.contains("START_PENDING") { - ServiceState::StartPending - } else if output_text.contains("STOP_PENDING") { - ServiceState::StopPending - } else { - ServiceState::Unknown - } -} - -fn query_service_state() -> Result { - let mut command = std::process::Command::new("sc.exe"); - command.args(["query", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - let output = command - .output() - .map_err(|e| format!("Failed to run sc.exe query: {e}"))?; - - if output.status.success() { - let text = output_text(&output); - return Ok(parse_service_state(&text)); - } - - let text = output_text(&output); - if text.contains("1060") || text.contains("does not exist") { - Ok(ServiceState::NotInstalled) - } else { - Err(text) - } -} - -fn parse_http_probe_response(raw: &[u8]) -> Result<(u16, String), String> { - let (status, body) = crate::transport::parse_http_response_bytes(raw, "Cortex daemon")?; - Ok((status.as_u16(), body)) -} - -fn should_use_partial_probe_response(err: &std::io::Error, response_len: usize) -> bool { - response_len > 0 - && matches!( - err.kind(), - std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock - ) -} - -fn daemon_probe(path: &str) -> Result<(u16, String), String> { - use std::io::{Read, Write}; - - let port = crate::auth::CortexPaths::resolve().port; - let mut stream = std::net::TcpStream::connect_timeout( - &std::net::SocketAddr::from(([127, 0, 0, 1], port)), - std::time::Duration::from_secs(HEALTH_PROBE_TIMEOUT_SECS), - ) - .map_err(|e| format!("connect failed: {e}"))?; - stream - .set_read_timeout(Some(std::time::Duration::from_secs( - HEALTH_PROBE_TIMEOUT_SECS, - ))) - .map_err(|e| format!("read timeout failed: {e}"))?; - stream - .set_write_timeout(Some(std::time::Duration::from_secs( - HEALTH_PROBE_TIMEOUT_SECS, - ))) - .map_err(|e| format!("write timeout failed: {e}"))?; - - let request = format!( - "GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nX-Cortex-Request: true\r\nConnection: close\r\n\r\n" - ); - stream - .write_all(request.as_bytes()) - .map_err(|e| format!("write failed: {e}"))?; - - let mut response = Vec::new(); - if let Err(err) = stream.read_to_end(&mut response) { - if !should_use_partial_probe_response(&err, response.len()) { - return Err(format!("read failed: {err}")); - } - } - parse_http_probe_response(&response) -} - -fn daemon_health_response() -> Option { - let paths = crate::auth::CortexPaths::resolve(); - - if let Ok((status, body)) = daemon_probe("/readiness") { - if daemon_ready_from_payload(status, &body, &paths) == Some(true) { - return Some(body); - } - } - - if let Ok((status, body)) = daemon_probe("/health") { - if daemon_ready_from_payload(status, &body, &paths).unwrap_or(false) { - return Some(body); - } - } - - None -} - -fn daemon_health_ready() -> bool { - daemon_health_response().is_some() -} - -fn wait_for_daemon_health(timeout: std::time::Duration) -> bool { - let start = std::time::Instant::now(); - loop { - if daemon_health_ready() { - return true; - } - if start.elapsed() >= timeout { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(ENSURE_POLL_MILLIS)); - } -} - -fn start_service_once() -> Result<(), String> { - let mut command = std::process::Command::new("sc.exe"); - command.args(["start", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - let output = command - .output() - .map_err(|e| format!("Failed to run sc.exe start: {e}"))?; - - if output.status.success() { - return Ok(()); - } - - let text = output_text(&output); - if text.contains("1056") { - Ok(()) - } else { - Err(text) - } -} - -fn stop_service_once() -> Result<(), String> { - let mut command = std::process::Command::new("sc.exe"); - command.args(["stop", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - let output = command - .output() - .map_err(|e| format!("Failed to run sc.exe stop: {e}"))?; - - if output.status.success() { - return Ok(()); - } - - let text = output_text(&output); - if text.contains("1062") { - Ok(()) - } else { - Err(text) - } -} - -#[cfg(windows)] -fn ensure_windows() -> bool { - if daemon_health_ready() { - eprintln!("[cortex] Daemon already healthy"); - return true; - } - - let mut state = match query_service_state() { - Ok(state) => state, - Err(err) => { - eprintln!("[cortex] Failed to query service state: {err}"); - return false; - } - }; - - if state == ServiceState::NotInstalled { - eprintln!("[cortex] Service not installed; installing"); - install(); - state = match query_service_state() { - Ok(next) => next, - Err(err) => { - eprintln!("[cortex] Failed to query service state after install: {err}"); - return false; - } - }; - if state == ServiceState::NotInstalled { - eprintln!("[cortex] Service install did not complete (run as Administrator if needed)"); - return false; - } - } - - eprintln!("[cortex] Service state before ensure: {}", state.as_str()); - - if state == ServiceState::Running { - if wait_for_daemon_health(std::time::Duration::from_secs(2)) { - eprintln!("[cortex] Service already running and healthy"); - return true; - } - eprintln!("[cortex] Service running but health failed; restarting once"); - if let Err(err) = stop_service_once() { - eprintln!("[cortex] Failed to stop unhealthy service: {err}"); - return false; - } - } - - if let Err(err) = start_service_once() { - eprintln!("[cortex] Failed to start service: {err}"); - return false; - } - - if wait_for_daemon_health(std::time::Duration::from_secs(ENSURE_HEALTH_TIMEOUT_SECS)) { - eprintln!("[cortex] Service ensured and daemon health is live"); - true - } else { - eprintln!("[cortex] Service started but daemon health endpoint is still unavailable"); - false - } -} - -// ---- CLI commands (work on any platform) ------------------------------------ - -pub fn install() -> bool { - let exe_path = match service_exe_path() { - Ok(path) => path, - Err(err) => { - eprintln!("[cortex] {err}"); - return false; - } - }; - - // COR-8 fix: detect current username to run service under user account, - // NOT LocalSystem. LocalSystem has a different USERPROFILE which would - // open a completely separate database at C:\Windows\system32\config\systemprofile. - let username_env = std::env::var("USERNAME").ok(); - let username = resolve_service_username_from_env(); - if let Some(raw) = username_env { - let trimmed = raw.trim(); - if !trimmed.is_empty() && trimmed != username { - eprintln!( - "[cortex] Warning: USERNAME contains unsupported characters; falling back to '{}'", - username - ); - } - } - - // COR-5 fix: use cmd /C for sc.exe to handle binPath quoting correctly. - // sc.exe has non-standard argument parsing that fights with Rust's Command. - let sc_cmd = build_sc_create_command(&exe_path, &username); - - let mut create_cmd = std::process::Command::new("cmd"); - create_cmd.args(["/V:OFF", "/C", &sc_cmd]); - apply_hidden_process_flags(&mut create_cmd); - let output = create_cmd.output(); - - match output { - Ok(o) if o.status.success() => { - eprintln!("[cortex] Service '{}' installed", SERVICE_NAME); - eprintln!("[cortex] Runs as: .\\{}", username); - - // Set description - let mut description_cmd = std::process::Command::new("sc.exe"); - description_cmd.args(["description", SERVICE_NAME, DESCRIPTION]); - apply_hidden_process_flags(&mut description_cmd); - let _ = description_cmd.output(); - - // Configure recovery: restart on failure (5s, 10s, 30s) - let mut failure_cmd = std::process::Command::new("cmd"); - failure_cmd.args([ - "/C", - &format!( - "sc.exe failure {} reset= 86400 actions= restart/5000/restart/10000/restart/30000", - SERVICE_NAME - ), - ]); - apply_hidden_process_flags(&mut failure_cmd); - let _ = failure_cmd.output(); - - eprintln!("[cortex] Auto-start on boot: disabled (manual start mode)"); - eprintln!("[cortex] To opt in later: sc.exe config CortexDaemon start= auto"); - eprintln!("[cortex] Recovery: restart on failure (5s / 10s / 30s)"); - eprintln!("[cortex] NOTE: You may need to set the password:"); - eprintln!("[cortex] sc.exe config CortexDaemon password= YOUR_PASSWORD"); - eprintln!("[cortex] Then: cortex service start"); - true - } - Ok(o) => { - let stderr = String::from_utf8_lossy(&o.stderr); - if stderr.contains("1073") { - eprintln!("[cortex] Service already exists. Run: cortex service uninstall"); - } else { - eprintln!("[cortex] Failed to install (run as Administrator)"); - eprintln!("{}", stderr); - } - false - } - Err(e) => { - eprintln!("[cortex] Failed to run sc.exe: {e}"); - false - } - } -} - -pub fn uninstall() -> bool { - // Stop first (ignore errors if not running) - let mut stop_cmd = std::process::Command::new("sc.exe"); - stop_cmd.args(["stop", SERVICE_NAME]); - apply_hidden_process_flags(&mut stop_cmd); - let _ = stop_cmd.output(); - std::thread::sleep(std::time::Duration::from_secs(2)); - - let mut delete_cmd = std::process::Command::new("sc.exe"); - delete_cmd.args(["delete", SERVICE_NAME]); - apply_hidden_process_flags(&mut delete_cmd); - match delete_cmd.output() { - Ok(o) if o.status.success() => { - eprintln!("[cortex] Service uninstalled"); - true - } - Ok(o) => { - eprintln!("[cortex] Failed to uninstall"); - eprintln!("{}", String::from_utf8_lossy(&o.stderr)); - false - } - Err(e) => { - eprintln!("[cortex] Failed to run sc.exe: {e}"); - false - } - } -} - -pub fn start() -> bool { - let mut command = std::process::Command::new("sc.exe"); - command.args(["start", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - match command.output() { - Ok(o) if o.status.success() => { - eprintln!("[cortex] Service started"); - // Wait and verify - std::thread::sleep(std::time::Duration::from_secs(3)); - let health_url = daemon_health_url(); - if daemon_health_ready() { - eprintln!("[cortex] Daemon is LIVE at {health_url}"); - if let Ok((_, body)) = daemon_probe("/health") { - eprintln!("{body}"); - } - } else { - eprintln!("[cortex] Service started but health check pending"); - } - true - } - Ok(o) => { - let stderr = String::from_utf8_lossy(&o.stderr); - if stderr.contains("1056") { - eprintln!("[cortex] Service is already running"); - true - } else { - eprintln!("[cortex] Failed to start service"); - eprintln!("{}", stderr); - false - } - } - Err(e) => { - eprintln!("[cortex] Failed to run sc.exe: {e}"); - false - } - } -} - -pub fn stop() -> bool { - let mut command = std::process::Command::new("sc.exe"); - command.args(["stop", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - match command.output() { - Ok(o) if o.status.success() => { - eprintln!("[cortex] Service stopped"); - true - } - Ok(o) => { - let stderr = String::from_utf8_lossy(&o.stderr); - if stderr.contains("1062") { - eprintln!("[cortex] Service is not running"); - true - } else { - eprintln!("[cortex] Failed to stop"); - eprintln!("{}", stderr); - false - } - } - Err(e) => { - eprintln!("[cortex] Failed to run sc.exe: {e}"); - false - } - } -} - -pub fn status() -> bool { - let mut command = std::process::Command::new("sc.exe"); - command.args(["query", SERVICE_NAME]); - apply_hidden_process_flags(&mut command); - match command.output() { - Ok(o) if o.status.success() => { - let stdout = String::from_utf8_lossy(&o.stdout); - let state = if stdout.contains("RUNNING") { - "RUNNING" - } else if stdout.contains("STOPPED") { - "STOPPED" - } else if stdout.contains("START_PENDING") { - "STARTING" - } else { - "UNKNOWN" - }; - eprintln!("[cortex] Service: {state}"); - - // Also check HTTP health - if daemon_health_ready() { - eprintln!("[cortex] HTTP: LIVE"); - if let Ok((_, body)) = daemon_probe("/health") { - eprintln!("{body}"); - } - } else { - eprintln!("[cortex] HTTP: not responding"); - } - true - } - Ok(_) => { - eprintln!("[cortex] Service not installed. Run: cortex service install"); - false - } - Err(e) => { - eprintln!("[cortex] Failed to run sc.exe: {e}"); - false - } - } -} - -pub fn ensure() -> bool { - #[cfg(not(windows))] - { - eprintln!("[cortex] `service ensure` is only available on Windows"); - false - } - - #[cfg(windows)] - { - ensure_windows() - } -} - -/// Service-first daemon ensure for library callers. -/// Returns true when daemon health is live after ensure. -#[cfg(windows)] -pub fn ensure_ready() -> bool { - ensure_windows() -} - -#[cfg(not(windows))] -pub fn ensure_ready() -> bool { - false -} - -// ---- Windows Service entry point (called by SCM) ---------------------------- - -#[cfg(windows)] -mod scm { - use std::ffi::OsString; - use std::sync::mpsc; - use windows_service::service::{ - ServiceControl, ServiceControlAccept, ServiceExitCode, ServiceState, ServiceStatus, - ServiceType, - }; - use windows_service::service_control_handler::{self, ServiceControlHandlerResult}; - use windows_service::{define_windows_service, service_dispatcher}; - - const SERVICE_TYPE: ServiceType = ServiceType::OWN_PROCESS; - - define_windows_service!(ffi_service_main, cortex_service_main); - - pub fn dispatch() { - if let Err(err) = service_dispatcher::start(super::SERVICE_NAME, ffi_service_main) { - eprintln!("[cortex] Failed to start service dispatcher: {err}"); - std::process::exit(1); - } - } - - fn cortex_service_main(_arguments: Vec) { - let (stop_tx, stop_rx) = mpsc::channel::<()>(); - - let event_handler = move |control_event| -> ServiceControlHandlerResult { - match control_event { - ServiceControl::Stop | ServiceControl::Shutdown => { - stop_tx.send(()).ok(); - ServiceControlHandlerResult::NoError - } - ServiceControl::Interrogate => ServiceControlHandlerResult::NoError, - _ => ServiceControlHandlerResult::NotImplemented, - } - }; - - let status_handle = - match service_control_handler::register(super::SERVICE_NAME, event_handler) { - Ok(handle) => handle, - Err(err) => { - eprintln!("[cortex-service] Failed to register service control handler: {err}"); - return; - } - }; - - // Report: Starting - let _ = status_handle.set_service_status(ServiceStatus { - service_type: SERVICE_TYPE, - current_state: ServiceState::StartPending, - controls_accepted: ServiceControlAccept::empty(), - exit_code: ServiceExitCode::Win32(0), - checkpoint: 0, - wait_hint: std::time::Duration::from_secs(15), - process_id: None, - }); - - // COR-4 fix: report Stopped with error if runtime creation fails - let rt = match tokio::runtime::Runtime::new() { - Ok(rt) => rt, - Err(e) => { - eprintln!("[cortex-service] Failed to create tokio runtime: {e}"); - let _ = status_handle.set_service_status(ServiceStatus { - service_type: SERVICE_TYPE, - current_state: ServiceState::Stopped, - controls_accepted: ServiceControlAccept::empty(), - exit_code: ServiceExitCode::Win32(1), - checkpoint: 0, - wait_hint: std::time::Duration::default(), - process_id: None, - }); - return; - } - }; - - // COR-3 fix: report Running AFTER entering rt.block_on but BEFORE - // run_daemon blocks on server::run. The daemon init (DB, indexing) - // happens first, then we report Running right before the server binds. - // Note: ideally we'd signal from inside run_daemon after bind, but - // the current architecture doesn't expose that hook. Reporting here - // is a reasonable compromise -- init is fast, server bind follows. - let _ = status_handle.set_service_status(ServiceStatus { - service_type: SERVICE_TYPE, - current_state: ServiceState::Running, - controls_accepted: ServiceControlAccept::STOP | ServiceControlAccept::SHUTDOWN, - exit_code: ServiceExitCode::Win32(0), - checkpoint: 0, - wait_hint: std::time::Duration::default(), - process_id: None, - }); - - // Run the daemon with service shutdown signal - rt.block_on(async { - crate::run_daemon(crate::auth::CortexPaths::resolve(), async move { - // Bridge std::sync::mpsc to async via spawn_blocking - tokio::task::spawn_blocking(move || { - stop_rx.recv().ok(); - }) - .await - .ok(); - eprintln!("[cortex-service] Stop signal received"); - }) - .await; - }); - - // Report: Stopped - let _ = status_handle.set_service_status(ServiceStatus { - service_type: SERVICE_TYPE, - current_state: ServiceState::Stopped, - controls_accepted: ServiceControlAccept::empty(), - exit_code: ServiceExitCode::Win32(0), - checkpoint: 0, - wait_hint: std::time::Duration::default(), - process_id: None, - }); - } -} - -/// Dispatch to Windows SCM. Called from main.rs `service-run` arm. -#[cfg(windows)] -pub fn dispatch_service() { - scm::dispatch(); -} - -#[cfg(not(windows))] -pub fn dispatch_service() { - eprintln!("[cortex] Windows Service is only available on Windows"); - std::process::exit(1); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_test_dir(name: &str) -> std::path::PathBuf { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - std::env::temp_dir().join(format!("cortex_service_{name}_{unique}")) - } - - #[test] - fn build_sc_create_command_defaults_to_manual_start() { - let cmd = build_sc_create_command(r"C:\Program Files\Cortex\cortex.exe", "alice"); - assert!( - cmd.contains("start= demand"), - "expected manual start mode: {cmd}" - ); - assert!( - !cmd.contains("start= auto"), - "must not auto-start by default: {cmd}" - ); - } - - #[test] - fn build_sc_create_command_includes_quoted_binpath_and_user() { - let exe = r"C:\Program Files\Cortex\cortex.exe"; - let cmd = build_sc_create_command(exe, "alice"); - let expected_bin = format!("binPath= \"\\\"{}\\\" service-run\"", exe); - assert!(cmd.contains(&format!("sc.exe create {}", SERVICE_NAME))); - assert!( - cmd.contains(&expected_bin), - "missing binPath quoting: {cmd}" - ); - assert!( - cmd.contains("obj= \".\\alice\""), - "missing user account object: {cmd}" - ); - } - - #[test] - fn build_sc_create_command_escapes_cmd_expansion_in_exe_path() { - let cmd = build_sc_create_command(r"C:\Tools\%PATH%\Cortex^Bin\cortex.exe", "alice"); - assert!( - cmd.contains(r"C:\Tools\^%PATH^%\Cortex^^Bin\cortex.exe"), - "executable path must survive cmd.exe parsing without expansion: {cmd}" - ); - } - - #[test] - fn username_is_safe_for_cmd_fragment_rejects_shell_metacharacters() { - assert!(username_is_safe_for_cmd_fragment("alice")); - assert!(username_is_safe_for_cmd_fragment("alice.svc")); - assert!(username_is_safe_for_cmd_fragment("alice svc")); - assert!(!username_is_safe_for_cmd_fragment("alice&whoami")); - assert!(!username_is_safe_for_cmd_fragment("alice|powershell")); - assert!(!username_is_safe_for_cmd_fragment("alice%PATH%")); - assert!(!username_is_safe_for_cmd_fragment("alice\"quoted")); - } - - #[test] - fn resolve_service_username_from_env_falls_back_when_username_is_unsafe() { - let _env_lock = crate::test_env::lock(); - let _username = crate::test_env::ScopedEnvVar::set("USERNAME", "alice&whoami"); - assert_eq!(resolve_service_username_from_env(), "cortex-user"); - std::env::set_var("USERNAME", "alice.svc"); - assert_eq!(resolve_service_username_from_env(), "alice.svc"); - std::env::remove_var("USERNAME"); - assert_eq!(resolve_service_username_from_env(), "cortex-user"); - } - - #[test] - fn service_exe_path_from_result_reports_resolution_failure() { - let err = service_exe_path_from_result(Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "missing exe", - ))) - .unwrap_err(); - assert!( - err.contains("Failed to get exe path"), - "expected contextual error: {err}" - ); - } - - #[test] - fn parse_service_state_recognizes_known_states() { - assert_eq!( - parse_service_state("STATE : 4 RUNNING"), - ServiceState::Running - ); - assert_eq!( - parse_service_state("STATE : 1 STOPPED"), - ServiceState::Stopped - ); - assert_eq!( - parse_service_state("STATE : 2 START_PENDING"), - ServiceState::StartPending - ); - assert_eq!( - parse_service_state("STATE : 3 STOP_PENDING"), - ServiceState::StopPending - ); - assert_eq!( - parse_service_state("STATE : ???"), - ServiceState::Unknown - ); - } - - #[test] - fn service_state_strings_are_stable() { - assert_eq!(ServiceState::NotInstalled.as_str(), "NOT_INSTALLED"); - assert_eq!(ServiceState::Running.as_str(), "RUNNING"); - assert_eq!(ServiceState::Stopped.as_str(), "STOPPED"); - assert_eq!(ServiceState::StartPending.as_str(), "START_PENDING"); - assert_eq!(ServiceState::StopPending.as_str(), "STOP_PENDING"); - assert_eq!(ServiceState::Unknown.as_str(), "UNKNOWN"); - } - - #[test] - fn daemon_ready_payload_accepts_readiness_ready_and_health_ok() { - let home_dir = temp_test_dir("ready_payload"); - let home = home_dir.to_string_lossy().to_string(); - let paths = crate::auth::CortexPaths::resolve_with_overrides( - Some(&home), - None, - Some(7437), - Some("127.0.0.1"), - ); - - let readiness = serde_json::json!({ - "status": "ready", - "ready": true, - "runtime": { - "port": 7437, - "token_path": paths.token.display().to_string(), - "db_path": paths.db.display().to_string(), - "pid_path": paths.pid.display().to_string(), - }, - "stats": { "home": paths.home.display().to_string() } - }) - .to_string(); - assert_eq!( - daemon_ready_from_payload(200, &readiness, &paths), - Some(true) - ); - - let health = serde_json::json!({ - "status": "ok", - "runtime": { - "port": 7437, - "token_path": paths.token.display().to_string(), - "db_path": paths.db.display().to_string(), - "pid_path": paths.pid.display().to_string(), - }, - "stats": { "home": paths.home.display().to_string(), "memories": 1 } - }) - .to_string(); - assert_eq!(daemon_ready_from_payload(200, &health, &paths), Some(true)); - } - - #[test] - fn daemon_ready_payload_preserves_starting_state() { - let home_dir = temp_test_dir("starting_payload"); - let home = home_dir.to_string_lossy().to_string(); - let paths = crate::auth::CortexPaths::resolve_with_overrides( - Some(&home), - None, - Some(7437), - Some("127.0.0.1"), - ); - - let readiness = serde_json::json!({ - "status": "starting", - "ready": false, - "runtime": { - "port": 7437, - "token_path": paths.token.display().to_string(), - "db_path": paths.db.display().to_string(), - "pid_path": paths.pid.display().to_string(), - }, - "stats": { "home": paths.home.display().to_string() } - }) - .to_string(); - assert_eq!( - daemon_ready_from_payload(503, &readiness, &paths), - Some(false) - ); - } - - #[test] - fn parse_http_probe_response_extracts_status_and_body() { - let raw = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"status\":\"ok\"}"; - let (status, body) = parse_http_probe_response(raw).expect("parse response"); - assert_eq!(status, 200); - assert_eq!(body, "{\"status\":\"ok\"}"); - } - - #[test] - fn parse_http_probe_response_rejects_invalid_payloads() { - let err = parse_http_probe_response(b"not-http").unwrap_err(); - assert!(err.contains("invalid HTTP response")); - - let err = parse_http_probe_response(b"not-http 200 OK\r\n\r\n{}").unwrap_err(); - assert!(err.contains("unsupported HTTP version")); - - let err = parse_http_probe_response(b"HTTP/1.1 099 TooLow\r\n\r\n{}").unwrap_err(); - assert!(err.contains("invalid status code")); - } - - #[test] - fn partial_probe_timeout_only_applies_when_bytes_exist() { - let timeout = std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out"); - let would_block = std::io::Error::new(std::io::ErrorKind::WouldBlock, "would block"); - let reset = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"); - - assert!(should_use_partial_probe_response(&timeout, 16)); - assert!(should_use_partial_probe_response(&would_block, 16)); - assert!(!should_use_partial_probe_response(&timeout, 0)); - assert!(!should_use_partial_probe_response(&reset, 16)); - } -} diff --git a/daemon-rs/src/service/mod.rs b/daemon-rs/src/service/mod.rs new file mode 100644 index 00000000..966a38ab --- /dev/null +++ b/daemon-rs/src/service/mod.rs @@ -0,0 +1,153 @@ +#[cfg(windows)] +const SERVICE_NAME: &str = "CortexDaemon"; +#[cfg(windows)] +const DISPLAY_NAME: &str = "Cortex Memory Daemon"; +#[cfg(windows)] +const DESCRIPTION: &str = "Always-on AI memory daemon -- serves Claude, Gemini, Codex, Cursor, and local LLMs via HTTP (:7437) and MCP."; + +#[cfg(windows)] +fn run_sc(args: &[&str], success: &str, tolerated: &[&str]) -> bool { + use std::os::windows::process::CommandExt; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + let mut command = std::process::Command::new("sc.exe"); + command.args(args).creation_flags(CREATE_NO_WINDOW); + match command.output() { + Ok(output) if output.status.success() => { + eprintln!("{success}"); + true + } + Ok(output) => { + let text = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + if tolerated.iter().any(|needle| text.contains(needle)) { + eprintln!("{success}"); + true + } else { + eprintln!("{text}"); + false + } + } + Err(err) => { + eprintln!("[cortex] Failed to run sc.exe: {err}"); + false + } + } +} + +#[cfg(not(windows))] +fn unsupported() -> bool { + eprintln!("[cortex] Windows service management is only available on Windows"); + false +} + +#[cfg(windows)] +pub fn install() -> bool { + let Ok(exe) = std::env::current_exe() else { + eprintln!("[cortex] Failed to get current executable path"); + return false; + }; + let exe = exe.to_string_lossy(); + let bin_path = format!("\"{exe}\" service-run"); + let ok = run_sc( + &["create", SERVICE_NAME, "binPath=", &bin_path, "start=", "demand", "DisplayName=", DISPLAY_NAME], + &format!("[cortex] Service '{SERVICE_NAME}' installed"), + &["1073"], + ); + if ok { + let _ = run_sc(&["description", SERVICE_NAME, DESCRIPTION], "", &[]); + } + ok +} + +#[cfg(not(windows))] +pub fn install() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn uninstall() -> bool { + let _ = run_sc(&["stop", SERVICE_NAME], "[cortex] Service stopped", &["1062"]); + run_sc(&["delete", SERVICE_NAME], "[cortex] Service uninstalled", &[]) +} + +#[cfg(not(windows))] +pub fn uninstall() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn start() -> bool { + run_sc(&["start", SERVICE_NAME], "[cortex] Service started", &["1056"]) +} + +#[cfg(not(windows))] +pub fn start() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn stop() -> bool { + run_sc(&["stop", SERVICE_NAME], "[cortex] Service stopped", &["1062"]) +} + +#[cfg(not(windows))] +pub fn stop() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn status() -> bool { + run_sc(&["query", SERVICE_NAME], "[cortex] Service status queried", &[]) +} + +#[cfg(not(windows))] +pub fn status() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn ensure() -> bool { + status() || install() && start() +} + +#[cfg(not(windows))] +pub fn ensure() -> bool { + unsupported() +} + +#[cfg(windows)] +pub fn ensure_ready() -> bool { + ensure() +} + +#[cfg(windows)] +mod scm { + use std::ffi::OsString; + use windows_service::define_windows_service; + use windows_service::service_dispatcher; + + define_windows_service!(ffi_service_main, cortex_service_main); + + pub fn dispatch() { + if let Err(err) = service_dispatcher::start(super::SERVICE_NAME, ffi_service_main) { + eprintln!("[cortex] Failed to start service dispatcher: {err}"); + std::process::exit(1); + } + } + + fn cortex_service_main(_arguments: Vec) { + match tokio::runtime::Runtime::new() { + Ok(rt) => rt.block_on(crate::run_daemon(crate::auth::CortexPaths::resolve(), std::future::pending::<()>())), + Err(err) => eprintln!("[cortex-service] Failed to create tokio runtime: {err}"), + } + } +} + +#[cfg(windows)] +pub fn dispatch_service() { + scm::dispatch(); +} + +#[cfg(not(windows))] +pub fn dispatch_service() { + eprintln!("[cortex] service-run is only available on Windows"); +} diff --git a/daemon-rs/src/service/tests/mod.rs b/daemon-rs/src/service/tests/mod.rs new file mode 100644 index 00000000..05edd935 --- /dev/null +++ b/daemon-rs/src/service/tests/mod.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +use std::time::{SystemTime, UNIX_EPOCH}; +fn temp_test_dir(name: &str) -> std::path::PathBuf { + let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos(); + std::env::temp_dir().join(format!("cortex_service_{name}_{unique}")) +} +#[test] +fn build_sc_create_command_defaults_to_manual_start() { + let cmd = build_sc_create_command(r"C:\Program Files\Cortex\cortex.exe", "alice"); + assert!(cmd.contains("start= demand"), "expected manual start mode: {cmd}"); + assert!(!cmd.contains("start= auto"), "must not auto-start by default: {cmd}"); +} +#[test] +fn build_sc_create_command_includes_quoted_binpath_and_user() { + let exe = r"C:\Program Files\Cortex\cortex.exe"; + let cmd = build_sc_create_command(exe, "alice"); + let expected_bin = format!("binPath= \"\\\"{}\\\" service-run\"", exe); + assert!(cmd.contains(&format!("sc.exe create {}", SERVICE_NAME))); + assert!(cmd.contains(&expected_bin), "missing binPath quoting: {cmd}"); + assert!(cmd.contains("obj= \".\\alice\""), "missing user account object: {cmd}"); +} +#[test] +fn build_sc_create_command_escapes_cmd_expansion_in_exe_path() { + let cmd = build_sc_create_command(r"C:\Tools\%PATH%\Cortex^Bin\cortex.exe", "alice"); + assert!(cmd.contains(r"C:\Tools\^%PATH^%\Cortex^^Bin\cortex.exe"), "executable path must survive cmd.exe parsing without expansion: {cmd}"); +} +#[test] +fn username_is_safe_for_cmd_fragment_rejects_shell_metacharacters() { + assert!(username_is_safe_for_cmd_fragment("alice")); + assert!(username_is_safe_for_cmd_fragment("alice.svc")); + assert!(username_is_safe_for_cmd_fragment("alice svc")); + assert!(!username_is_safe_for_cmd_fragment("alice&whoami")); + assert!(!username_is_safe_for_cmd_fragment("alice|powershell")); + assert!(!username_is_safe_for_cmd_fragment("alice%PATH%")); + assert!(!username_is_safe_for_cmd_fragment("alice\"quoted")); +} +#[test] +fn resolve_service_username_from_env_falls_back_when_username_is_unsafe() { + let _env_lock = crate::test_env::lock(); + let _username = crate::test_env::ScopedEnvVar::set("USERNAME", "alice&whoami"); + assert_eq!(resolve_service_username_from_env(), "cortex-user"); + std::env::set_var("USERNAME", "alice.svc"); + assert_eq!(resolve_service_username_from_env(), "alice.svc"); + std::env::remove_var("USERNAME"); + assert_eq!(resolve_service_username_from_env(), "cortex-user"); +} +#[test] +fn service_exe_path_from_result_reports_resolution_failure() { + let err = service_exe_path_from_result(Err(std::io::Error::new(std::io::ErrorKind::NotFound, "missing exe"))).unwrap_err(); + assert!(err.contains("Failed to get exe path"), "expected contextual error: {err}"); +} +#[test] +#[cfg(windows)] +fn parse_service_state_recognizes_known_states() { + assert_eq!(parse_service_state("STATE : 4 RUNNING"), ServiceState::Running); + assert_eq!(parse_service_state("STATE : 1 STOPPED"), ServiceState::Stopped); + assert_eq!(parse_service_state("STATE : 2 START_PENDING"), ServiceState::StartPending); + assert_eq!(parse_service_state("STATE : 3 STOP_PENDING"), ServiceState::StopPending); + assert_eq!(parse_service_state("STATE : ???"), ServiceState::Unknown); +} +#[test] +#[cfg(windows)] +fn service_state_strings_are_stable() { + assert_eq!(ServiceState::NotInstalled.as_str(), "NOT_INSTALLED"); + assert_eq!(ServiceState::Running.as_str(), "RUNNING"); + assert_eq!(ServiceState::Stopped.as_str(), "STOPPED"); + assert_eq!(ServiceState::StartPending.as_str(), "START_PENDING"); + assert_eq!(ServiceState::StopPending.as_str(), "STOP_PENDING"); + assert_eq!(ServiceState::Unknown.as_str(), "UNKNOWN"); +} +#[test] +fn daemon_ready_payload_accepts_readiness_ready_and_health_ok() { + let home_dir = temp_test_dir("ready_payload"); + let home = home_dir.to_string_lossy().to_string(); + let paths = crate::auth::CortexPaths::resolve_with_overrides(Some(&home), None, Some(7437), Some("127.0.0.1")); + let readiness = serde_json::json!({ + "status": "ready", + "ready": true, + "runtime": { + "port": 7437, + "token_path": paths.token.display().to_string(), + "db_path": paths.db.display().to_string(), + "pid_path": paths.pid.display().to_string(), + }, + "stats": { "home": paths.home.display().to_string() } + }) + .to_string(); + assert_eq!(daemon_ready_from_payload(200, &readiness, &paths), Some(true)); + let health = serde_json::json!({ + "status": "ok", + "runtime": { + "port": 7437, + "token_path": paths.token.display().to_string(), + "db_path": paths.db.display().to_string(), + "pid_path": paths.pid.display().to_string(), + }, + "stats": { "home": paths.home.display().to_string(), "memories": 1 } + }) + .to_string(); + assert_eq!(daemon_ready_from_payload(200, &health, &paths), Some(true)); +} +#[test] +fn daemon_ready_payload_preserves_starting_state() { + let home_dir = temp_test_dir("starting_payload"); + let home = home_dir.to_string_lossy().to_string(); + let paths = crate::auth::CortexPaths::resolve_with_overrides(Some(&home), None, Some(7437), Some("127.0.0.1")); + let readiness = serde_json::json!({ + "status": "starting", + "ready": false, + "runtime": { + "port": 7437, + "token_path": paths.token.display().to_string(), + "db_path": paths.db.display().to_string(), + "pid_path": paths.pid.display().to_string(), + }, + "stats": { "home": paths.home.display().to_string() } + }) + .to_string(); + assert_eq!(daemon_ready_from_payload(503, &readiness, &paths), Some(false)); +} +#[test] +fn parse_http_probe_response_extracts_status_and_body() { + let raw = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"status\":\"ok\"}"; + let (status, body) = parse_http_probe_response(raw).expect("parse response"); + assert_eq!(status, 200); + assert_eq!(body, "{\"status\":\"ok\"}"); +} +#[test] +fn parse_http_probe_response_rejects_invalid_payloads() { + let err = parse_http_probe_response(b"not-http").unwrap_err(); + assert!(err.contains("invalid HTTP response")); + let err = parse_http_probe_response(b"not-http 200 OK\r\n\r\n{}").unwrap_err(); + assert!(err.contains("unsupported HTTP version")); + let err = parse_http_probe_response(b"HTTP/1.1 099 TooLow\r\n\r\n{}").unwrap_err(); + assert!(err.contains("invalid status code")); +} +#[test] +fn partial_probe_timeout_only_applies_when_bytes_exist() { + let timeout = std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out"); + let would_block = std::io::Error::new(std::io::ErrorKind::WouldBlock, "would block"); + let reset = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"); + assert!(should_use_partial_probe_response(&timeout, 16)); + assert!(should_use_partial_probe_response(&would_block, 16)); + assert!(!should_use_partial_probe_response(&timeout, 0)); + assert!(!should_use_partial_probe_response(&reset, 16)); +} diff --git a/daemon-rs/src/setup/configure.rs b/daemon-rs/src/setup/configure.rs index 0ea85f47..cba548f9 100644 --- a/daemon-rs/src/setup/configure.rs +++ b/daemon-rs/src/setup/configure.rs @@ -1,23 +1,15 @@ -// SPDX-License-Identifier: MIT +use super::types::{ConfigMethod, DetectedTool, StepResult}; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use serde_json::Value; - -use super::helpers::{copy_if_changed, stable_mcp_binary_path}; -use super::types::{ConfigMethod, DetectedTool, StepResult}; - pub(crate) fn step_configure(tools: &[DetectedTool], cortex_exe: &str) -> Vec<(&'static str, StepResult)> { let mut results = Vec::new(); - for tool in tools { let result = configure_tool(tool, cortex_exe); results.push((tool.name, result)); } - results } - fn configure_tool(tool: &DetectedTool, cortex_exe: &str) -> StepResult { match &tool.config_method { ConfigMethod::JsonMerge => { @@ -38,57 +30,30 @@ fn configure_tool(tool: &DetectedTool, cortex_exe: &str) -> StepResult { Err(e) => StepResult::Warn(format!("Auto-config failed: {e}. Configure manually.")), } } - ConfigMethod::CliCommand { program, args } => { - match run_mcp_add(program, args, cortex_exe, tool.agent_name) { - Ok(()) => StepResult::Ok("Registered via CLI".into()), - Err(e) => StepResult::Warn(format!( - "CLI failed: {e}. Run manually: {} {} {cortex_exe} mcp --agent {}", - program, - args.join(" "), - tool.agent_name - )), - } - } - ConfigMethod::Manual(instructions) => { - StepResult::Ok(format!("Manual setup needed: {instructions}")) - } + ConfigMethod::CliCommand { program, args } => match run_mcp_add(program, args, cortex_exe, tool.agent_name) { + Ok(()) => StepResult::Ok("Registered via CLI".into()), + Err(e) => StepResult::Warn(format!("CLI failed: {e}. Run manually: {} {} {cortex_exe} mcp --agent {}", program, args.join(" "), tool.agent_name)), + }, + ConfigMethod::Manual(instructions) => StepResult::Ok(format!("Manual setup needed: {instructions}")), } } - -/// Merge a Cortex MCP server entry into a JSON config file. -/// Reads existing config, adds/updates the "cortex" entry under "mcpServers", -/// writes back. Preserves all existing config. -pub(crate) fn merge_mcp_config( - config_path: &Path, - cortex_exe: &str, - agent_name: &str, -) -> Result { +pub(crate) fn merge_mcp_config(config_path: &Path, cortex_exe: &str, agent_name: &str) -> Result { let original: serde_json::Value = if config_path.exists() { - let content = fs::read_to_string(config_path) - .map_err(|e| format!("Cannot read {}: {e}", config_path.display()))?; - serde_json::from_str(&content) - .map_err(|e| format!("Invalid JSON in {}: {e}", config_path.display()))? + let content = fs::read_to_string(config_path).map_err(|e| format!("Cannot read {}: {e}", config_path.display()))?; + serde_json::from_str(&content).map_err(|e| format!("Invalid JSON in {}: {e}", config_path.display()))? } else { serde_json::json!({}) }; let mut config = original.clone(); - let mcp_servers = config .as_object_mut() .ok_or("Config is not a JSON object")? .entry("mcpServers") .or_insert_with(|| serde_json::json!({})); - let exe_path = PathBuf::from(cortex_exe).to_string_lossy().to_string(); - let desired_registration = serde_json::json!({ - "command": exe_path, - "args": ["mcp", "--agent", agent_name] - }); - mcp_servers - .as_object_mut() - .ok_or("mcpServers is not a JSON object")? - .insert("cortex".to_string(), desired_registration); - + let desired_registration = serde_json::json!({"command": +exe_path,"args":["mcp","--agent",agent_name]}); + mcp_servers.as_object_mut().ok_or("mcpServers is not a JSON object")?.insert("cortex".to_string(), desired_registration); let action = if config == original { "Already configured" } else if config_path.exists() { @@ -96,60 +61,30 @@ pub(crate) fn merge_mcp_config( } else { "Configured" }; - if config != original { if let Some(parent) = config_path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("Cannot create {}: {e}", parent.display()))?; + fs::create_dir_all(parent).map_err(|e| format!("Cannot create {}: {e}", parent.display()))?; } - let output = serde_json::to_string_pretty(&config) - .map_err(|e| format!("JSON serialize failed: {e}"))?; - fs::write(config_path, output) - .map_err(|e| format!("Cannot write {}: {e}", config_path.display()))?; + let output = serde_json::to_string_pretty(&config).map_err(|e| format!("JSON serialize failed: {e}"))?; + fs::write(config_path, output).map_err(|e| format!("Cannot write {}: {e}", config_path.display()))?; } - Ok(format!("{action} at {}", config_path.display())) } - -pub(crate) fn merge_toml_config( - config_path: &Path, - cortex_exe: &str, - agent_name: &str, -) -> Result { +pub(crate) fn merge_toml_config(config_path: &Path, cortex_exe: &str, agent_name: &str) -> Result { let original: toml::Value = if config_path.exists() { - let content = fs::read_to_string(config_path) - .map_err(|e| format!("Cannot read {}: {e}", config_path.display()))?; - toml::from_str(&content) - .map_err(|e| format!("Invalid TOML in {}: {e}", config_path.display()))? + let content = fs::read_to_string(config_path).map_err(|e| format!("Cannot read {}: {e}", config_path.display()))?; + toml::from_str(&content).map_err(|e| format!("Invalid TOML in {}: {e}", config_path.display()))? } else { toml::Value::Table(Default::default()) }; let mut config = original.clone(); - let root = config.as_table_mut().ok_or("Config is not a TOML table")?; - let servers = root - .entry("mcp_servers") - .or_insert_with(|| toml::Value::Table(Default::default())); - let servers_table = servers - .as_table_mut() - .ok_or("mcp_servers is not a TOML table")?; - + let servers = root.entry("mcp_servers").or_insert_with(|| toml::Value::Table(Default::default())); + let servers_table = servers.as_table_mut().ok_or("mcp_servers is not a TOML table")?; let mut server = toml::map::Map::new(); - server.insert( - "command".into(), - toml::Value::String(PathBuf::from(cortex_exe).to_string_lossy().to_string()), - ); - server.insert( - "args".into(), - toml::Value::Array( - ["mcp", "--agent", agent_name] - .into_iter() - .map(|value| toml::Value::String(value.to_string())) - .collect(), - ), - ); + server.insert("command".into(), toml::Value::String(PathBuf::from(cortex_exe).to_string_lossy().to_string())); + server.insert("args".into(), toml::Value::Array(["mcp", "--agent", agent_name].into_iter().map(|value| toml::Value::String(value.to_string())).collect())); servers_table.insert("cortex".into(), toml::Value::Table(server)); - let action = if config == original { "Already configured" } else if config_path.exists() { @@ -157,38 +92,25 @@ pub(crate) fn merge_toml_config( } else { "Configured" }; - if config != original { if let Some(parent) = config_path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("Cannot create {}: {e}", parent.display()))?; + fs::create_dir_all(parent).map_err(|e| format!("Cannot create {}: {e}", parent.display()))?; } - let output = - toml::to_string_pretty(&config).map_err(|e| format!("TOML serialize failed: {e}"))?; - fs::write(config_path, output) - .map_err(|e| format!("Cannot write {}: {e}", config_path.display()))?; + let output = toml::to_string_pretty(&config).map_err(|e| format!("TOML serialize failed: {e}"))?; + fs::write(config_path, output).map_err(|e| format!("Cannot write {}: {e}", config_path.display()))?; } - Ok(format!("{action} at {}", config_path.display())) } - -fn run_mcp_add( - program: &str, - args: &[&str], - cortex_exe: &str, - agent_name: &str, -) -> Result<(), String> { +fn run_mcp_add(program: &str, args: &[&str], cortex_exe: &str, agent_name: &str) -> Result<(), String> { let output = Command::new(program) .args(args) .args([cortex_exe, "mcp", "--agent", agent_name]) .output() .map_err(|e| format!("Failed to run {program} CLI: {e}"))?; - if output.status.success() { Ok(()) } else { let stderr = String::from_utf8_lossy(&output.stderr); - // "already exists" is not an error if stderr.contains("already exists") || stderr.contains("Already") { Ok(()) } else { @@ -196,32 +118,17 @@ fn run_mcp_add( } } } - pub(crate) fn summarize_configs(results: &[(&str, StepResult)]) -> StepResult { if results.is_empty() { return StepResult::Warn("No tools to configure".into()); } - let ok_count = results - .iter() - .filter(|(_, r)| matches!(r, StepResult::Ok(_))) - .count(); - let warn_count = results - .iter() - .filter(|(_, r)| matches!(r, StepResult::Warn(_))) - .count(); - let fail_count = results - .iter() - .filter(|(_, r)| matches!(r, StepResult::Fail(_))) - .count(); - + let ok_count = results.iter().filter(|(_, r)| matches!(r, StepResult::Ok(_))).count(); + let warn_count = results.iter().filter(|(_, r)| matches!(r, StepResult::Warn(_))).count(); + let fail_count = results.iter().filter(|(_, r)| matches!(r, StepResult::Fail(_))).count(); if fail_count > 0 { - StepResult::Warn(format!( - "{ok_count} configured, {warn_count} warnings, {fail_count} failed" - )) + StepResult::Warn(format!("{ok_count} configured, {warn_count} warnings, {fail_count} failed")) } else if warn_count > 0 { - StepResult::Warn(format!( - "{ok_count} configured, {warn_count} need manual setup" - )) + StepResult::Warn(format!("{ok_count} configured, {warn_count} need manual setup")) } else { StepResult::Ok(format!("{ok_count}/{} tools configured", results.len())) } diff --git a/daemon-rs/src/setup/detect.rs b/daemon-rs/src/setup/detect.rs index 3b653794..97f75e9c 100644 --- a/daemon-rs/src/setup/detect.rs +++ b/daemon-rs/src/setup/detect.rs @@ -1,13 +1,8 @@ -// SPDX-License-Identifier: MIT +use super::types::{ConfigMethod, DetectedTool}; use std::path::PathBuf; use std::process::Command; - -use super::types::{ConfigMethod, DetectedTool}; - pub(crate) fn step_detect() -> Vec { let mut found = Vec::new(); - - // Claude Code if let Some(config_path) = find_claude_code_config() { found.push(DetectedTool { name: "Claude Code", @@ -20,14 +15,9 @@ pub(crate) fn step_detect() -> Vec { name: "Claude Code", agent_name: "claude", config_path: None, - config_method: ConfigMethod::CliCommand { - program: "claude", - args: &["mcp", "add", "cortex", "-s", "user", "--"], - }, + config_method: ConfigMethod::CliCommand { program: "claude", args: &["mcp", "add", "cortex", "-s", "user", "--"] }, }); } - - // Claude Desktop if let Some(config_path) = find_claude_desktop_config() { found.push(DetectedTool { name: "Claude Desktop", @@ -36,8 +26,6 @@ pub(crate) fn step_detect() -> Vec { config_method: ConfigMethod::JsonMerge, }); } - - // Codex if let Some(config_path) = find_codex_config() { found.push(DetectedTool { name: "Codex CLI", @@ -50,14 +38,9 @@ pub(crate) fn step_detect() -> Vec { name: "Codex CLI", agent_name: "codex", config_path: None, - config_method: ConfigMethod::CliCommand { - program: "codex", - args: &["mcp", "add", "cortex", "--"], - }, + config_method: ConfigMethod::CliCommand { program: "codex", args: &["mcp", "add", "cortex", "--"] }, }); } - - // Cursor if let Some(config_path) = find_cursor_config() { found.push(DetectedTool { name: "Cursor", @@ -66,8 +49,6 @@ pub(crate) fn step_detect() -> Vec { config_method: ConfigMethod::JsonMerge, }); } - - // Windsurf if let Some(config_path) = find_windsurf_config() { found.push(DetectedTool { name: "Windsurf", @@ -76,84 +57,54 @@ pub(crate) fn step_detect() -> Vec { config_method: ConfigMethod::JsonMerge, }); } - found } - fn find_claude_desktop_config() -> Option { find_first_config_path(claude_desktop_config_paths()) } - fn find_claude_code_config() -> Option { let home = dirs::home_dir()?; find_existing_config(home.join(".claude").join("settings.json")) } - fn find_codex_config() -> Option { let home = dirs::home_dir()?; find_existing_config(home.join(".codex").join("config.toml")) } - fn claude_desktop_config_paths() -> Vec { let mut paths = Vec::new(); - #[cfg(windows)] { if let Ok(appdata) = std::env::var("APPDATA") { - paths.push( - PathBuf::from(appdata) - .join("Claude") - .join("claude_desktop_config.json"), - ); + paths.push(PathBuf::from(appdata).join("Claude").join("claude_desktop_config.json")); } } - #[cfg(target_os = "macos")] { if let Some(home) = dirs::home_dir() { - paths.push( - home.join("Library") - .join("Application Support") - .join("Claude") - .join("claude_desktop_config.json"), - ); + paths.push(home.join("Library").join("Application Support").join("Claude").join("claude_desktop_config.json")); } } - #[cfg(target_os = "linux")] { if let Ok(config) = std::env::var("XDG_CONFIG_HOME") { - paths.push( - PathBuf::from(config) - .join("Claude") - .join("claude_desktop_config.json"), - ); + paths.push(PathBuf::from(config).join("Claude").join("claude_desktop_config.json")); } else if let Some(home) = dirs::home_dir() { - paths.push( - home.join(".config") - .join("Claude") - .join("claude_desktop_config.json"), - ); + paths.push(home.join(".config").join("Claude").join("claude_desktop_config.json")); } } - paths } - fn find_cursor_config() -> Option { let home = dirs::home_dir()?; find_existing_config(home.join(".cursor").join("mcp.json")) } - fn find_windsurf_config() -> Option { let home = dirs::home_dir()?; find_existing_config(home.join(".windsurf").join("mcp.json")) } - fn find_first_config_path(paths: Vec) -> Option { paths.into_iter().find_map(find_existing_config) } - pub(crate) fn find_existing_config(path: PathBuf) -> Option { if path.exists() || path.parent().is_some_and(|p| p.exists()) { Some(path) @@ -161,23 +112,13 @@ pub(crate) fn find_existing_config(path: PathBuf) -> Option { None } } - fn command_exists(cmd: &str) -> bool { #[cfg(windows)] { - Command::new("where") - .arg(cmd) - .output() - .map(|o| o.status.success()) - .unwrap_or(false) + Command::new("where").arg(cmd).output().map(|o| o.status.success()).unwrap_or(false) } - #[cfg(not(windows))] { - Command::new("which") - .arg(cmd) - .output() - .map(|o| o.status.success()) - .unwrap_or(false) + Command::new("which").arg(cmd).output().map(|o| o.status.success()).unwrap_or(false) } } diff --git a/daemon-rs/src/setup/helpers.rs b/daemon-rs/src/setup/helpers.rs index f6f34e2b..145ea8a8 100644 --- a/daemon-rs/src/setup/helpers.rs +++ b/daemon-rs/src/setup/helpers.rs @@ -1,32 +1,23 @@ -// SPDX-License-Identifier: MIT -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; - +use super::types::StepResult; use crate::auth; use crate::db; - -use super::types::StepResult; - +use std::fs; +use std::path::{Path, PathBuf}; pub(crate) fn daemon_port() -> u16 { auth::CortexPaths::resolve().port } - pub(crate) fn daemon_base_url() -> String { format!("http://localhost:{}", daemon_port()) } - pub(crate) fn daemon_url(path: &str) -> String { format!("{}{}", daemon_base_url(), path) } pub(crate) fn rollback_team_setup(conn: &rusqlite::Connection) { let _ = conn.execute_batch("ROLLBACK"); } - pub(crate) fn persist_team_owner_token(paths: &auth::CortexPaths, owner_key: &str) -> Result<(), String> { auth::try_write_token_for(paths, owner_key) } - pub(crate) fn restore_previous_token(paths: &auth::CortexPaths, previous_token: Option>) { match previous_token { Some(contents) => { @@ -40,71 +31,41 @@ pub(crate) fn restore_previous_token(paths: &auth::CortexPaths, previous_token: } } } - pub(crate) fn print_step(num: usize, name: &str, result: &StepResult) { - eprintln!( - " {} Step {}: {} -- {}", - result.icon(), - num, - name, - result.message() - ); + eprintln!(" {} Step {}: {} -- {}", result.icon(), num, name, result.message()); } - fn current_exe_path() -> String { - std::env::current_exe() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|_| "cortex".to_string()) + std::env::current_exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_else(|_| "cortex".to_string()) } - pub(crate) fn copy_if_changed(src: &Path, dest: &Path) -> Result<(), String> { let needs_copy = match fs::read(dest) { - Ok(existing) => { - existing != fs::read(src).map_err(|e| format!("Cannot read {}: {e}", src.display()))? - } + Ok(existing) => existing != fs::read(src).map_err(|e| format!("Cannot read {}: {e}", src.display()))?, Err(err) if err.kind() == std::io::ErrorKind::NotFound => true, Err(err) => return Err(format!("Cannot read {}: {err}", dest.display())), }; - if needs_copy { - fs::copy(src, dest) - .map_err(|e| format!("Cannot copy {} to {}: {e}", src.display(), dest.display()))?; + fs::copy(src, dest).map_err(|e| format!("Cannot copy {} to {}: {e}", src.display(), dest.display()))?; } - Ok(()) } - pub(crate) fn stable_mcp_binary_path() -> String { let current = PathBuf::from(current_exe_path()); - let installed = auth::cortex_dir().join("bin").join(if cfg!(windows) { - "cortex.exe" - } else { - "cortex" - }); - + let installed = auth::cortex_dir().join("bin").join(if cfg!(windows) { "cortex.exe" } else { "cortex" }); if current == installed { return installed.to_string_lossy().to_string(); } - if let Some(parent) = installed.parent() { if let Err(err) = fs::create_dir_all(parent) { - eprintln!( - " [!!] Failed to create stable MCP binary dir {}: {}", - parent.display(), - err - ); + eprintln!(" [!!] Failed to create stable MCP binary dir {}: {}", parent.display(), err); return current.to_string_lossy().to_string(); } } - if let Err(err) = copy_if_changed(¤t, &installed) { eprintln!(" [!!] Failed to refresh stable MCP binary: {err}"); return current.to_string_lossy().to_string(); } - installed.to_string_lossy().to_string() } - pub(crate) fn arg_value(args: &[String], key: &str) -> Option { for (idx, arg) in args.iter().enumerate() { if arg == key { @@ -113,7 +74,6 @@ pub(crate) fn arg_value(args: &[String], key: &str) -> Option { } None } - pub(crate) fn collect_reembed_backlog_counts(db_path: &Path, model_key: &str) -> Option<(i64, i64)> { if !db_path.exists() { return None; @@ -150,6 +110,3 @@ pub(crate) fn collect_reembed_backlog_counts(db_path: &Path, model_key: &str) -> .ok()?; Some((backlog_memories, backlog_decisions)) } - -// ─── Step 1: Init ─────────────────────────────────────────────────────────── - diff --git a/daemon-rs/src/setup/mod.rs b/daemon-rs/src/setup/mod.rs index 8327479e..bfecf22c 100644 --- a/daemon-rs/src/setup/mod.rs +++ b/daemon-rs/src/setup/mod.rs @@ -1,18 +1,10 @@ -// SPDX-License-Identifier: MIT -//! `cortex setup` -- Beta installer that detects AI tools and configures them. - -mod types; -mod helpers; -mod team; -mod detect; mod configure; +mod detect; +mod helpers; mod steps; - +mod team; #[cfg(test)] -mod tests { - // Setup wizard internals are not release-gated; see Info/testing-philosophy.md. -} - -pub use types::{ConfigMethod, DetectedTool, StepResult}; -pub use team::run_setup_team; +mod tests; +mod types; pub use steps::run_setup; +pub use team::run_setup_team; diff --git a/daemon-rs/src/setup/steps.rs b/daemon-rs/src/setup/steps.rs index 9aa77849..fc343910 100644 --- a/daemon-rs/src/setup/steps.rs +++ b/daemon-rs/src/setup/steps.rs @@ -1,29 +1,18 @@ -// SPDX-License-Identifier: MIT -use std::fs; - -use crate::auth; -use crate::db; -use crate::embeddings; - use super::configure::{step_configure, summarize_configs}; use super::detect::step_detect; -use super::helpers::{ - collect_reembed_backlog_counts, daemon_base_url, daemon_port, daemon_url, print_step, - stable_mcp_binary_path, -}; +use super::helpers::{collect_reembed_backlog_counts, daemon_base_url, daemon_port, daemon_url, print_step, stable_mcp_binary_path}; use super::types::StepResult; - +use crate::auth; +use crate::embeddings; +use std::fs; pub async fn run_setup() { eprintln!(); eprintln!(" Cortex Setup -- Universal AI Memory"); eprintln!(" ===================================="); eprintln!(); - let init_result = step_init().await; print_step(1, "Initialize", &init_result); - let cortex_exe = stable_mcp_binary_path(); - let detected = step_detect(); print_step( 2, @@ -35,33 +24,19 @@ pub async fn run_setup() { StepResult::Ok(format!("Found: {}", names.join(", "))) }, ); - let config_results = step_configure(&detected, &cortex_exe); print_step(3, "Configure AI tools", &summarize_configs(&config_results)); - for (tool_name, result) in &config_results { - eprintln!( - " {} {}: {}", - result.icon(), - tool_name, - result.message() - ); + eprintln!(" {} {}: {}", result.icon(), tool_name, result.message()); } - let daemon_result = step_daemon().await; print_step(4, "Daemon availability", &daemon_result); - let verify_result = step_verify().await; print_step(5, "Verify", &verify_result); - - // Summary eprintln!(); let token = auth::read_token().unwrap_or_else(|| "???".into()); let token_preview = if token.len() > 8 { &token[..8] } else { &token }; - eprintln!( - " Your API token: {}... (full token in ~/.cortex/cortex.token)", - token_preview - ); + eprintln!(" Your API token: {}... (full token in ~/.cortex/cortex.token)", token_preview); eprintln!(" Daemon: {}", daemon_base_url()); eprintln!(" Health check: curl {}", daemon_url("/health")); eprintln!(" Readiness: curl {}", daemon_url("/readiness")); @@ -76,14 +51,10 @@ async fn step_init() -> StepResult { let rerank_config = crate::rerank::RerankConfig::from_env(); let reranker_model = crate::rerank::selected_reranker_selection(); let mut notes = Vec::new(); - - // Create directory if let Err(e) = fs::create_dir_all(&cortex_dir) { return StepResult::Fail(format!("Cannot create {}: {e}", cortex_dir.display())); } notes.push(format!("Directory: {}", cortex_dir.display())); - - // Generate or reuse token if auth::read_token().is_some() { notes.push("Token: exists (reusing)".into()); } else { @@ -92,113 +63,62 @@ async fn step_init() -> StepResult { } notes.push("Token: generated".into()); } - - // Check ONNX model let models_dir = cortex_dir.join("models"); let model_exists = embeddings::selected_model_assets_exist(&models_dir); - if model_exists { - notes.push(format!( - "Embedding model: ready ({})", - embedding_model.display_name - )); + notes.push(format!("Embedding model: ready ({})", embedding_model.display_name)); } else { - eprintln!( - " Downloading embedding model ({})...", - embedding_model.display_name - ); + eprintln!(" Downloading embedding model ({})...", embedding_model.display_name); match embeddings::ensure_model_downloaded().await { - Some(_) => notes.push(format!( - "Embedding model: downloaded ({})", - embedding_model.display_name - )), - None => { - notes.push("Embedding model: download failed (will retry on daemon start)".into()) - } + Some(_) => notes.push(format!("Embedding model: downloaded ({})", embedding_model.display_name)), + None => notes.push("Embedding model: download failed (will retry on daemon start)".into()), } } - notes.push(format!( "Embedding profile: {} [{} | {}d | {} pooling | {} tokens]", - embedding_model.display_name, - embedding_model.key, - embedding_model.dimension, - embedding_model.pooling, - embedding_model.max_input_tokens + embedding_model.display_name, embedding_model.key, embedding_model.dimension, embedding_model.pooling, embedding_model.max_input_tokens )); - if rerank_config.is_active() { let reranker_exists = crate::rerank::selected_reranker_assets_exist(&models_dir); if reranker_exists { - notes.push(format!( - "Reranker: ready ({} | mode={})", - reranker_model.display_name, - rerank_config.mode.as_str() - )); + notes.push(format!("Reranker: ready ({} | mode={})", reranker_model.display_name, rerank_config.mode.as_str())); } else { - eprintln!( - " Downloading reranker model ({})...", - reranker_model.display_name - ); + eprintln!(" Downloading reranker model ({})...", reranker_model.display_name); match crate::rerank::ensure_reranker_downloaded().await { - Some(_) => notes.push(format!( - "Reranker: downloaded ({} | mode={})", - reranker_model.display_name, - rerank_config.mode.as_str() - )), - None => { - notes.push("Reranker: download failed (rerank will stay unavailable)".into()) - } + Some(_) => notes.push(format!("Reranker: downloaded ({} | mode={})", reranker_model.display_name, rerank_config.mode.as_str())), + None => notes.push("Reranker: download failed (rerank will stay unavailable)".into()), } } } - - if let Some((backlog_memories, backlog_decisions)) = - collect_reembed_backlog_counts(&db_path, embedding_model.key) - { - notes.push(format!( - "Re-embed backlog: memories={backlog_memories}, decisions={backlog_decisions}, total={}", - backlog_memories + backlog_decisions - )); + if let Some((backlog_memories, backlog_decisions)) = collect_reembed_backlog_counts(&db_path, embedding_model.key) { + notes.push(format!("Re-embed backlog: memories={backlog_memories}, decisions={backlog_decisions}, total={}", backlog_memories + backlog_decisions)); notes.push( "Backfill policy: daemon drains backlog in bounded background passes (batch + interval controlled by CORTEX_EMBED_BACKFILL_* env vars)".into(), ); } - StepResult::Ok(notes.join(" | ")) } - async fn step_daemon() -> StepResult { let port = daemon_port(); if is_daemon_healthy().await { return StepResult::Ok(format!("Daemon already running on :{port}")); } - - StepResult::Warn(format!( - "No daemon is running on :{port}. Start Cortex from Control Center or let your client launch `cortex mcp --agent `." - )) + StepResult::Warn(format!("No daemon is running on :{port}. Start Cortex from Control Center or let your client launch `cortex mcp --agent `.")) } - async fn is_daemon_healthy() -> bool { let paths = auth::CortexPaths::resolve(); crate::daemon_lifecycle::daemon_healthy(&paths).await } - -// ─── Step 5: Verify ───────────────────────────────────────────────────────── - async fn step_verify() -> StepResult { if !is_daemon_healthy().await { return StepResult::Warn( - "Skipped live verification because no daemon is currently running. Start Cortex from Control Center or `cortex mcp --agent `, then rerun setup if you want a round-trip check." - .into(), - ); +"Skipped live verification because no daemon is currently running. Start Cortex from Control Center or `cortex mcp --agent `, then rerun setup if you want a round-trip check." +.into(),); } - let token = match auth::read_token() { Some(t) => t, None => return StepResult::Fail("No auth token found".into()), }; - let client = match reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(3)) .timeout(std::time::Duration::from_secs(5)) @@ -207,33 +127,21 @@ async fn step_verify() -> StepResult { Ok(c) => c, Err(e) => return StepResult::Fail(format!("HTTP client error: {e}")), }; - - // Store a test memory let store_resp = client .post(daemon_url("/store")) .header("Authorization", format!("Bearer {token}")) .header("X-Cortex-Request", "true") - .json(&serde_json::json!({ - "decision": "Cortex installed and verified", - "context": "Automated setup verification", - "type": "memory", - "source_agent": "cortex-setup" - })) + .json(&serde_json::json!({"decision" +:"Cortex installed and verified","context":"Automated setup verification","type":"memory","source_agent":"cortex-setup"})) .send() .await; - match store_resp { Ok(r) if r.status().is_success() => {} Ok(r) => { - return StepResult::Warn(format!( - "Store returned {}: daemon is running but store failed", - r.status() - )); + return StepResult::Warn(format!("Store returned {}: daemon is running but store failed", r.status())); } Err(e) => return StepResult::Fail(format!("Cannot reach daemon: {e}")), } - - // Recall it back let recall_resp = client .get(daemon_url("/recall")) .header("Authorization", format!("Bearer {token}")) @@ -241,15 +149,9 @@ async fn step_verify() -> StepResult { .query(&[("q", "Cortex installed"), ("k", "1"), ("budget", "100")]) .send() .await; - match recall_resp { - Ok(r) if r.status().is_success() => { - StepResult::Ok("Store + recall round-trip verified".into()) - } - Ok(r) => StepResult::Warn(format!( - "Recall returned {}: store worked but recall did not", - r.status() - )), + Ok(r) if r.status().is_success() => StepResult::Ok("Store + recall round-trip verified".into()), + Ok(r) => StepResult::Warn(format!("Recall returned {}: store worked but recall did not", r.status())), Err(e) => StepResult::Warn(format!("Recall failed: {e}. Store succeeded.")), } } diff --git a/daemon-rs/src/setup/team.rs b/daemon-rs/src/setup/team.rs index f14028a5..382146e2 100644 --- a/daemon-rs/src/setup/team.rs +++ b/daemon-rs/src/setup/team.rs @@ -1,23 +1,12 @@ -// SPDX-License-Identifier: MIT -use std::fs; -use std::path::Path; - +use super::helpers::{arg_value, persist_team_owner_token, restore_previous_token, rollback_team_setup}; use crate::auth; use crate::db; - -use super::helpers::{ - arg_value, collect_reembed_backlog_counts, persist_team_owner_token, restore_previous_token, - rollback_team_setup, -}; -use super::types::StepResult; - +use std::fs; pub async fn run_setup_team(args: &[String], dry_run: bool) { let db_path = auth::db_path(); if let Some(parent) = db_path.parent() { let _ = fs::create_dir_all(parent); } - - // Open DB early so we can check current mode before prompting. let conn = match db::open(&db_path) { Ok(v) => v, Err(e) => { @@ -35,20 +24,13 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { } db::migrate_focus_table(&conn); crate::crystallize::migrate_crystal_tables(&conn); - - // Idempotency: refuse to re-migrate. if db::is_team_mode(&conn) { eprintln!(); eprintln!(" Already in team mode. No changes needed."); eprintln!(); return; } - - // Resolve owner: flag > interactive prompt > env fallback. - let default_owner = std::env::var("USERNAME") - .or_else(|_| std::env::var("USER")) - .unwrap_or_else(|_| "owner".to_string()); - + let default_owner = std::env::var("USERNAME").or_else(|_| std::env::var("USER")).unwrap_or_else(|_| "owner".to_string()); let owner = if let Some(v) = arg_value(args, "--owner") { v } else { @@ -60,9 +42,7 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { default_owner.clone() } }; - let display_name = arg_value(args, "--display-name").unwrap_or_else(|| owner.clone()); - eprintln!(); if dry_run { eprintln!(" [DRY RUN] Cortex Team Migration Preview"); @@ -74,23 +54,16 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { eprintln!(); eprintln!(" Owner username: {owner}"); eprintln!(); - - // ── Pre-migration backup (skip for dry-run) ──────────────────────────── if !dry_run && db_path.exists() { let bak_path = db_path.with_extension("db.bak"); eprint!(" Backing up database to {}... ", bak_path.display()); - - // Close the connection to release the WAL lock before copying. drop(conn); - if let Err(e) = fs::copy(&db_path, &bak_path) { eprintln!("FAILED"); eprintln!(" [FAIL] Backup failed: {e} -- aborting migration."); return; } eprintln!("done"); - - // Also copy WAL/SHM if they exist (ensures consistent backup). let wal = db_path.with_extension("db-wal"); let shm = db_path.with_extension("db-shm"); if wal.exists() { @@ -100,10 +73,7 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { let _ = fs::copy(&shm, bak_path.with_extension("db.bak-shm")); } } else if !dry_run { - // No DB yet -- nothing to back up, will be created fresh. } - - // Re-open the connection (closed above for backup). let conn = match db::open(&db_path) { Ok(v) => v, Err(e) => { @@ -121,17 +91,12 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { } db::migrate_focus_table(&conn); crate::crystallize::migrate_crystal_tables(&conn); - - // SAFETY: team setup is a write transaction; take SQLite's writer lock - // up front so contention fails/retries at the transaction boundary. if let Err(e) = conn.execute_batch("BEGIN IMMEDIATE") { eprintln!(" [FAIL] Cannot begin transaction: {e}"); return; } - eprintln!(" Migrating to team mode..."); eprintln!(); - let owner_key = auth::generate_ctx_api_key(); let owner_hash = match auth::hash_api_key_argon2id(&owner_key) { Ok(v) => v, @@ -141,7 +106,6 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { return; } }; - eprint!(" Creating team tables... "); if let Err(e) = db::create_team_mode_tables(&conn) { eprintln!("FAILED"); @@ -150,7 +114,6 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { return; } eprintln!("done"); - let owner_id = match db::upsert_owner_user(&conn, &owner, Some(&display_name), &owner_hash) { Ok(v) => v, Err(e) => { @@ -159,7 +122,6 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { return; } }; - eprint!(" Adding ownership columns... "); if let Err(e) = db::migrate_to_team_mode(&conn, owner_id) { eprintln!("FAILED"); @@ -169,51 +131,29 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { } eprintln!("done"); eprintln!(); - - // ── Row count report ─────────────────────────────────────────────────── let counts = db::migration_counts(&conn); let total: i64 = counts.iter().map(|(_, n)| n).sum(); - if dry_run { eprintln!(" [DRY RUN] Would migrate to team mode:"); } else { eprintln!(" Assigned ownership:"); } - let label_width = 22; for (table, count) in &counts { if dry_run { - eprintln!( - " {:6} rows would be assigned", - format!("{table}:"), - count, - width = label_width, - ); + eprintln!(" {:6} rows would be assigned", format!("{table}:"), count, width = label_width,); } else { - eprintln!( - " {:6} rows", - format!("{table}:"), - count, - width = label_width, - ); + eprintln!(" {:6} rows", format!("{table}:"), count, width = label_width,); } } - if dry_run { - eprintln!( - " {:6} rows", - "Total:", - total, - width = label_width - ); + eprintln!(" {:6} rows", "Total:", total, width = label_width); eprintln!(); rollback_team_setup(&conn); eprintln!(" No changes made."); eprintln!(); return; } - - // Non-dry-run: finish migration. let default_team_id = match db::ensure_default_team_membership(&conn, owner_id) { Ok(v) => v, Err(e) => { @@ -222,15 +162,11 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { return; } }; - - // Keep the existing auth path compatible with current handlers/MCP proxy. let paths = auth::CortexPaths::resolve(); let previous_token = fs::read(&paths.token).ok(); if let Err(e) = persist_team_owner_token(&paths, &owner_key) { rollback_team_setup(&conn); - eprintln!( - " [FAIL] Team migration rolled back because owner token persistence failed: {e}" - ); + eprintln!(" [FAIL] Team migration rolled back because owner token persistence failed: {e}"); return; } if let Err(e) = conn.execute_batch("COMMIT") { @@ -239,16 +175,9 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { eprintln!(" [FAIL] Failed to commit team migration: {e}"); return; } - let key_preview: String = owner_key.chars().take(8).collect(); - eprintln!(" ────────────────────────────"); - eprintln!( - " {:6} rows -> owner \"{owner}\" (id: {owner_id})", - "Total:", - total, - width = label_width, - ); + eprintln!(" {:6} rows -> owner \"{owner}\" (id: {owner_id})", "Total:", total, width = label_width,); eprintln!(); eprintln!(" All rows set to visibility: private"); eprintln!(); @@ -261,4 +190,3 @@ pub async fn run_setup_team(args: &[String], dry_run: bool) { eprintln!(" Migration complete. Restart daemon: cortex serve"); eprintln!(); } - diff --git a/daemon-rs/src/setup/tests/mod.rs b/daemon-rs/src/setup/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/setup/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/setup/types.rs b/daemon-rs/src/setup/types.rs index e6ccaeec..11e56318 100644 --- a/daemon-rs/src/setup/types.rs +++ b/daemon-rs/src/setup/types.rs @@ -1,8 +1,4 @@ -// SPDX-License-Identifier: MIT use std::path::PathBuf; - -// ─── Types ────────────────────────────────────────────────────────────────── - #[derive(Debug, Clone)] pub struct DetectedTool { pub name: &'static str, @@ -10,30 +6,23 @@ pub struct DetectedTool { pub config_path: Option, pub config_method: ConfigMethod, } - #[derive(Debug, Clone)] pub enum ConfigMethod { - /// Write MCP server entry to a JSON config file JsonMerge, - /// Write MCP server entry to a TOML config file TomlMerge, - /// Run a CLI command (e.g., `claude mcp add`) CliCommand { program: &'static str, args: &'static [&'static str], }, - /// Show manual instructions to the user #[allow(dead_code)] Manual(String), } - #[derive(Debug)] pub enum StepResult { Ok(String), Warn(String), Fail(String), } - impl StepResult { pub(crate) fn icon(&self) -> &str { match self { @@ -42,7 +31,6 @@ impl StepResult { StepResult::Fail(_) => "[FAIL]", } } - pub(crate) fn message(&self) -> &str { match self { StepResult::Ok(m) | StepResult::Warn(m) | StepResult::Fail(m) => m, diff --git a/daemon-rs/src/state/init.rs b/daemon-rs/src/state/init.rs index 35386d09..fd7dd112 100644 --- a/daemon-rs/src/state/init.rs +++ b/daemon-rs/src/state/init.rs @@ -1,55 +1,25 @@ -// SPDX-License-Identifier: MIT +use super::read_pool::{open_query_only_connection, read_pool_size_from_env, ReadConnectionPool, ReadConnectionProvider}; +use super::runtime::{current_unix_secs, RuntimeState}; +use super::types::{BrainFiringEvent, DaemonEvent, SqliteVecCanaryConfig, SqliteVecRouteMode}; +use crate::auth::CortexPaths; +use rusqlite::Connection; use std::collections::HashMap; -use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::Arc; - -use rusqlite::Connection; use tokio::sync::{broadcast, oneshot, Mutex}; - -use crate::auth::CortexPaths; - -use super::read_pool::{open_query_only_connection, read_pool_size_from_env, ReadConnectionPool, ReadConnectionProvider}; -use super::runtime::{current_unix_secs, RuntimeState}; -use super::types::{ - BrainFiringEvent, DaemonEvent, PreCacheEntry, RecallHistoryEntry, SqliteVecCanaryConfig, - SqliteVecRouteMode, -}; - -pub fn initialize( - paths: &CortexPaths, - allow_token_rotation: bool, -) -> Result<(RuntimeState, oneshot::Receiver<()>), String> { +pub fn initialize(paths: &CortexPaths, allow_token_rotation: bool) -> Result<(RuntimeState, oneshot::Receiver<()>), String> { let db_path = &paths.db; - // 1. Open and configure the database. - let conn = crate::db::open(db_path) - .map_err(|e| format!("Failed to open database at {}: {e}", db_path.display()))?; - + let conn = crate::db::open(db_path).map_err(|e| format!("Failed to open database at {}: {e}", db_path.display()))?; crate::db::configure(&conn).map_err(|e| format!("Failed to configure database: {e}"))?; - crate::db::initialize_schema(&conn).map_err(|e| format!("Failed to initialise schema: {e}"))?; - - // 2. Startup integrity gate. - // Use fast quick_check on boot for low-latency restarts. Only escalate to - // full integrity_check + auto-repair when quick_check fails. if crate::db::quick_check(&conn) { eprintln!("[cortex] DB quick_check: OK"); } else { - eprintln!( - "[cortex] WARNING: PRAGMA quick_check FAILED on {} -- running full integrity_check", - db_path.display() - ); - + eprintln!("[cortex] WARNING: PRAGMA quick_check FAILED on {} -- running full integrity_check", db_path.display()); let integrity_ok = crate::db::verify_integrity(&conn).unwrap_or(false); if !integrity_ok { - eprintln!( - "[cortex] WARNING: PRAGMA integrity_check FAILED on {} -- attempting auto-repair", - db_path.display() - ); - - // Drop the write connection before auto_repair renames the file. + eprintln!("[cortex] WARNING: PRAGMA integrity_check FAILED on {} -- attempting auto-repair", db_path.display()); drop(conn); - let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S").to_string(); match crate::db::auto_repair(db_path, ×tamp) { Ok(result) => { @@ -60,11 +30,8 @@ pub fn initialize( result.decisions_recovered, result.corrupt_db_path.display() ); - // Reopen the repaired DB and continue normal startup. - let conn = crate::db::open(db_path) - .map_err(|e| format!("Failed to open repaired DB: {e}"))?; - crate::db::configure(&conn) - .map_err(|e| format!("Failed to configure repaired DB: {e}"))?; + let conn = crate::db::open(db_path).map_err(|e| format!("Failed to open repaired DB: {e}"))?; + crate::db::configure(&conn).map_err(|e| format!("Failed to configure repaired DB: {e}"))?; return initialize_with_conn(conn, paths, allow_token_rotation); } Err(e) => { @@ -74,20 +41,12 @@ pub fn initialize( DB path: {}", db_path.display() ); - // Reopen whatever DB exists (may be the corrupted original if - // auto_repair failed before the rename step). - let conn = crate::db::open(db_path).map_err(|open_err| { - format!( - "Database corrupt and could not be reopened after failed repair: {open_err}" - ) - })?; + let conn = + crate::db::open(db_path).map_err(|open_err| format!("Database corrupt and could not be reopened after failed repair: {open_err}"))?; crate::db::configure(&conn).ok(); crate::db::initialize_schema(&conn).ok(); let (state, rx) = initialize_with_conn(conn, paths, allow_token_rotation)?; - // Signal degraded mode so /health reflects corruption. - state - .db_corrupted - .store(true, std::sync::atomic::Ordering::SeqCst); + state.db_corrupted.store(true, std::sync::atomic::Ordering::SeqCst); return Ok((state, rx)); } } @@ -95,55 +54,35 @@ pub fn initialize( eprintln!("[cortex] DB integrity: OK (after quick_check failure)"); } } - initialize_with_conn(conn, paths, allow_token_rotation) } - -fn initialize_with_conn( - conn: Connection, - paths: &CortexPaths, - allow_token_rotation: bool, -) -> Result<(RuntimeState, oneshot::Receiver<()>), String> { - // Rebuild FTS indexes only when they appear empty for non-empty source data. +fn initialize_with_conn(conn: Connection, paths: &CortexPaths, allow_token_rotation: bool) -> Result<(RuntimeState, oneshot::Receiver<()>), String> { match crate::db::rebuild_fts_if_needed(&conn) { Ok(true) => eprintln!("[cortex] FTS baseline rebuilt"), Ok(false) => {} Err(e) => eprintln!("[cortex] WARNING: FTS rebuild check failed: {e}"), } - - // Open a small query-only read pool so bursty read load does not queue on a - // single async mutex. let read_pool_size = read_pool_size_from_env(); let mut read_connections = Vec::with_capacity(read_pool_size); for _ in 0..read_pool_size { read_connections.push(open_query_only_connection(&paths.db)?); } - let db_read: Arc = - Arc::new(ReadConnectionPool::new(read_connections)); + let db_read: Arc = Arc::new(ReadConnectionPool::new(read_connections)); eprintln!( "[cortex] Read pool opened with {} query-only connection{} (WAL concurrent reads enabled)", db_read.pool_size(), if db_read.pool_size() == 1 { "" } else { "s" } ); - let mode = crate::db::current_mode(&conn); let team_mode = mode == "team"; let default_owner_id = if team_mode { let from_config = conn - .query_row( - "SELECT value FROM config WHERE key = 'owner_user_id' LIMIT 1", - [], - |row| row.get::<_, String>(0), - ) + .query_row("SELECT value FROM config WHERE key = 'owner_user_id' LIMIT 1", [], |row| row.get::<_, String>(0)) .ok() .and_then(|v| v.parse::().ok()); from_config.or_else(|| { - conn.query_row( - "SELECT id FROM users ORDER BY CASE role WHEN 'owner' THEN 0 ELSE 1 END, id ASC LIMIT 1", - [], - |row| row.get::<_, i64>(0), - ) - .ok() + conn.query_row("SELECT id FROM users ORDER BY CASE role WHEN 'owner' THEN 0 ELSE 1 END, id ASC LIMIT 1", [], |row| row.get::<_, i64>(0)) + .ok() }) } else { None @@ -151,9 +90,7 @@ fn initialize_with_conn( let team_api_key_hashes = if team_mode { let mut hashes: Vec<(i64, String)> = Vec::new(); if let Ok(mut stmt) = conn.prepare("SELECT id, api_key_hash FROM users") { - if let Ok(rows) = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - }) { + if let Ok(rows) = stmt.query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))) { for row in rows.flatten() { hashes.push(row); } @@ -163,45 +100,28 @@ fn initialize_with_conn( } else { Arc::new(std::sync::RwLock::new(Vec::new())) }; - - // Auth token. let token = if team_mode { crate::auth::read_token_from(paths).unwrap_or_else(crate::auth::generate_ephemeral_token) } else if allow_token_rotation { - crate::auth::try_generate_token_for(paths) - .map_err(|e| format!("Failed to generate shared auth token: {e}"))? + crate::auth::try_generate_token_for(paths).map_err(|e| format!("Failed to generate shared auth token: {e}"))? } else { crate::auth::read_token_from(paths).unwrap_or_else(crate::auth::generate_ephemeral_token) }; - - // Channels. let (events_tx, _) = broadcast::channel::(256); let (brain_firing_tx, _) = broadcast::channel::(256); let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let home = paths.home.clone(); let models_dir = paths.models.clone(); let embedding_engine = crate::embeddings::EmbeddingEngine::load(&models_dir).map(Arc::new); - if let Some(engine) = embedding_engine.as_ref() { - eprintln!( - "[cortex] Embedding engine loaded (model={}, {}-dim, in-process ONNX)", - engine.model_key(), - engine.dimension() - ); + eprintln!("[cortex] Embedding engine loaded (model={}, {}-dim, in-process ONNX)", engine.model_key(), engine.dimension()); } else { - eprintln!( - "[cortex] Embedding engine not available -- keyword search only until model downloaded" - ); + eprintln!("[cortex] Embedding engine not available -- keyword search only until model downloaded"); } - let write_buffer_path = paths.write_buffer.clone(); let sqlite_vec_canary = SqliteVecCanaryConfig::from_env(); if sqlite_vec_canary.force_off { - eprintln!( - "[cortex] sqlite-vec routing force-off (configured mode={}, effective mode=baseline)", - sqlite_vec_canary.route_mode.as_str() - ); + eprintln!("[cortex] sqlite-vec routing force-off (configured mode={}, effective mode=baseline)", sqlite_vec_canary.route_mode.as_str()); } else { match sqlite_vec_canary.route_mode { SqliteVecRouteMode::Baseline => { @@ -209,20 +129,13 @@ fn initialize_with_conn( } SqliteVecRouteMode::Trial => { if sqlite_vec_canary.trial_percent > 0 { - eprintln!( - "[cortex] sqlite-vec routing mode=trial ({}% sampled)", - sqlite_vec_canary.trial_percent - ); + eprintln!("[cortex] sqlite-vec routing mode=trial ({}% sampled)", sqlite_vec_canary.trial_percent); } else { - eprintln!( - "[cortex] sqlite-vec routing mode=trial but trial percent is 0 (baseline-only)" - ); + eprintln!("[cortex] sqlite-vec routing mode=trial but trial percent is 0 (baseline-only)"); } } SqliteVecRouteMode::Primary => { - eprintln!( - "[cortex] sqlite-vec routing mode=primary (guarded vec0 routing enabled)" - ); + eprintln!("[cortex] sqlite-vec routing mode=primary (guarded vec0 routing enabled)"); } } } @@ -240,17 +153,13 @@ fn initialize_with_conn( Some(Arc::new(engine) as Arc) } None => { - eprintln!( - "[cortex] Reranker unavailable (mode={} requested, missing or invalid assets)", - rerank_config.mode.as_str() - ); + eprintln!("[cortex] Reranker unavailable (mode={} requested, missing or invalid assets)", rerank_config.mode.as_str()); None } } } else { None }; - let budget_config_status = crate::budgets::BudgetConfigStatus::load_from_home(&paths.home); if let Some(error) = budget_config_status.error.as_ref() { eprintln!( @@ -260,12 +169,8 @@ fn initialize_with_conn( error.message ); } else if budget_config_status.enabled() { - eprintln!( - "[cortex] Budget governance enabled from {}", - budget_config_status.source.display() - ); + eprintln!("[cortex] Budget governance enabled from {}", budget_config_status.source.display()); } - let state = RuntimeState { db: Arc::new(Mutex::new(conn)), db_read, @@ -274,8 +179,6 @@ fn initialize_with_conn( brain_firing: brain_firing_tx, mcp_calls: Arc::new(AtomicU64::new(0)), mcp_sessions: Arc::new(Mutex::new(HashMap::new())), - recall_history: Arc::new(Mutex::new(HashMap::new())), - pre_cache: Arc::new(Mutex::new(HashMap::new())), served_content: Arc::new(Mutex::new(HashMap::>::new())), shutdown_tx: Arc::new(Mutex::new(Some(shutdown_tx))), home, @@ -297,6 +200,5 @@ fn initialize_with_conn( rerank_config, reranker, }; - Ok((state, shutdown_rx)) } diff --git a/daemon-rs/src/state/mod.rs b/daemon-rs/src/state/mod.rs index 243ba842..415e73db 100644 --- a/daemon-rs/src/state/mod.rs +++ b/daemon-rs/src/state/mod.rs @@ -1,15 +1,9 @@ -// SPDX-License-Identifier: MIT -mod types; +mod init; mod read_pool; mod runtime; -mod init; - #[cfg(test)] -mod tests { - // Runtime tuning internals are not release-gated; see Info/testing-philosophy.md. -} - -pub use types::*; -pub use read_pool::{ReadConnLockFuture, ReadConnectionProvider}; -pub use runtime::RuntimeState; +mod tests; +mod types; pub use init::initialize; +pub use runtime::RuntimeState; +pub use types::*; diff --git a/daemon-rs/src/state/read_pool.rs b/daemon-rs/src/state/read_pool.rs index 6d5e6a4a..46430d8e 100644 --- a/daemon-rs/src/state/read_pool.rs +++ b/daemon-rs/src/state/read_pool.rs @@ -1,93 +1,57 @@ -// SPDX-License-Identifier: MIT +use rusqlite::Connection; use std::future::Future; use std::path::Path; use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; - -use rusqlite::Connection; use tokio::sync::Mutex; - const READ_POOL_SIZE_ENV: &str = "CORTEX_DB_READ_POOL_SIZE"; const READ_POOL_DEFAULT_MIN: usize = 4; const READ_POOL_DEFAULT_MAX: usize = 16; const READ_POOL_HARD_MAX: usize = 32; const READ_POOL_HARD_MIN: usize = 2; - -pub type ReadConnLockFuture<'a> = - Pin> + Send + 'a>>; - -/// Shared read handle abstraction so runtime can use a pooled implementation -/// while tests and fixtures can continue to inject a single Mutex connection. +pub type ReadConnLockFuture<'a> = Pin> + Send + 'a>>; pub trait ReadConnectionProvider: Send + Sync { fn lock<'a>(&'a self) -> ReadConnLockFuture<'a>; - fn pool_size(&self) -> usize { 1 } } - impl ReadConnectionProvider for Mutex { fn lock<'a>(&'a self) -> ReadConnLockFuture<'a> { Box::pin(async move { tokio::sync::Mutex::lock(self).await }) } } - pub(crate) struct ReadConnectionPool { connections: Vec>, next_index: AtomicUsize, } - impl ReadConnectionPool { pub(crate) fn new(connections: Vec) -> Self { - assert!( - !connections.is_empty(), - "read connection pool requires at least one connection" - ); - Self { - connections: connections.into_iter().map(Mutex::new).collect(), - next_index: AtomicUsize::new(0), - } + assert!(!connections.is_empty(), "read connection pool requires at least one connection"); + Self { connections: connections.into_iter().map(Mutex::new).collect(), next_index: AtomicUsize::new(0) } } } - impl ReadConnectionProvider for ReadConnectionPool { fn lock<'a>(&'a self) -> ReadConnLockFuture<'a> { let idx = self.next_index.fetch_add(1, Ordering::Relaxed) % self.connections.len(); Box::pin(async move { self.connections[idx].lock().await }) } - fn pool_size(&self) -> usize { self.connections.len() } } - pub(crate) fn derive_read_pool_size(configured: Option, cpu_hint: Option) -> usize { - let default = cpu_hint - .unwrap_or(READ_POOL_DEFAULT_MIN) - .clamp(READ_POOL_DEFAULT_MIN, READ_POOL_DEFAULT_MAX); - configured - .unwrap_or(default) - .clamp(READ_POOL_HARD_MIN, READ_POOL_HARD_MAX) + let default = cpu_hint.unwrap_or(READ_POOL_DEFAULT_MIN).clamp(READ_POOL_DEFAULT_MIN, READ_POOL_DEFAULT_MAX); + configured.unwrap_or(default).clamp(READ_POOL_HARD_MIN, READ_POOL_HARD_MAX) } - pub(crate) fn read_pool_size_from_env() -> usize { - let configured = std::env::var(READ_POOL_SIZE_ENV) - .ok() - .and_then(|raw| raw.trim().parse::().ok()); - let cpu_hint = std::thread::available_parallelism() - .ok() - .map(|cpus| cpus.get()); + let configured = std::env::var(READ_POOL_SIZE_ENV).ok().and_then(|raw| raw.trim().parse::().ok()); + let cpu_hint = std::thread::available_parallelism().ok().map(|cpus| cpus.get()); derive_read_pool_size(configured, cpu_hint) } - pub(crate) fn open_query_only_connection(db_path: &Path) -> Result { - let read_conn = - crate::db::open(db_path).map_err(|e| format!("Failed to open read connection: {e}"))?; - crate::db::configure(&read_conn) - .map_err(|e| format!("Failed to configure read connection: {e}"))?; - read_conn - .execute_batch("PRAGMA query_only = ON;") - .map_err(|e| e.to_string())?; + let read_conn = crate::db::open(db_path).map_err(|e| format!("Failed to open read connection: {e}"))?; + crate::db::configure(&read_conn).map_err(|e| format!("Failed to configure read connection: {e}"))?; + read_conn.execute_batch("PRAGMA query_only = ON;").map_err(|e| e.to_string())?; Ok(read_conn) } diff --git a/daemon-rs/src/state/runtime.rs b/daemon-rs/src/state/runtime.rs index 194411c0..03e7aeec 100644 --- a/daemon-rs/src/state/runtime.rs +++ b/daemon-rs/src/state/runtime.rs @@ -1,127 +1,60 @@ -// SPDX-License-Identifier: MIT -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::Arc; - +use super::read_pool::ReadConnectionProvider; +use super::types::{BrainFiringEvent, DaemonEvent, SqliteVecCanaryConfig}; use rusqlite::Connection; use serde_json::Value; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; use tokio::sync::{broadcast, oneshot, Mutex}; - -use super::read_pool::ReadConnectionProvider; -use super::types::{ - BrainFiringEvent, DaemonEvent, PreCacheEntry, RecallHistoryEntry, SqliteVecCanaryConfig, -}; - -/// Shared state threaded through every Axum handler via `axum::extract::State`. -/// -/// All fields are cheaply `Clone`able — most are wrapped in `Arc`. #[derive(Clone)] pub struct RuntimeState { - /// SQLite write connection -- used by store, forget, resolve, diary, indexer. pub db: Arc>, - /// SQLite read connection provider -- used by recall, peek, health, digest, boot. - /// Runtime uses a small pool of query-only connections so concurrent reads do - /// not serialize on one async mutex. pub db_read: Arc, - /// Auth token loaded from or written to the resolved runtime token path. pub token: Arc, - /// Broadcast channel for SSE events; clone the sender to fan-out. pub events: broadcast::Sender, - /// Broadcast channel for Brain-tab firing telemetry. Subscribed only by - /// `/brain/firing`; full payloads, owner-scoped at the handler. pub brain_firing: broadcast::Sender, - /// Monotonic counter for MCP call IDs. pub mcp_calls: Arc, - /// Active MCP sessions: session-id → last-heartbeat (Unix seconds). #[allow(dead_code)] pub mcp_sessions: Arc>>, - /// Per-agent recall history, capped at MAX_RECALL_HISTORY entries. - pub recall_history: Arc>>>, - /// Short-lived pre-warmed recall cache. - pub pre_cache: Arc>>, - /// Tracks which content hashes have been served to each agent recently. - /// Maps hash → Unix-ms timestamp. Entries older than SERVED_TTL_MS are - /// evicted, so content can be re-served after the cooldown. pub served_content: Arc>>>, - /// Sending half of the graceful-shutdown oneshot. The `/shutdown` endpoint - /// takes this and fires it; the Axum server listens on the receiving half. pub shutdown_tx: Arc>>>, - /// The user's home directory (used when constructing runtime paths). pub home: std::path::PathBuf, - /// Absolute path of the SQLite database file. #[allow(dead_code)] pub db_path: std::path::PathBuf, - /// Absolute path of the runtime auth token file. pub token_path: std::path::PathBuf, - /// Absolute path of the runtime PID file. pub pid_path: std::path::PathBuf, - /// Active HTTP port for this daemon instance. pub port: u16, - /// In-process ONNX embedding engine (None if model not downloaded yet). pub embedding_engine: Option>, - /// Per-IP sliding-window rate limiter. pub rate_limiter: crate::rate_limit::RateLimiter, - /// True when running with team-mode schema enabled. pub team_mode: bool, - /// Default owner used for owner-scoped conductor rows. pub default_owner_id: Option, - /// Team-mode API key hashes loaded from `users` for Argon2 verification. - /// Wrapped in RwLock so admin endpoints can add/remove keys at runtime. pub team_api_key_hashes: Arc>>, - /// Set to true when ONNX embedding fails at runtime (graceful degradation). pub degraded_mode: Arc, - /// Set to true when a runtime `quick_check` detects B-tree corruption. - /// Exposed on the `/health` endpoint as `db_corrupted`. pub db_corrupted: Arc, - /// Readiness gate for daemon startup sequencing. - /// `/readiness` reports this directly while `/health` remains diagnostic. pub readiness: Arc, - /// Last observed request activity timestamp (Unix seconds). pub last_activity_unix_secs: Arc, - /// Path for buffering writes when daemon is unreachable in proxy mode. - /// Used by mcp_proxy via cortex_dir() directly; kept here for discoverability. #[allow(dead_code)] pub write_buffer_path: std::path::PathBuf, - /// Guarded sqlite-vec semantic trial routing controls. pub sqlite_vec_canary: SqliteVecCanaryConfig, - /// Cross-encoder reranker config. Default is off; shadow/primary are opt-in. pub rerank_config: crate::rerank::RerankConfig, - /// Optional cross-encoder reranker loaded from local assets. pub reranker: Option>, } - impl RuntimeState { - /// Broadcast an event to all current SSE subscribers. Silently drops the - /// result — a send error just means there are no active subscribers. pub fn emit(&self, event_type: &str, data: Value) { - let _ = self.events.send(DaemonEvent { - event_type: event_type.to_string(), - data, - }); + let _ = self.events.send(DaemonEvent { event_type: event_type.to_string(), data }); } - - /// Increment the MCP call counter and return the new value. pub fn next_mcp_call(&self) -> u64 { use std::sync::atomic::Ordering; self.mcp_calls.fetch_add(1, Ordering::SeqCst) + 1 } - - /// Mark daemon activity to support idle-shutdown economics. pub fn mark_activity_now(&self) { - self.last_activity_unix_secs - .store(current_unix_secs(), Ordering::SeqCst); + self.last_activity_unix_secs.store(current_unix_secs(), Ordering::SeqCst); } - - /// Seconds since the last observed request activity. pub fn idle_for_secs(&self) -> u64 { let last = self.last_activity_unix_secs.load(Ordering::SeqCst); current_unix_secs().saturating_sub(last) } } - pub(crate) fn current_unix_secs() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs() } diff --git a/daemon-rs/src/state/tests/mod.rs b/daemon-rs/src/state/tests/mod.rs new file mode 100644 index 00000000..ed6f38aa --- /dev/null +++ b/daemon-rs/src/state/tests/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: MIT +use super::*; diff --git a/daemon-rs/src/state/types.rs b/daemon-rs/src/state/types.rs index b5422ff4..e8c21fd1 100644 --- a/daemon-rs/src/state/types.rs +++ b/daemon-rs/src/state/types.rs @@ -1,38 +1,10 @@ -// SPDX-License-Identifier: MIT use serde_json::Value; - -/// A single entry in the per-session recall history, recording what was queried -/// and when (Unix milliseconds). -#[derive(Clone, Debug)] -pub struct RecallHistoryEntry { - pub query: String, - pub timestamp: i64, -} - -/// A cached recall result set. `expires_at` is a Unix-millisecond deadline -/// after which the entry should be discarded. -#[derive(Clone, Debug)] -pub struct PreCacheEntry { - pub query: String, - /// Serialised recall results — stored as `Value` so this module does not - /// need to know about the full recall pipeline types. - pub results: Value, - pub expires_at: i64, -} - -/// A typed event broadcast to all SSE subscribers. #[derive(Clone, Debug)] pub struct DaemonEvent { pub event_type: String, - // Event payloads are retained on the bus for future internal consumers, - // but the public SSE stream currently redacts them before emission. #[allow(dead_code)] pub data: Value, } - -/// Brain-tab firing telemetry. Carries full payloads (unscrubbed), but the -/// `/brain/firing` SSE handler filters per-subscriber by `owner_id` before -/// forwarding. Public `/events/stream` is unaffected. #[derive(Clone, Debug)] pub enum BrainKind { ConsolidationStarted, @@ -40,7 +12,6 @@ pub enum BrainKind { ClusterFinalized, Recall, } - impl BrainKind { pub fn as_str(&self) -> &'static str { match self { @@ -51,21 +22,18 @@ impl BrainKind { } } } - #[derive(Clone, Debug)] pub struct BrainFiringEvent { pub kind: BrainKind, pub payload: Value, pub owner_id: Option, } - #[derive(Clone, Debug)] pub enum SqliteVecRouteMode { Baseline, Trial, Primary, } - impl SqliteVecRouteMode { pub(crate) fn from_env() -> Self { match std::env::var("CORTEX_SQLITE_VEC_ROUTE") { @@ -74,16 +42,13 @@ impl SqliteVecRouteMode { "trial" | "canary" | "sampled" => Self::Trial, "primary" | "vec0" | "production" | "on" => Self::Primary, unknown => { - eprintln!( - "[cortex] WARNING: invalid CORTEX_SQLITE_VEC_ROUTE={unknown:?}; using primary" - ); + eprintln!("[cortex] WARNING: invalid CORTEX_SQLITE_VEC_ROUTE={unknown:?}; using primary"); Self::Primary } }, Err(_) => Self::Primary, } } - pub fn as_str(&self) -> &'static str { match self { Self::Baseline => "baseline", @@ -92,14 +57,12 @@ impl SqliteVecRouteMode { } } } - #[derive(Clone, Debug)] pub struct SqliteVecCanaryConfig { pub trial_percent: u8, pub force_off: bool, pub route_mode: SqliteVecRouteMode, } - impl SqliteVecCanaryConfig { pub(crate) fn from_env() -> Self { let route_mode = SqliteVecRouteMode::from_env(); @@ -113,9 +76,7 @@ impl SqliteVecCanaryConfig { match trimmed.parse::() { Ok(percent) => Some(percent.min(100)), Err(_) => { - eprintln!( - "[cortex] WARNING: invalid CORTEX_SQLITE_VEC_TRIAL_PERCENT={trimmed:?}; using 0" - ); + eprintln!("[cortex] WARNING: invalid CORTEX_SQLITE_VEC_TRIAL_PERCENT={trimmed:?}; using 0"); Some(0) } } @@ -123,19 +84,9 @@ impl SqliteVecCanaryConfig { .unwrap_or(0); let force_off = std::env::var("CORTEX_SQLITE_VEC_TRIAL_FORCE_OFF") .ok() - .is_some_and(|value| { - matches!( - value.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ) - }); - Self { - trial_percent, - force_off, - route_mode, - } + .is_some_and(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")); + Self { trial_percent, force_off, route_mode } } - pub fn effective_route_mode(&self) -> SqliteVecRouteMode { if self.force_off { SqliteVecRouteMode::Baseline diff --git a/daemon-rs/src/test_env.rs b/daemon-rs/src/test_env.rs index b145cada..9d5e6ed9 100644 --- a/daemon-rs/src/test_env.rs +++ b/daemon-rs/src/test_env.rs @@ -1,38 +1,29 @@ -// SPDX-License-Identifier: MIT - use std::ffi::{OsStr, OsString}; use std::sync::OnceLock; use tokio::sync::{Mutex, MutexGuard}; - static ENV_LOCK: OnceLock> = OnceLock::new(); - pub fn lock() -> MutexGuard<'static, ()> { ENV_LOCK.get_or_init(|| Mutex::new(())).blocking_lock() } - pub async fn lock_async() -> MutexGuard<'static, ()> { ENV_LOCK.get_or_init(|| Mutex::new(())).lock().await } - pub struct ScopedEnvVar { key: &'static str, previous: Option, } - impl ScopedEnvVar { pub fn set(key: &'static str, value: impl AsRef) -> Self { let previous = std::env::var_os(key); std::env::set_var(key, value); Self { key, previous } } - pub fn remove(key: &'static str) -> Self { let previous = std::env::var_os(key); std::env::remove_var(key); Self { key, previous } } } - impl Drop for ScopedEnvVar { fn drop(&mut self) { if let Some(previous) = self.previous.as_ref() { diff --git a/daemon-rs/src/test_support.rs b/daemon-rs/src/test_support.rs index 416bc1fb..6a9dd112 100644 --- a/daemon-rs/src/test_support.rs +++ b/daemon-rs/src/test_support.rs @@ -1,17 +1,11 @@ -// SPDX-License-Identifier: MIT -//! Shared in-process test fixtures for handler/unit tests. - +use crate::db; +use crate::rerank::{RerankConfig, Reranker}; +use crate::state::RuntimeState; use std::collections::HashMap; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::Arc; - use tokio::sync::{broadcast, Mutex}; - -use crate::db; -use crate::rerank::{RerankConfig, Reranker}; -use crate::state::RuntimeState; - pub fn test_conn() -> rusqlite::Connection { let conn = rusqlite::Connection::open_in_memory().expect("open in-memory db"); db::configure(&conn).expect("configure db"); @@ -19,28 +13,14 @@ pub fn test_conn() -> rusqlite::Connection { db::run_pending_migrations(&conn); conn } - pub fn solo_state() -> RuntimeState { runtime_state(test_conn(), test_conn(), false, None, RerankConfig::off(), None) } - pub fn team_state(default_owner_id: i64) -> RuntimeState { - runtime_state( - test_conn(), - test_conn(), - true, - Some(default_owner_id), - RerankConfig::off(), - None, - ) + runtime_state(test_conn(), test_conn(), true, Some(default_owner_id), RerankConfig::off(), None) } - pub fn runtime_state( - write_conn: rusqlite::Connection, - read_conn: rusqlite::Connection, - team_mode: bool, - default_owner_id: Option, - rerank_config: RerankConfig, + write_conn: rusqlite::Connection, read_conn: rusqlite::Connection, team_mode: bool, default_owner_id: Option, rerank_config: RerankConfig, reranker: Option>, ) -> RuntimeState { let (events, _) = broadcast::channel(8); @@ -53,8 +33,6 @@ pub fn runtime_state( brain_firing, mcp_calls: Arc::new(AtomicU64::new(0)), mcp_sessions: Arc::new(Mutex::new(HashMap::new())), - recall_history: Arc::new(Mutex::new(HashMap::new())), - pre_cache: Arc::new(Mutex::new(HashMap::new())), served_content: Arc::new(Mutex::new(HashMap::new())), shutdown_tx: Arc::new(Mutex::new(None)), home: PathBuf::from("."), @@ -72,11 +50,7 @@ pub fn runtime_state( readiness: Arc::new(AtomicBool::new(true)), last_activity_unix_secs: Arc::new(AtomicU64::new(0)), write_buffer_path: PathBuf::from("write_buffer.jsonl"), - sqlite_vec_canary: crate::state::SqliteVecCanaryConfig { - trial_percent: 0, - force_off: false, - route_mode: crate::state::SqliteVecRouteMode::Trial, - }, + sqlite_vec_canary: crate::state::SqliteVecCanaryConfig { trial_percent: 0, force_off: false, route_mode: crate::state::SqliteVecRouteMode::Trial }, rerank_config, reranker, } diff --git a/daemon-rs/src/tls.rs b/daemon-rs/src/tls.rs index 7e3153d0..d40997a0 100644 --- a/daemon-rs/src/tls.rs +++ b/daemon-rs/src/tls.rs @@ -1,83 +1,42 @@ -// SPDX-License-Identifier: MIT -//! Optional TLS via rustls. -//! -//! Solo mode (default): no TLS, plain HTTP on 127.0.0.1. -//! Team mode: TLS enabled when cert + key found at `~/.cortex/tls/`. -//! -//! Configurable modes: -//! - No TLS files → plain HTTP (solo default, satisfies localhost-only constraint) -//! - User-provided cert/key → TLS with those certs -//! - CORTEX_TLS_CERT / CORTEX_TLS_KEY env vars → override paths - use rustls::ServerConfig; use rustls_pki_types::{pem::PemObject, CertificateDer, PrivateKeyDer}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio_rustls::TlsAcceptor; - fn default_tls_dir() -> PathBuf { crate::auth::cortex_dir().join("tls") } - fn cert_path() -> PathBuf { - std::env::var("CORTEX_TLS_CERT") - .map(PathBuf::from) - .unwrap_or_else(|_| default_tls_dir().join("cert.pem")) + std::env::var("CORTEX_TLS_CERT").map(PathBuf::from).unwrap_or_else(|_| default_tls_dir().join("cert.pem")) } - fn key_path() -> PathBuf { - std::env::var("CORTEX_TLS_KEY") - .map(PathBuf::from) - .unwrap_or_else(|_| default_tls_dir().join("key.pem")) + std::env::var("CORTEX_TLS_KEY").map(PathBuf::from).unwrap_or_else(|_| default_tls_dir().join("key.pem")) } - -/// Try to build a TLS acceptor from cert/key files. -/// Returns `Ok(None)` if no TLS files are found (solo mode -- plain HTTP). -/// Returns `Err` if files exist but are invalid. -/// Caller decides whether to refuse startup (team) or fall back (solo). pub fn try_load_tls() -> Result, String> { let cert = cert_path(); let key = key_path(); - if !cert.exists() && !key.exists() { return Ok(None); } - if !cert.exists() { - return Err(format!( - "TLS key found but cert missing at {}", - cert.display() - )); + return Err(format!("TLS key found but cert missing at {}", cert.display())); } if !key.exists() { - return Err(format!( - "TLS cert found but key missing at {}", - key.display() - )); + return Err(format!("TLS cert found but key missing at {}", key.display())); } - let config = load_rustls_config(&cert, &key)?; Ok(Some(TlsAcceptor::from(Arc::new(config)))) } - fn load_rustls_config(cert_path: &Path, key_path: &Path) -> Result { - let cert_file = std::fs::File::open(cert_path) - .map_err(|e| format!("Failed to open cert {}: {e}", cert_path.display()))?; - let key_file = std::fs::File::open(key_path) - .map_err(|e| format!("Failed to open key {}: {e}", key_path.display()))?; - - let certs: Vec> = - CertificateDer::pem_reader_iter(std::io::BufReader::new(cert_file)) - .collect::, _>>() - .map_err(|e| format!("Failed to parse certs: {e}"))?; - + let cert_file = std::fs::File::open(cert_path).map_err(|e| format!("Failed to open cert {}: {e}", cert_path.display()))?; + let key_file = std::fs::File::open(key_path).map_err(|e| format!("Failed to open key {}: {e}", key_path.display()))?; + let certs: Vec> = CertificateDer::pem_reader_iter(std::io::BufReader::new(cert_file)) + .collect::, _>>() + .map_err(|e| format!("Failed to parse certs: {e}"))?; if certs.is_empty() { return Err("No certificates found in cert file".to_string()); } - - let key = PrivateKeyDer::from_pem_reader(std::io::BufReader::new(key_file)) - .map_err(|e| format!("Failed to parse key: {e}"))?; - + let key = PrivateKeyDer::from_pem_reader(std::io::BufReader::new(key_file)).map_err(|e| format!("Failed to parse key: {e}"))?; ServerConfig::builder() .with_no_client_auth() .with_single_cert(certs, key) diff --git a/daemon-rs/src/transport.rs b/daemon-rs/src/transport.rs deleted file mode 100644 index b10f3349..00000000 --- a/daemon-rs/src/transport.rs +++ /dev/null @@ -1,359 +0,0 @@ -// SPDX-License-Identifier: MIT -use crate::auth::CortexPaths; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; - -fn normalized_host(value: &str) -> String { - value - .trim() - .trim_start_matches('[') - .trim_end_matches(']') - .to_ascii_lowercase() -} - -pub(crate) fn http_host_for_bind(bind: &str) -> String { - let bind = bind.trim(); - if bind.is_empty() || matches!(bind, "0.0.0.0" | "::" | "[::]") { - "127.0.0.1".to_string() - } else if bind.starts_with('[') && bind.ends_with(']') { - bind.to_string() - } else if bind.contains(':') { - format!("[{bind}]") - } else { - bind.to_string() - } -} - -pub fn local_http_base_url(paths: &CortexPaths) -> String { - let host = http_host_for_bind(&paths.bind); - format!("http://{host}:{}", paths.port) -} - -pub fn is_local_http_base_url(base_url: &str, paths: &CortexPaths) -> bool { - let Ok(url) = reqwest::Url::parse(base_url) else { - return false; - }; - let Some(host) = url.host_str() else { - return false; - }; - if url.port_or_known_default() != Some(paths.port) { - return false; - } - let host_norm = normalized_host(host); - let bind_norm = normalized_host(&paths.bind); - matches!(host_norm.as_str(), "127.0.0.1" | "localhost" | "::1") - || (!bind_norm.is_empty() - && !matches!(bind_norm.as_str(), "0.0.0.0" | "::") - && host_norm == bind_norm) -} - -pub fn local_ipc_endpoint_for_base_url(base_url: &str, paths: &CortexPaths) -> Option { - if !is_local_http_base_url(base_url, paths) { - return None; - } - paths.ipc_endpoint.clone() -} - -fn split_base_and_path(url: &str) -> Option<(String, String)> { - let parsed = reqwest::Url::parse(url).ok()?; - let mut base = parsed.clone(); - base.set_path(""); - base.set_query(None); - base.set_fragment(None); - let mut path = parsed.path().to_string(); - if path.is_empty() { - path.push('/'); - } - if let Some(query) = parsed.query() { - path.push('?'); - path.push_str(query); - } - Some((base.to_string().trim_end_matches('/').to_string(), path)) -} - -pub(crate) fn parse_http_response_bytes( - raw: &[u8], - source_label: &str, -) -> Result<(reqwest::StatusCode, String), String> { - let Some(header_end) = raw.windows(4).position(|window| window == b"\r\n\r\n") else { - return Err(format!("invalid HTTP response from {source_label}")); - }; - - let header = std::str::from_utf8(&raw[..header_end]) - .map_err(|_| format!("{source_label} response headers are not valid UTF-8"))?; - let status_line = header - .lines() - .next() - .ok_or_else(|| format!("{source_label} response missing valid HTTP status line"))?; - let status = parse_http_status_line(status_line, source_label)?; - let body = String::from_utf8_lossy(&raw[header_end + 4..]).to_string(); - Ok((status, body)) -} - -fn parse_http_status_line( - status_line: &str, - source_label: &str, -) -> Result { - let mut fields = status_line.split_whitespace(); - let version = fields - .next() - .ok_or_else(|| format!("{source_label} response missing HTTP version"))?; - if !matches!(version, "HTTP/1.0" | "HTTP/1.1") { - return Err(format!( - "{source_label} response has unsupported HTTP version '{version}'" - )); - } - - let status_code_raw = fields - .next() - .ok_or_else(|| format!("{source_label} response missing status code"))?; - if status_code_raw.len() != 3 || !status_code_raw.bytes().all(|byte| byte.is_ascii_digit()) { - return Err(format!( - "{source_label} response has malformed status code '{status_code_raw}'" - )); - } - - let status_code = status_code_raw - .parse::() - .map_err(|_| format!("{source_label} response has malformed status code"))?; - reqwest::StatusCode::from_u16(status_code) - .map_err(|_| format!("{source_label} response returned invalid status code {status_code}")) -} - -fn parse_http_response(raw: &[u8]) -> Result<(reqwest::StatusCode, String), String> { - parse_http_response_bytes(raw, "IPC endpoint") -} - -async fn send_http_over_stream( - stream: &mut S, - method: &str, - path: &str, - headers: &[(String, String)], - body: Option<&str>, -) -> Result<(reqwest::StatusCode, String), String> -where - S: AsyncRead + AsyncWrite + Unpin, -{ - let body = body.unwrap_or(""); - let mut request = String::new(); - request.push_str(method); - request.push(' '); - request.push_str(path); - request.push_str(" HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n"); - for (name, value) in headers { - request.push_str(name); - request.push_str(": "); - request.push_str(value); - request.push_str("\r\n"); - } - request.push_str("Content-Length: "); - request.push_str(&body.len().to_string()); - request.push_str("\r\n\r\n"); - request.push_str(body); - - stream - .write_all(request.as_bytes()) - .await - .map_err(|e| format!("IPC write failed: {e}"))?; - stream - .flush() - .await - .map_err(|e| format!("IPC flush failed: {e}"))?; - - let mut response = Vec::new(); - stream - .read_to_end(&mut response) - .await - .map_err(|e| format!("IPC read failed: {e}"))?; - parse_http_response(&response) -} - -async fn ipc_http_request( - endpoint: &str, - method: &str, - path: &str, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, -) -> Result<(reqwest::StatusCode, String), String> { - let fut = async { - #[cfg(unix)] - { - let mut stream = tokio::net::UnixStream::connect(endpoint) - .await - .map_err(|e| format!("IPC connect failed: {e}"))?; - return send_http_over_stream(&mut stream, method, path, headers, body).await; - } - #[cfg(windows)] - { - let mut stream = tokio::net::windows::named_pipe::ClientOptions::new() - .open(endpoint) - .map_err(|e| format!("IPC connect failed: {e}"))?; - return send_http_over_stream(&mut stream, method, path, headers, body).await; - } - #[allow(unreachable_code)] - Err("IPC transport is unsupported on this platform".to_string()) - }; - tokio::time::timeout(timeout, fut) - .await - .map_err(|_| "IPC request timed out".to_string())? -} - -async fn send_http_request( - client: &reqwest::Client, - method: &str, - url: &str, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, -) -> Result<(reqwest::StatusCode, String), String> { - let mut req = match method { - "GET" => client.get(url), - "POST" => client.post(url), - other => return Err(format!("Unsupported request method '{other}'")), - }; - req = req.timeout(timeout); - for (name, value) in headers { - req = req.header(name, value); - } - if let Some(payload) = body { - req = req.body(payload.to_string()); - } - - let response = req.send().await.map_err(|e| e.to_string())?; - let status = response.status(); - let body = response.text().await.map_err(|e| e.to_string())?; - Ok((status, body)) -} - -#[allow(clippy::too_many_arguments)] -pub async fn request_with_local_ipc_fallback( - client: &reqwest::Client, - method: &str, - base_url: &str, - path: &str, - paths: &CortexPaths, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, -) -> Result<(reqwest::StatusCode, String), String> { - if let Some(endpoint) = local_ipc_endpoint_for_base_url(base_url, paths) { - match ipc_http_request(&endpoint, method, path, headers, body, timeout).await { - Ok(response) => return Ok(response), - Err(err) => { - eprintln!( - "[cortex-transport] IPC request failed for {method} {path} ({endpoint}): {err}; falling back to HTTP" - ); - } - } - } - - let normalized_base = base_url.trim_end_matches('/'); - let normalized_path = if path.starts_with('/') { - path.to_string() - } else { - format!("/{path}") - }; - let url = format!("{normalized_base}{normalized_path}"); - send_http_request(client, method, &url, headers, body, timeout).await -} - -pub async fn request_url_with_local_ipc_fallback( - client: &reqwest::Client, - method: &str, - url: &str, - paths: &CortexPaths, - headers: &[(String, String)], - body: Option<&str>, - timeout: std::time::Duration, -) -> Result<(reqwest::StatusCode, String), String> { - if let Some((base_url, path)) = split_base_and_path(url) { - return request_with_local_ipc_fallback( - client, method, &base_url, &path, paths, headers, body, timeout, - ) - .await; - } - send_http_request(client, method, url, headers, body, timeout).await -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_paths(bind: &str, port: u16, ipc_endpoint: Option<&str>) -> CortexPaths { - let temp = std::env::temp_dir().join("cortex_transport_tests"); - CortexPaths { - home: temp.clone(), - db: temp.join("cortex.db"), - token: temp.join("cortex.token"), - pid: temp.join("cortex.pid"), - lock: temp.join("cortex.lock"), - port, - bind: bind.to_string(), - ipc_endpoint: ipc_endpoint.map(|value| value.to_string()), - models: temp.join("models"), - write_buffer: temp.join("write_buffer.jsonl"), - } - } - - #[test] - fn local_ipc_endpoint_only_resolves_for_local_targets() { - let paths = test_paths("127.0.0.1", 7437, Some(r"\\.\pipe\cortex-daemon-7437")); - assert_eq!( - local_ipc_endpoint_for_base_url("http://127.0.0.1:7437", &paths), - Some(r"\\.\pipe\cortex-daemon-7437".to_string()) - ); - assert_eq!( - local_ipc_endpoint_for_base_url("https://api.example.com:443", &paths), - None - ); - } - - #[test] - fn local_http_base_url_uses_loopback_for_wildcard_bind() { - let paths = test_paths("0.0.0.0", 7437, None); - assert_eq!(local_http_base_url(&paths), "http://127.0.0.1:7437"); - } - - #[test] - fn local_http_base_url_formats_wildcard_and_ipv6_hosts() { - assert_eq!( - local_http_base_url(&test_paths("", 7437, None)), - "http://127.0.0.1:7437" - ); - assert_eq!( - local_http_base_url(&test_paths("::", 7437, None)), - "http://127.0.0.1:7437" - ); - assert_eq!( - local_http_base_url(&test_paths("[::]", 7437, None)), - "http://127.0.0.1:7437" - ); - assert_eq!( - local_http_base_url(&test_paths("::1", 7437, None)), - "http://[::1]:7437" - ); - assert_eq!( - local_http_base_url(&test_paths("[::1]", 7437, None)), - "http://[::1]:7437" - ); - } - - #[test] - fn parse_http_response_rejects_malformed_status_lines() { - let malformed = [ - b"garbage 200 OK\r\n\r\n{}" as &[u8], - b"HTTP/9.9 200 OK\r\n\r\n{}", - b"HTTP/1.1 99 TooLow\r\n\r\n{}", - b"HTTP/1.1 200OK\r\n\r\n{}", - b"HTTP/1.1 abc Nope\r\n\r\n{}", - ]; - - for raw in malformed { - assert!( - parse_http_response(raw).is_err(), - "malformed response should be rejected: {:?}", - String::from_utf8_lossy(raw) - ); - } - } -} diff --git a/daemon-rs/src/transport/mod.rs b/daemon-rs/src/transport/mod.rs new file mode 100644 index 00000000..d98f8453 --- /dev/null +++ b/daemon-rs/src/transport/mod.rs @@ -0,0 +1,179 @@ +use crate::auth::CortexPaths; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +fn normalized_host(value: &str) -> String { + value.trim().trim_start_matches('[').trim_end_matches(']').to_ascii_lowercase() +} +pub(crate) fn http_host_for_bind(bind: &str) -> String { + let bind = bind.trim(); + if bind.is_empty() || matches!(bind, "0.0.0.0" | "::" | "[::]") { + "127.0.0.1".to_string() + } else if bind.starts_with('[') && bind.ends_with(']') { + bind.to_string() + } else if bind.contains(':') { + format!("[{bind}]") + } else { + bind.to_string() + } +} +pub fn local_http_base_url(paths: &CortexPaths) -> String { + let host = http_host_for_bind(&paths.bind); + format!("http://{host}:{}", paths.port) +} +pub fn is_local_http_base_url(base_url: &str, paths: &CortexPaths) -> bool { + let Ok(url) = reqwest::Url::parse(base_url) else { + return false; + }; + let Some(host) = url.host_str() else { + return false; + }; + if url.port_or_known_default() != Some(paths.port) { + return false; + } + let host_norm = normalized_host(host); + let bind_norm = normalized_host(&paths.bind); + matches!(host_norm.as_str(), "127.0.0.1" | "localhost" | "::1") + || (!bind_norm.is_empty() && !matches!(bind_norm.as_str(), "0.0.0.0" | "::") && host_norm == bind_norm) +} +pub fn local_ipc_endpoint_for_base_url(base_url: &str, paths: &CortexPaths) -> Option { + if !is_local_http_base_url(base_url, paths) { + return None; + } + paths.ipc_endpoint.clone() +} +pub(crate) fn split_base_and_path(url: &str) -> Option<(String, String)> { + let parsed = reqwest::Url::parse(url).ok()?; + let mut base = parsed.clone(); + base.set_path(""); + base.set_query(None); + base.set_fragment(None); + let mut path = parsed.path().to_string(); + if path.is_empty() { + path.push('/'); + } + if let Some(query) = parsed.query() { + path.push('?'); + path.push_str(query); + } + Some((base.to_string().trim_end_matches('/').to_string(), path)) +} +pub(crate) fn parse_http_response_bytes(raw: &[u8], source_label: &str) -> Result<(reqwest::StatusCode, String), String> { + let Some(header_end) = raw.windows(4).position(|window| window == b"\r\n\r\n") else { + return Err(format!("invalid HTTP response from {source_label}")); + }; + let header = std::str::from_utf8(&raw[..header_end]).map_err(|_| format!("{source_label} response headers are not valid UTF-8"))?; + let status_line = header.lines().next().ok_or_else(|| format!("{source_label} response missing valid HTTP status line"))?; + let status = parse_http_status_line(status_line, source_label)?; + let body = String::from_utf8_lossy(&raw[header_end + 4..]).to_string(); + Ok((status, body)) +} +fn parse_http_status_line(status_line: &str, source_label: &str) -> Result { + let mut fields = status_line.split_whitespace(); + let version = fields.next().ok_or_else(|| format!("{source_label} response missing HTTP version"))?; + if !matches!(version, "HTTP/1.0" | "HTTP/1.1") { + return Err(format!("{source_label} response has unsupported HTTP version '{version}'")); + } + let status_code_raw = fields.next().ok_or_else(|| format!("{source_label} response missing status code"))?; + if status_code_raw.len() != 3 || !status_code_raw.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(format!("{source_label} response has malformed status code '{status_code_raw}'")); + } + let status_code = status_code_raw.parse::().map_err(|_| format!("{source_label} response has malformed status code"))?; + reqwest::StatusCode::from_u16(status_code).map_err(|_| format!("{source_label} response returned invalid status code {status_code}")) +} +fn parse_http_response(raw: &[u8]) -> Result<(reqwest::StatusCode, String), String> { + parse_http_response_bytes(raw, "IPC endpoint") +} +async fn send_http_over_stream( + stream: &mut S, method: &str, path: &str, headers: &[(String, String)], body: Option<&str>, +) -> Result<(reqwest::StatusCode, String), String> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let body = body.unwrap_or(""); + let mut request = String::new(); + request.push_str(method); + request.push(' '); + request.push_str(path); + request.push_str(" HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n"); + for (name, value) in headers { + request.push_str(name); + request.push_str(": "); + request.push_str(value); + request.push_str("\r\n"); + } + request.push_str("Content-Length: "); + request.push_str(&body.len().to_string()); + request.push_str("\r\n\r\n"); + request.push_str(body); + stream.write_all(request.as_bytes()).await.map_err(|e| format!("IPC write failed: {e}"))?; + stream.flush().await.map_err(|e| format!("IPC flush failed: {e}"))?; + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.map_err(|e| format!("IPC read failed: {e}"))?; + parse_http_response(&response) +} +async fn ipc_http_request( + endpoint: &str, method: &str, path: &str, headers: &[(String, String)], body: Option<&str>, timeout: std::time::Duration, +) -> Result<(reqwest::StatusCode, String), String> { + let fut = async { + #[cfg(unix)] + { + let mut stream = tokio::net::UnixStream::connect(endpoint).await.map_err(|e| format!("IPC connect failed: {e}"))?; + return send_http_over_stream(&mut stream, method, path, headers, body).await; + } + #[cfg(windows)] + { + let mut stream = tokio::net::windows::named_pipe::ClientOptions::new().open(endpoint).map_err(|e| format!("IPC connect failed: {e}"))?; + return send_http_over_stream(&mut stream, method, path, headers, body).await; + } + #[allow(unreachable_code)] + Err("IPC transport is unsupported on this platform".to_string()) + }; + tokio::time::timeout(timeout, fut).await.map_err(|_| "IPC request timed out".to_string())? +} +pub(crate) async fn send_http_request( + client: &reqwest::Client, method: &str, url: &str, headers: &[(String, String)], body: Option<&str>, timeout: std::time::Duration, +) -> Result<(reqwest::StatusCode, String), String> { + let mut req = match method { + "GET" => client.get(url), + "POST" => client.post(url), + other => return Err(format!("Unsupported request method '{other}'")), + }; + req = req.timeout(timeout); + for (name, value) in headers { + req = req.header(name, value); + } + if let Some(payload) = body { + req = req.body(payload.to_string()); + } + let response = req.send().await.map_err(|e| e.to_string())?; + let status = response.status(); + let body = response.text().await.map_err(|e| e.to_string())?; + Ok((status, body)) +} +#[allow(clippy::too_many_arguments)] +pub async fn request_with_local_ipc_fallback( + client: &reqwest::Client, method: &str, base_url: &str, path: &str, paths: &CortexPaths, headers: &[(String, String)], body: Option<&str>, + timeout: std::time::Duration, +) -> Result<(reqwest::StatusCode, String), String> { + if let Some(endpoint) = local_ipc_endpoint_for_base_url(base_url, paths) { + match ipc_http_request(&endpoint, method, path, headers, body, timeout).await { + Ok(response) => return Ok(response), + Err(err) => { + eprintln!("[cortex-transport] IPC request failed for {method} {path} ({endpoint}): {err}; falling back to HTTP"); + } + } + } + let normalized_base = base_url.trim_end_matches('/'); + let normalized_path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") }; + let url = format!("{normalized_base}{normalized_path}"); + send_http_request(client, method, &url, headers, body, timeout).await +} +pub async fn request_url_with_local_ipc_fallback( + client: &reqwest::Client, method: &str, url: &str, paths: &CortexPaths, headers: &[(String, String)], body: Option<&str>, timeout: std::time::Duration, +) -> Result<(reqwest::StatusCode, String), String> { + if let Some((base_url, path)) = split_base_and_path(url) { + return request_with_local_ipc_fallback(client, method, &base_url, &path, paths, headers, body, timeout).await; + } + send_http_request(client, method, url, headers, body, timeout).await +} +#[cfg(test)] +mod tests; diff --git a/daemon-rs/src/transport/tests/mod.rs b/daemon-rs/src/transport/tests/mod.rs new file mode 100644 index 00000000..5c2d4ae1 --- /dev/null +++ b/daemon-rs/src/transport/tests/mod.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +use super::*; + +use super::*; +fn test_paths(bind: &str, port: u16, ipc_endpoint: Option<&str>) -> CortexPaths { + let temp = std::env::temp_dir().join("cortex_transport_tests"); + CortexPaths { + home: temp.clone(), + db: temp.join("cortex.db"), + token: temp.join("cortex.token"), + pid: temp.join("cortex.pid"), + lock: temp.join("cortex.lock"), + port, + bind: bind.to_string(), + ipc_endpoint: ipc_endpoint.map(|value| value.to_string()), + models: temp.join("models"), + write_buffer: temp.join("write_buffer.jsonl"), + } +} +#[test] +fn local_ipc_endpoint_only_resolves_for_local_targets() { + let paths = test_paths("127.0.0.1", 7437, Some(r"\\.\pipe\cortex-daemon-7437")); + assert_eq!(local_ipc_endpoint_for_base_url("http://127.0.0.1:7437", &paths), Some(r"\\.\pipe\cortex-daemon-7437".to_string())); + assert_eq!(local_ipc_endpoint_for_base_url("https://api.example.com:443", &paths), None); +} +#[test] +fn local_http_base_url_uses_loopback_for_wildcard_bind() { + let paths = test_paths("0.0.0.0", 7437, None); + assert_eq!(local_http_base_url(&paths), "http://127.0.0.1:7437"); +} +#[test] +fn local_http_base_url_formats_wildcard_and_ipv6_hosts() { + assert_eq!(local_http_base_url(&test_paths("", 7437, None)), "http://127.0.0.1:7437"); + assert_eq!(local_http_base_url(&test_paths("::", 7437, None)), "http://127.0.0.1:7437"); + assert_eq!(local_http_base_url(&test_paths("[::]", 7437, None)), "http://127.0.0.1:7437"); + assert_eq!(local_http_base_url(&test_paths("::1", 7437, None)), "http://[::1]:7437"); + assert_eq!(local_http_base_url(&test_paths("[::1]", 7437, None)), "http://[::1]:7437"); +} +#[test] +fn parse_http_response_rejects_malformed_status_lines() { + let malformed = [ + b"garbage 200 OK\r\n\r\n{}" as &[u8], + b"HTTP/9.9 200 OK\r\n\r\n{}", + b"HTTP/1.1 99 TooLow\r\n\r\n{}", + b"HTTP/1.1 200OK\r\n\r\n{}", + b"HTTP/1.1 abc Nope\r\n\r\n{}", + ]; + for raw in malformed { + assert!(parse_http_response(raw).is_err(), "malformed response should be rejected: {:?}", String::from_utf8_lossy(raw)); + } +} diff --git a/daemon-rs/src/workspace.rs b/daemon-rs/src/workspace.rs index 12ecd149..51c99b1b 100644 --- a/daemon-rs/src/workspace.rs +++ b/daemon-rs/src/workspace.rs @@ -1,10 +1,4 @@ -// SPDX-License-Identifier: MIT -//! Workspace-level helpers shared by boot compilation and indexing. - use std::env; - -/// Derive the Claude Code project folder slug from the current working directory. -/// Claude encodes paths with separators flattened, e.g. `C--Users-project-cortex`. pub(crate) fn claude_project_slug() -> Option { let cwd = env::current_dir().ok()?; let canonical = cwd.to_string_lossy().to_string(); diff --git a/desktop/cortex-control-center/EXPECT_SMOKE.md b/desktop/cortex-control-center/EXPECT_SMOKE.md index 3e77141f..b855efc8 100644 --- a/desktop/cortex-control-center/EXPECT_SMOKE.md +++ b/desktop/cortex-control-center/EXPECT_SMOKE.md @@ -84,7 +84,6 @@ The smoke step is intentionally opt-in because hosted runners do not come with a ## Files -- `scripts/mock-cortex-server.mjs` - `scripts/run-expect-smoke.mjs` - `package.json` - `.github/workflows/ci.yml` diff --git a/desktop/cortex-control-center/generate-icon.py b/desktop/cortex-control-center/generate-icon.py deleted file mode 100644 index f5ee9d69..00000000 --- a/desktop/cortex-control-center/generate-icon.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Generate Cortex icon v4 — Big bold C, white nodes branching right.""" -from PIL import Image, ImageDraw, ImageFont -import math -import os - - -def draw_cortex_icon(size: int) -> Image.Image: - img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) - draw = ImageDraw.Draw(img) - cx, cy = size / 2, size / 2 - r = size * 0.46 - - # Background circle — deep indigo - for i in range(int(r + 2), 0, -1): - t = i / r - cr = int(18 * (1 - t * 0.5)) - cg = int(12 * (1 - t * 0.4)) - cb = int(50 * (1 - t * 0.3)) - draw.ellipse([cx - i, cy - i, cx + i, cy + i], fill=(cr, cg, cb, 255)) - - # Subtle border - ring_w = max(2, size // 140) - draw.ellipse( - [cx - r, cy - r, cx + r, cy + r], - outline=(80, 120, 240, 60), - width=ring_w, - ) - - # --- Big bold white C, left-center, nearly filling the circle height --- - font_size = int(size * 0.58) - font = None - for font_name in ["segoeuib.ttf", "arialbd.ttf", "segoeui.ttf"]: - try: - font = ImageFont.truetype(f"C:/Windows/Fonts/{font_name}", font_size) - break - except OSError: - continue - - # Position: left side, vertically centered - bbox = draw.textbbox((0, 0), "C", font=font) - tw = bbox[2] - bbox[0] - th = bbox[3] - bbox[1] - # C sits in the left ~45% of the circle - tx = cx * 0.42 - tw / 2 - bbox[0] - ty = cy - th / 2 - bbox[1] - - # C center and radius for placing branch origins - c_optical_x = cx * 0.42 - c_optical_y = cy - c_arc_radius = th * 0.44 - - # --- Branch origin points along the C's opening (right side of C) --- - # The C opens to the right. Branches sprout from the opening and outer curve. - branch_origins = [] - - # Points along the open side of C (top-right to bottom-right, ~330 to 30 deg) - for deg in [345, 0, 15]: - rad = math.radians(deg) - ox = c_optical_x + c_arc_radius * math.cos(rad) - oy = c_optical_y - c_arc_radius * math.sin(rad) - branch_origins.append((ox, oy, "open")) - - # Points along the outer curve of C (top, right-top, etc) - for deg in [60, 90, 120, 160, 200, 240, 270, 300]: - rad = math.radians(deg) - ox = c_optical_x + c_arc_radius * 1.05 * math.cos(rad) - oy = c_optical_y - c_arc_radius * 1.05 * math.sin(rad) - branch_origins.append((ox, oy, "curve")) - - # --- Generate branching nodes --- - nodes = [] # (x, y, size_class) size_class: "big", "med", "small" - edges = [] # (idx_a, idx_b) - - # Gen 1: first nodes branching from C toward the right - for ox, oy, src in branch_origins: - # Direction: away from C center, biased right - dx = ox - c_optical_x - dy = oy - c_optical_y - length = math.hypot(dx, dy) or 1 - dx, dy = dx / length, dy / length - - # Strong rightward bias for open-side origins - if src == "open": - dx = dx * 0.3 + 0.9 - dy = dy * 0.5 - else: - dx = dx * 0.5 + 0.5 - dy = dy * 0.7 - - length = math.hypot(dx, dy) - dx, dy = dx / length, dy / length - - branch_len = r * 0.30 - nx = ox + dx * branch_len - ny = oy + dy * branch_len - - # Clamp inside circle - dist = math.hypot(nx - cx, ny - cy) - if dist > r * 0.85: - scale = (r * 0.85) / dist - nx = cx + (nx - cx) * scale - ny = cy + (ny - cy) * scale - - parent_idx = len(nodes) - nodes.append((nx, ny, "big")) - edges.append((-1, parent_idx, ox, oy)) # -1 means origin is on the C - - # Gen 2: secondary branches from gen-1 - gen1_count = len(nodes) - for i in range(gen1_count): - nx, ny, _ = nodes[i] - # Find original direction - edge = edges[i] - ox, oy = edge[2], edge[3] - dx = nx - ox - dy = ny - oy - length = math.hypot(dx, dy) or 1 - dx, dy = dx / length, dy / length - - # 1-2 sub-branches - spreads = [-0.45, 0.45] if i % 2 == 0 else [0.0] - for spread in spreads: - sdx = dx * math.cos(spread) - dy * math.sin(spread) - sdy = dx * math.sin(spread) + dy * math.cos(spread) - - branch_len = r * 0.22 - bx = nx + sdx * branch_len - by = ny + sdy * branch_len - - dist = math.hypot(bx - cx, by - cy) - if dist > r * 0.88: - scale = (r * 0.88) / dist - bx = cx + (bx - cx) * scale - by = cy + (by - cy) * scale - - child_idx = len(nodes) - nodes.append((bx, by, "med")) - edges.append((i, child_idx, nx, ny)) - - # Gen 3: tiny terminal nodes - gen2_count = len(nodes) - for i in range(gen1_count, gen2_count): - nx, ny, _ = nodes[i] - edge = edges[i] - px, py = edge[2], edge[3] - dx = nx - px - dy = ny - py - length = math.hypot(dx, dy) or 1 - dx, dy = dx / length, dy / length - - branch_len = r * 0.14 - bx = nx + dx * branch_len - by = ny + dy * branch_len - - dist = math.hypot(bx - cx, by - cy) - if dist > r * 0.90: - scale = (r * 0.90) / dist - bx = cx + (bx - cx) * scale - by = cy + (by - cy) * scale - - child_idx = len(nodes) - nodes.append((bx, by, "small")) - edges.append((i, child_idx, nx, ny)) - - # --- Draw edges (from C and between nodes) --- - line_w = max(1, size // 180) - for edge in edges: - a_idx, b_idx = edge[0], edge[1] - x2, y2 = nodes[b_idx][0], nodes[b_idx][1] - if a_idx == -1: - x1, y1 = edge[2], edge[3] - else: - x1, y1 = nodes[a_idx][0], nodes[a_idx][1] - - gen = nodes[b_idx][2] - if gen == "big": - color = (200, 220, 255, 60) - elif gen == "med": - color = (180, 200, 250, 45) - else: - color = (160, 185, 240, 30) - draw.line([(x1, y1), (x2, y2)], fill=color, width=line_w) - - # Lateral connections between nearby same-gen nodes - for i in range(len(nodes)): - for j in range(i + 1, len(nodes)): - if nodes[i][2] == nodes[j][2]: - d = math.hypot(nodes[j][0] - nodes[i][0], nodes[j][1] - nodes[i][1]) - if d < r * 0.22: - draw.line( - [(nodes[i][0], nodes[i][1]), (nodes[j][0], nodes[j][1])], - fill=(150, 180, 240, 25), - width=max(1, size // 256), - ) - - # --- Draw nodes (all white) --- - for x, y, sz in nodes: - if sz == "big": - nr = size * 0.024 - elif sz == "med": - nr = size * 0.016 - else: - nr = size * 0.010 - - # Subtle glow - glow_r = nr * 2.5 - for g in range(int(glow_r), 0, -1): - a = int(30 * (1 - g / glow_r)) - draw.ellipse([x - g, y - g, x + g, y + g], fill=(200, 220, 255, a)) - - # White node - draw.ellipse( - [x - nr, y - nr, x + nr, y + nr], - fill=(255, 255, 255, 240), - outline=(220, 230, 255, 255), - width=max(1, size // 300), - ) - - # --- Draw white C on top --- - # Dark halo behind - halo_size = max(5, size // 40) - for offset in range(halo_size, 0, -1): - a = int(180 * (1 - offset / halo_size)) - draw.text((tx, ty), "C", fill=(12, 8, 35, a), font=font, - stroke_width=offset, stroke_fill=(12, 8, 35, a)) - - draw.text((tx, ty), "C", fill=(255, 255, 255, 255), font=font) - - return img - - -icons_dir = os.path.join(os.path.dirname(__file__), "src-tauri", "icons") - smile_path = os.path.join(icons_dir, "icon_source.png") -master = Image.open(smile_path).convert("RGBA").resize((512, 512), Image.LANCZOS) - -sizes = { - "icon.png": 512, "32x32.png": 32, "64x64.png": 64, - "128x128.png": 128, "128x128@2x.png": 256, - "Square30x30Logo.png": 30, "Square44x44Logo.png": 44, - "Square71x71Logo.png": 71, "Square89x89Logo.png": 89, - "Square107x107Logo.png": 107, "Square142x142Logo.png": 142, - "Square150x150Logo.png": 150, "Square284x284Logo.png": 284, - "Square310x310Logo.png": 310, "StoreLogo.png": 50, -} - -for name, sz in sizes.items(): - (master.copy() if sz == 512 else master.resize((sz, sz), Image.LANCZOS)).save( - os.path.join(icons_dir, name)) - print(f" {name} ({sz}x{sz})") - -ico_sizes = [16, 24, 32, 48, 64, 128, 256] -ico_imgs = [master.resize((s, s), Image.LANCZOS) for s in ico_sizes] -ico_imgs[0].save(os.path.join(icons_dir, "icon.ico"), format="ICO", - sizes=[(s, s) for s in ico_sizes], append_images=ico_imgs[1:]) -print(" icon.ico") -master.save(os.path.join(icons_dir, "icon.icns")) -print(" icon.icns\nDone!") diff --git a/desktop/cortex-control-center/package-lock.json b/desktop/cortex-control-center/package-lock.json index 078c3193..ddade92c 100644 --- a/desktop/cortex-control-center/package-lock.json +++ b/desktop/cortex-control-center/package-lock.json @@ -187,9 +187,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -207,9 +204,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -227,9 +221,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -247,9 +238,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -267,9 +255,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -287,9 +272,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -482,9 +464,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 OR MIT", "optional": true, "os": [ @@ -502,9 +481,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0 OR MIT", "optional": true, "os": [ @@ -522,9 +498,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 OR MIT", "optional": true, "os": [ @@ -542,9 +515,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 OR MIT", "optional": true, "os": [ @@ -562,9 +532,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0 OR MIT", "optional": true, "os": [ @@ -1065,9 +1032,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1089,9 +1053,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1113,9 +1074,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1137,9 +1095,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/desktop/cortex-control-center/package.json b/desktop/cortex-control-center/package.json index ea7e667c..63fb7382 100644 --- a/desktop/cortex-control-center/package.json +++ b/desktop/cortex-control-center/package.json @@ -17,7 +17,6 @@ "tauri": "tauri", "test": "vitest run", "test:watch": "vitest", - "mock:cortex": "node scripts/mock-cortex-server.mjs", "verify:lifecycle:dev": "node scripts/run-dev-lifecycle-verification.mjs" }, "dependencies": { diff --git a/desktop/cortex-control-center/scripts/cleanup-dev-runtime.mjs b/desktop/cortex-control-center/scripts/cleanup-dev-runtime.mjs index ceba348d..70e0dba3 100644 --- a/desktop/cortex-control-center/scripts/cleanup-dev-runtime.mjs +++ b/desktop/cortex-control-center/scripts/cleanup-dev-runtime.mjs @@ -1,22 +1,4 @@ -import { spawnSync } from "node:child_process"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const projectDir = resolve(scriptDir, ".."); - -function escapeForPowerShell(value) { - return String(value).replace(/'/g, "''"); -} - -export function cleanupDevRuntime({ quiet = false } = {}) { - if (process.platform !== "win32") { - return { cleaned: false, reason: "non-windows" }; - } - - const psProject = escapeForPowerShell(projectDir); - const psSelfPid = Number(process.pid) || 0; - const psScript = ` +import{spawnSync}from"node:child_process";import{dirname,resolve}from"node:path";import{fileURLToPath,pathToFileURL}from"node:url";const scriptDir=dirname(fileURLToPath(import.meta.url)),projectDir=resolve(scriptDir,"..");function escapeForPowerShell(value){return String(value).replace(/'/g,"''")}function cleanupDevRuntime({quiet=!1}={}){if(process.platform!=="win32")return{cleaned:!1,reason:"non-windows"};const psProject=escapeForPowerShell(projectDir),psSelfPid=Number(process.pid)||0,psScript=` $project = '${psProject}' $selfPid = ${psSelfPid} $killed = @() @@ -94,55 +76,4 @@ $result = [pscustomobject]@{ errors = $errors } $result | ConvertTo-Json -Compress -`; - - const result = spawnSync( - "powershell.exe", - ["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", psScript], - { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - windowsHide: true, - }, - ); - - if (result.error) { - if (!quiet) { - console.warn(`[dev-cleanup] failed to run cleanup: ${result.error.message}`); - } - return { cleaned: false, reason: result.error.message }; - } - - if (!quiet && result.stderr?.trim()) { - console.warn(`[dev-cleanup] ${result.stderr.trim()}`); - } - - let payload = { killed: [], cleanedSessions: [], errors: [] }; - try { - payload = JSON.parse(result.stdout?.trim() || "{\"killed\":[],\"cleanedSessions\":[],\"errors\":[]}"); - } catch { - // Keep defaults if parsing fails. - } - - if (!quiet && payload.killed.length) { - console.log(`[dev-cleanup] removed stale processes: ${payload.killed.join(", ")}`); - } - if (!quiet && payload.cleanedSessions.length) { - console.log(`[dev-cleanup] removed stale session wrappers: ${payload.cleanedSessions.length}`); - } - if (!quiet && payload.errors.length) { - console.warn(`[dev-cleanup] cleanup warnings: ${payload.errors.join("; ")}`); - } - - return { - cleaned: payload.killed.length > 0 || payload.cleanedSessions.length > 0, - killed: payload.killed, - cleanedSessions: payload.cleanedSessions, - errors: payload.errors, - }; -} - -const invokedUrl = process.argv[1] ? pathToFileURL(process.argv[1]).href : ""; -if (invokedUrl && import.meta.url === invokedUrl) { - cleanupDevRuntime(); -} +`,result=spawnSync("powershell.exe",["-NoProfile","-ExecutionPolicy","Bypass","-Command",psScript],{encoding:"utf8",stdio:["ignore","pipe","pipe"],windowsHide:!0});if(result.error)return quiet||console.warn(`[dev-cleanup] failed to run cleanup: ${result.error.message}`),{cleaned:!1,reason:result.error.message};!quiet&&result.stderr?.trim()&&console.warn(`[dev-cleanup] ${result.stderr.trim()}`);let payload={killed:[],cleanedSessions:[],errors:[]};try{payload=JSON.parse(result.stdout?.trim()||'{"killed":[],"cleanedSessions":[],"errors":[]}')}catch{}return!quiet&&payload.killed.length&&console.log(`[dev-cleanup] removed stale processes: ${payload.killed.join(", ")}`),!quiet&&payload.cleanedSessions.length&&console.log(`[dev-cleanup] removed stale session wrappers: ${payload.cleanedSessions.length}`),!quiet&&payload.errors.length&&console.warn(`[dev-cleanup] cleanup warnings: ${payload.errors.join("; ")}`),{cleaned:payload.killed.length>0||payload.cleanedSessions.length>0,killed:payload.killed,cleanedSessions:payload.cleanedSessions,errors:payload.errors}}const invokedUrl=process.argv[1]?pathToFileURL(process.argv[1]).href:"";invokedUrl&&import.meta.url===invokedUrl&&cleanupDevRuntime();export{cleanupDevRuntime}; diff --git a/desktop/cortex-control-center/scripts/ensure-daemon-dev-binary.mjs b/desktop/cortex-control-center/scripts/ensure-daemon-dev-binary.mjs index 0cc8b93b..634dfccd 100644 --- a/desktop/cortex-control-center/scripts/ensure-daemon-dev-binary.mjs +++ b/desktop/cortex-control-center/scripts/ensure-daemon-dev-binary.mjs @@ -1,164 +1,2 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, readdirSync, statSync } from "node:fs"; -import { dirname, extname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const projectDir = resolve(scriptDir, ".."); -const repoRoot = resolve(projectDir, "..", ".."); -const daemonRoot = resolve(repoRoot, "daemon-rs"); -const daemonManifestPath = resolve(repoRoot, "daemon-rs", "Cargo.toml"); -const daemonLockPath = resolve(repoRoot, "daemon-rs", "Cargo.lock"); -const daemonSrcPath = resolve(repoRoot, "daemon-rs", "src"); -const daemonTargetDir = resolve(repoRoot, "daemon-rs", "target-control-center-dev"); -const daemonBinary = process.platform === "win32" - ? resolve(daemonTargetDir, "debug", "cortex.exe") - : resolve(daemonTargetDir, "debug", "cortex"); - -const WATCHED_EXTENSIONS = new Set([".rs", ".toml", ".lock"]); -const IGNORED_DIR_NAMES = new Set([ - "target", - "target-control-center-dev", - "target-control-center-release", - ".tmp", - "builds", -]); - -function runCargoBuild() { - const command = process.platform === "win32" ? "cargo.exe" : "cargo"; - const args = [ - "build", - "--target-dir", - daemonTargetDir, - "--manifest-path", - daemonManifestPath, - ]; - const invokeBuild = () => spawnSync(command, args, { - cwd: projectDir, - stdio: "pipe", - windowsHide: true, - encoding: "utf8", - }); - const emitBuildOutput = (result) => { - if (result.stdout) process.stdout.write(result.stdout); - if (result.stderr) process.stderr.write(result.stderr); - }; - const isWindowsDevBinaryLockError = (result) => { - if (process.platform !== "win32") return false; - const text = `${result.stdout || ""}\n${result.stderr || ""}`.toLowerCase(); - return ( - text.includes("failed to remove file") && - text.includes("target-control-center-dev") && - text.includes("cortex.exe") && - text.includes("access is denied") - ); - }; - const stopConflictingDevDaemons = () => { - if (process.platform !== "win32") return; - const repoRootLiteral = repoRoot.replace(/'/g, "''"); - const daemonPathLiteral = daemonBinary.replace(/'/g, "''"); - const sharedDebugLiteral = resolve(repoRoot, "daemon-rs", "target", "debug", "cortex.exe") - .replace(/'/g, "''"); - const runtimeRootLiteral = resolve( - process.env.USERPROFILE || process.env.HOME || "", - ".cortex", - "runtime", - "control-center-dev", - ).replace(/'/g, "''"); - const script = [ - `$repoRoot='${repoRootLiteral}'.ToLower()`, - `$target='${daemonPathLiteral}'`, - `$sharedDebug='${sharedDebugLiteral}'.ToLower()`, - `$runtimeRoot='${runtimeRootLiteral}'.ToLower()`, - "$matches = Get-CimInstance Win32_Process | Where-Object { $_.ExecutablePath -and $_.ExecutablePath -ieq $target }", - "$stale = Get-CimInstance Win32_Process | Where-Object {", - " if (-not $_.ExecutablePath) { return $false }", - " $exe = $_.ExecutablePath.ToLower()", - " (($exe.StartsWith($repoRoot) -and $exe -eq $sharedDebug) -or ($runtimeRoot -and $exe.StartsWith($runtimeRoot) -and [System.IO.Path]::GetFileName($exe).StartsWith('cortex-dev-run')))", - "}", - "foreach ($proc in $matches) {", - " Write-Host \"[ensure-daemon] stopping locked dev daemon pid=$($proc.ProcessId) path=$($proc.ExecutablePath)\"", - " Stop-Process -Id $proc.ProcessId -Force -ErrorAction SilentlyContinue", - "}", - "foreach ($proc in $stale) {", - " Write-Host \"[ensure-daemon] stopping stale conflicting daemon pid=$($proc.ProcessId) path=$($proc.ExecutablePath)\"", - " Stop-Process -Id $proc.ProcessId -Force -ErrorAction SilentlyContinue", - "}", - ].join("; "); - const cleanupResult = spawnSync("powershell.exe", ["-NoProfile", "-Command", script], { - cwd: projectDir, - stdio: "pipe", - encoding: "utf8", - windowsHide: true, - }); - if (cleanupResult.stdout) { - process.stdout.write(cleanupResult.stdout); - } - if (cleanupResult.stderr) { - process.stderr.write(cleanupResult.stderr); - } - }; - - stopConflictingDevDaemons(); - let result = invokeBuild(); - emitBuildOutput(result); - if (typeof result.status === "number" && result.status === 0) { - return; - } - - if (isWindowsDevBinaryLockError(result)) { - console.warn("[ensure-daemon] dev daemon binary is locked; stopping old process and retrying build once"); - stopConflictingDevDaemons(); - result = invokeBuild(); - emitBuildOutput(result); - if (typeof result.status === "number" && result.status === 0) { - return; - } - } - - process.exit(result.status ?? 1); -} - -function latestInputMtimeMs(path) { - if (!existsSync(path)) return 0; - const info = statSync(path); - if (info.isFile()) return info.mtimeMs; - if (!info.isDirectory()) return 0; - - let newest = 0; - for (const entry of readdirSync(path, { withFileTypes: true })) { - if (entry.isDirectory()) { - if (IGNORED_DIR_NAMES.has(entry.name)) continue; - newest = Math.max(newest, latestInputMtimeMs(resolve(path, entry.name))); - continue; - } - if (!entry.isFile()) continue; - if (!WATCHED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue; - newest = Math.max(newest, statSync(resolve(path, entry.name)).mtimeMs); - } - return newest; -} - -function shouldRebuild() { - if (!existsSync(daemonBinary)) { - return { rebuild: true, reason: "missing binary" }; - } - const binaryMtime = statSync(daemonBinary).mtimeMs; - const inputMtime = Math.max( - latestInputMtimeMs(daemonManifestPath), - latestInputMtimeMs(daemonLockPath), - latestInputMtimeMs(daemonSrcPath), - ); - if (inputMtime > binaryMtime) { - return { rebuild: true, reason: "source newer than binary" }; - } - return { rebuild: false, reason: "binary up to date" }; -} - -const decision = shouldRebuild(); -if (decision.rebuild) { - console.log(`[ensure-daemon] building dev daemon binary (${decision.reason}) at ${daemonBinary}`); - runCargoBuild(); -} else { - console.log(`[ensure-daemon] using existing dev daemon binary (${decision.reason}) at ${daemonBinary}`); -} +import{spawnSync}from"node:child_process";import{existsSync,readdirSync,statSync}from"node:fs";import{dirname,extname,resolve}from"node:path";import{fileURLToPath}from"node:url";const scriptDir=dirname(fileURLToPath(import.meta.url)),projectDir=resolve(scriptDir,".."),repoRoot=resolve(projectDir,"..",".."),daemonRoot=resolve(repoRoot,"daemon-rs"),daemonManifestPath=resolve(repoRoot,"daemon-rs","Cargo.toml"),daemonLockPath=resolve(repoRoot,"daemon-rs","Cargo.lock"),daemonSrcPath=resolve(repoRoot,"daemon-rs","src"),daemonTargetDir=resolve(repoRoot,"daemon-rs","target-control-center-dev"),daemonBinary=process.platform==="win32"?resolve(daemonTargetDir,"debug","cortex.exe"):resolve(daemonTargetDir,"debug","cortex"),WATCHED_EXTENSIONS=new Set([".rs",".toml",".lock"]),IGNORED_DIR_NAMES=new Set(["target","target-control-center-dev","target-control-center-release",".tmp","builds"]);function runCargoBuild(){const command=process.platform==="win32"?"cargo.exe":"cargo",args=["build","--target-dir",daemonTargetDir,"--manifest-path",daemonManifestPath],invokeBuild=()=>spawnSync(command,args,{cwd:projectDir,stdio:"pipe",windowsHide:!0,encoding:"utf8"}),emitBuildOutput=result2=>{result2.stdout&&process.stdout.write(result2.stdout),result2.stderr&&process.stderr.write(result2.stderr)},isWindowsDevBinaryLockError=result2=>{if(process.platform!=="win32")return!1;const text=`${result2.stdout||""} +${result2.stderr||""}`.toLowerCase();return text.includes("failed to remove file")&&text.includes("target-control-center-dev")&&text.includes("cortex.exe")&&text.includes("access is denied")},stopConflictingDevDaemons=()=>{if(process.platform!=="win32")return;const repoRootLiteral=repoRoot.replace(/'/g,"''"),daemonPathLiteral=daemonBinary.replace(/'/g,"''"),sharedDebugLiteral=resolve(repoRoot,"daemon-rs","target","debug","cortex.exe").replace(/'/g,"''"),runtimeRootLiteral=resolve(process.env.USERPROFILE||process.env.HOME||"",".cortex","runtime","control-center-dev").replace(/'/g,"''"),script=[`$repoRoot='${repoRootLiteral}'.ToLower()`,`$target='${daemonPathLiteral}'`,`$sharedDebug='${sharedDebugLiteral}'.ToLower()`,`$runtimeRoot='${runtimeRootLiteral}'.ToLower()`,"$matches = Get-CimInstance Win32_Process | Where-Object { $_.ExecutablePath -and $_.ExecutablePath -ieq $target }","$stale = Get-CimInstance Win32_Process | Where-Object {"," if (-not $_.ExecutablePath) { return $false }"," $exe = $_.ExecutablePath.ToLower()"," (($exe.StartsWith($repoRoot) -and $exe -eq $sharedDebug) -or ($runtimeRoot -and $exe.StartsWith($runtimeRoot) -and [System.IO.Path]::GetFileName($exe).StartsWith('cortex-dev-run')))","}","foreach ($proc in $matches) {",' Write-Host "[ensure-daemon] stopping locked dev daemon pid=$($proc.ProcessId) path=$($proc.ExecutablePath)"'," Stop-Process -Id $proc.ProcessId -Force -ErrorAction SilentlyContinue","}","foreach ($proc in $stale) {",' Write-Host "[ensure-daemon] stopping stale conflicting daemon pid=$($proc.ProcessId) path=$($proc.ExecutablePath)"'," Stop-Process -Id $proc.ProcessId -Force -ErrorAction SilentlyContinue","}"].join("; "),cleanupResult=spawnSync("powershell.exe",["-NoProfile","-Command",script],{cwd:projectDir,stdio:"pipe",encoding:"utf8",windowsHide:!0});cleanupResult.stdout&&process.stdout.write(cleanupResult.stdout),cleanupResult.stderr&&process.stderr.write(cleanupResult.stderr)};stopConflictingDevDaemons();let result=invokeBuild();emitBuildOutput(result),!(typeof result.status=="number"&&result.status===0)&&(isWindowsDevBinaryLockError(result)&&(console.warn("[ensure-daemon] dev daemon binary is locked; stopping old process and retrying build once"),stopConflictingDevDaemons(),result=invokeBuild(),emitBuildOutput(result),typeof result.status=="number"&&result.status===0)||process.exit(result.status??1))}function latestInputMtimeMs(path){if(!existsSync(path))return 0;const info=statSync(path);if(info.isFile())return info.mtimeMs;if(!info.isDirectory())return 0;let newest=0;for(const entry of readdirSync(path,{withFileTypes:!0})){if(entry.isDirectory()){if(IGNORED_DIR_NAMES.has(entry.name))continue;newest=Math.max(newest,latestInputMtimeMs(resolve(path,entry.name)));continue}entry.isFile()&&WATCHED_EXTENSIONS.has(extname(entry.name).toLowerCase())&&(newest=Math.max(newest,statSync(resolve(path,entry.name)).mtimeMs))}return newest}function shouldRebuild(){if(!existsSync(daemonBinary))return{rebuild:!0,reason:"missing binary"};const binaryMtime=statSync(daemonBinary).mtimeMs;return Math.max(latestInputMtimeMs(daemonManifestPath),latestInputMtimeMs(daemonLockPath),latestInputMtimeMs(daemonSrcPath))>binaryMtime?{rebuild:!0,reason:"source newer than binary"}:{rebuild:!1,reason:"binary up to date"}}const decision=shouldRebuild();decision.rebuild?(console.log(`[ensure-daemon] building dev daemon binary (${decision.reason}) at ${daemonBinary}`),runCargoBuild()):console.log(`[ensure-daemon] using existing dev daemon binary (${decision.reason}) at ${daemonBinary}`); diff --git a/desktop/cortex-control-center/scripts/extract-panels.mjs b/desktop/cortex-control-center/scripts/extract-panels.mjs deleted file mode 100644 index d93bfb54..00000000 --- a/desktop/cortex-control-center/scripts/extract-panels.mjs +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env node -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); -const bak = fs.readFileSync(path.join(SRC, "App.jsx.bak"), "utf8").split("\n"); - -function slice(start, end) { - return bak.slice(start - 1, end).join("\n"); -} - -function write(rel, content) { - fs.mkdirSync(path.dirname(path.join(SRC, rel)), { recursive: true }); - fs.writeFileSync(path.join(SRC, rel), content.endsWith("\n") ? content : `${content}\n`); -} - -const HOOK_KEYS = fs.readFileSync(path.join(ROOT, "scripts/split-panels-and-hooks.mjs"), "utf8") - .match(/const HOOK_RETURN_KEYS = `([\s\S]*?)`;/)[1]; - -const IMPORTS = `import { AppIcon } from "../../ui-icons.jsx"; -import { CURRENCY_OPTIONS, SAVINGS_OPERATION_LABELS, SAVINGS_USD_PER_MILLION, SAVINGS_HISTORY_DAYS, timeAgo, MISSION_METRIC_LEGEND, CONTROL_CENTER_VERSION, ANALYTICS_METRIC_LEGEND } from "../../constants.js"; -import { BUDGET_ENDPOINT_DEFINITIONS } from "../../settings/settings-state.js"; -import { handleKeyboardActivation } from "../../keyboard-access.js"; -import { sameAgent } from "../../live-surface.js"; -import { normalizeCurrencyCode, formatDaemonEndpoint } from "../utils/format.js"; -import { conflictBadgeClass } from "../normalize/conflicts.js"; -import { agentColor } from "../utils/agent-color.js"; -import { AnimatedNumber } from "../components/AnimatedNumber.jsx"; -import { Sparkline } from "../components/Sparkline.jsx"; -import { MonteCarloProjectionChart } from "../components/MonteCarloProjectionChart.jsx"; -import { EmptyItem } from "../components/common.jsx"; -import { AgentItem } from "../components/AgentItem.jsx"; -import { OperatorSelector } from "../components/OperatorSelector.jsx"; -import { TaskItem } from "../components/TaskItem.jsx"; -import { LockItem } from "../components/LockItem.jsx"; -import { FeedItem } from "../components/FeedItem.jsx"; -import { MessageItem } from "../components/MessageItem.jsx"; -import { ActivityItem } from "../components/ActivityItem.jsx"; -import { ConflictPairCard } from "../components/ConflictPairCard.jsx"; -import { PANEL_SEQUENCE } from "../constants.js"; -`; - -function panel(name, start, end) { - const body = slice(start, end); - write( - `app/panels/${name}.jsx`, - `${IMPORTS} -export function ${name}(p) { - const { - ${HOOK_KEYS}, - } = p; - - return ( - <> -${body} - - ); -} -`, - ); -} - -panel("SettingsPanel", 4606, 4869); -panel("OverviewPanel", 4871, 5208); -panel("AgentsPanel", 5211, 5296); -panel("WorkPanel", 5298, 5581); -panel("MemoryPanel", 5583, 5838); -panel("AnalyticsPanel", 5846, 6307); -panel("ConflictsPanel", 6337, 6364); -panel("AboutPanel", 6366, 6465); - -write( - "app/panels/panel-stage.jsx", - `${IMPORTS}import { BrainVisualizerPanel } from "../components/BrainVisualizerPanel.jsx"; -import { SettingsPanel } from "./SettingsPanel.jsx"; -import { OverviewPanel } from "./OverviewPanel.jsx"; -import { AgentsPanel } from "./AgentsPanel.jsx"; -import { WorkPanel } from "./WorkPanel.jsx"; -import { MemoryPanel } from "./MemoryPanel.jsx"; -import { AnalyticsPanel } from "./AnalyticsPanel.jsx"; -import { ConflictsPanel } from "./ConflictsPanel.jsx"; -import { AboutPanel } from "./AboutPanel.jsx"; - -export function PanelStage(p) { - return ( -
- - - - - - - - - -
- ); -} -`, -); - -console.log("Panels extracted from backup"); diff --git a/desktop/cortex-control-center/scripts/fix-refactor.mjs b/desktop/cortex-control-center/scripts/fix-refactor.mjs deleted file mode 100644 index 834f05f7..00000000 --- a/desktop/cortex-control-center/scripts/fix-refactor.mjs +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env node -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); - -function read(p) { - return fs.readFileSync(path.join(SRC, p), "utf8"); -} - -function write(p, content) { - fs.writeFileSync(path.join(SRC, p), content.endsWith("\n") ? content : `${content}\n`); -} - -function exportConsts(filePath) { - let content = read(filePath); - content = content.replace(/^\/\/ Matches daemon-rs.*\n\/\/ Matches daemon-rs.*\n/m, "// Matches daemon-rs/src/main.rs:DEFAULT_CORTEX_PORT. Bump both simultaneously.\n"); - content = content - .replace(/^const /gm, "export const ") - .replace(/^function /gm, "export function "); - write(filePath, content); -} - -function exportFunctions(filePath) { - let content = read(filePath); - content = content.replace(/^function /gm, "export function ").replace(/^async function /gm, "export async function "); - write(filePath, content); -} - -exportConsts("app/constants.js"); -exportFunctions("app/browser-bootstrap.js"); -exportFunctions("app/utils/format.js"); -exportFunctions("app/utils/daemon.js"); -exportFunctions("app/utils/agent-color.js"); -exportFunctions("app/components/sparkline-utils.js"); - -// conflicts.js - export functions but keep internal pickDefined -{ - let content = read("app/normalize/conflicts.js"); - content = content.replace(/^import.*\n\n/, ""); - content = content - .replace(/^function pickDefined/gm, "function pickDefined") - .replace(/^function normalizeConflict/gm, "export function normalizeConflict") - .replace(/^function extractEntity/gm, "export function extractEntity") - .replace(/^function formatConfidence/gm, "export function formatConfidence") - .replace(/^function formatTrust/gm, "export function formatTrust") - .replace(/^function formatTimestamp/gm, "export function formatTimestamp") - .replace(/^function conflictBadge/gm, "export function conflictBadge") - .replace(/^function isRouteMissing/gm, "export function isRouteMissing") - .replace(/^function toFinite/gm, "export function toFinite") - .replace(/^const CONFLICT_/gm, "export const CONFLICT_"); - // Fix normalizeConflictEntry etc - content = content.replace(/^function normalizeConflictEntry/gm, "export function normalizeConflictEntry"); - content = content.replace(/^function normalizeConflictResolution/gm, "export function normalizeConflictResolution"); - content = content.replace(/^function normalizeConflictPair/gm, "export function normalizeConflictPair"); - content = content.replace(/^function normalizeConflictPairsPayload/gm, "export function normalizeConflictPairsPayload"); - content = content.replace(/^function normalizeConflictClassification/gm, "export function normalizeConflictClassification"); - content = content.replace(/^function normalizeConflictStatus/gm, "export function normalizeConflictStatus"); - write("app/normalize/conflicts.js", content); -} - -exportFunctions("app/normalize/permissions.js"); -exportFunctions("app/normalize/sessions.js"); - -// Bundle CSS for tests + backward compat -const cssFiles = [ - "styles/base.css", - "styles/layout.css", - "styles/components.css", - "styles/topbar.css", - "styles/animations.css", - "styles/charts.css", - "styles/panels/analytics.css", - "styles/panels/coming-soon.css", - "styles/panels/brain.css", - "styles/overrides-2026.css", - "styles/sidebar-collapse.css", - "styles/connection-dialog.css", - "styles/panels/conflicts.css", - "styles/accessibility.css", -]; -const bundled = cssFiles.map((f) => read(f)).join("\n"); -write("styles.css", bundled); - -// Panel-stage: full destructuring from hook return keys -const returnBlock = read("app/hooks/useDashboardHooks.js").match(/return \{([\s\S]*?)\n \};\n\}/)?.[1] ?? ""; -const keys = [...returnBlock.matchAll(/^\s{4}(\w+),?$/gm)].map((m) => m[1]); -const destructure = `export function PanelStage(p) {\n const {\n ${keys.join(",\n ")},\n } = p;\n`; - -let panelStage = read("app/panels/panel-stage.jsx"); -panelStage = panelStage.replace( - /export function PanelStage\(props\) \{[\s\S]*?return \(/, - `${destructure}\n return (`, -); -write("app/panels/panel-stage.jsx", panelStage); - -// Hook imports fix -{ - let hook = read("app/hooks/useDashboardHooks.js"); - hook = hook.replace( - 'import {\n normalizeConflictPairsPayload,\n} from "../normalize/conflicts.js";', - `import { - isRouteMissingError, - normalizeConflictPairsPayload, -} from "../normalize/conflicts.js";`, - ); - hook = hook.replace( - 'import {\n readBrowserBootstrap,', - `import { priorityRank } from "../browser-bootstrap.js";\nimport {\n readBrowserBootstrap,`, - ); - // Add refreshAllRef to return if missing - if (!hook.includes("refreshAllRef,")) { - hook = hook.replace(" invokeRef,", " invokeRef,\n refreshAllRef,"); - } - if (!hook.includes("setUpdateInstalling,")) { - hook = hook.replace(" updateInstalling,", " updateInstalling,\n setUpdateInstalling,\n setFeedbackMessage,"); - } - write("app/hooks/useDashboardHooks.js", hook); -} - -// browser-bootstrap: move normalizeCurrencyCode to format.js -{ - let bootstrap = read("app/browser-bootstrap.js"); - bootstrap = bootstrap.replace(/\nfunction normalizeCurrencyCode[\s\S]*?\n\}/, ""); - bootstrap = bootstrap.replace(/\nfunction priorityRank[\s\S]*?\n\}/, ""); - write("app/browser-bootstrap.js", bootstrap); - - let format = read("app/utils/format.js"); - if (!format.includes("normalizeCurrencyCode")) { - format = `import { CURRENCY_OPTIONS } from "../constants.js";\n${format.replace(/^import[^\n]+\nimport[^\n]+\n\n/, "")}`; - format = `${format.trim()}\n\nexport function normalizeCurrencyCode(raw) {\n const candidate = String(raw || "").trim().toUpperCase();\n return CURRENCY_OPTIONS.includes(candidate) ? candidate : "USD";\n}\n\nexport function priorityRank(priority) {\n const map = { critical: 4, high: 3, medium: 2, low: 1 };\n return map[priority] || 0;\n}\n`; - write("app/utils/format.js", format); - } - - let hook = read("app/hooks/useDashboardHooks.js"); - hook = hook.replace( - 'import { priorityRank } from "../browser-bootstrap.js";\n', - "", - ); - hook = hook.replace( - 'import { normalizeCurrencyCode, formatDaemonEndpoint, getOsReducedMotionPreference } from "../utils/format.js";', - 'import { normalizeCurrencyCode, formatDaemonEndpoint, getOsReducedMotionPreference, priorityRank } from "../utils/format.js";', - ); - write("app/hooks/useDashboardHooks.js", hook); -} - -console.log("Fixed exports, CSS bundle, panel destructuring"); diff --git a/desktop/cortex-control-center/scripts/mock-cortex-server.mjs b/desktop/cortex-control-center/scripts/mock-cortex-server.mjs deleted file mode 100644 index 82dc22a4..00000000 --- a/desktop/cortex-control-center/scripts/mock-cortex-server.mjs +++ /dev/null @@ -1,707 +0,0 @@ -import http from "node:http"; -import { fileURLToPath } from "node:url"; - -const DEFAULT_TOKEN = process.env.EXPECT_SMOKE_TOKEN || "expect-smoke-token"; - -function isoMinutesAgo(minutes) { - return new Date(Date.now() - minutes * 60_000).toISOString(); -} - -function isoDaysAgo(days) { - return new Date(Date.now() - days * 86_400_000).toISOString(); -} - -function nowIso() { - return new Date().toISOString(); -} - -function buildSavingsFixture() { - const dailyValues = [ - { saved: 92_000, boots: 5, hitRatePct: 96 }, - { saved: 104_000, boots: 6, hitRatePct: 95 }, - { saved: 121_000, boots: 5, hitRatePct: 97 }, - { saved: 118_000, boots: 6, hitRatePct: 96 }, - { saved: 134_000, boots: 7, hitRatePct: 96 }, - { saved: 148_000, boots: 8, hitRatePct: 97 }, - { saved: 162_000, boots: 8, hitRatePct: 98 }, - { saved: 171_000, boots: 7, hitRatePct: 97 }, - { saved: 188_000, boots: 9, hitRatePct: 98 }, - { saved: 214_000, boots: 10, hitRatePct: 98 }, - { saved: 226_000, boots: 9, hitRatePct: 97 }, - { saved: 239_000, boots: 11, hitRatePct: 98 }, - { saved: 253_000, boots: 10, hitRatePct: 99 }, - { saved: 268_000, boots: 11, hitRatePct: 98 }, - ]; - - let savedTotal = 0; - let servedTotal = 0; - let baselineTotal = 0; - - const daily = dailyValues.map((row, index) => { - const baseline = 480_000 + index * 24_000; - const served = baseline - row.saved; - savedTotal += row.saved; - servedTotal += served; - baselineTotal += baseline; - return { - date: isoDaysAgo(dailyValues.length - 1 - index).slice(0, 10), - saved: row.saved, - boots: row.boots, - baseline, - served, - hitRatePct: row.hitRatePct, - }; - }); - - const totalBoots = daily.reduce((sum, row) => sum + row.boots, 0); - - return { - summary: { - totalSaved: savedTotal, - totalServed: servedTotal, - totalBaseline: baselineTotal, - totalBoots, - avgPercent: Math.round((savedTotal / baselineTotal) * 100), - avgSavedPerBoot: Math.round(savedTotal / totalBoots), - avgServedPerBoot: Math.round(servedTotal / totalBoots), - avgBaselinePerBoot: Math.round(baselineTotal / totalBoots), - }, - daily, - cumulative: daily.map((row, index) => ({ - date: row.date, - savedTotal: daily.slice(0, index + 1).reduce((sum, item) => sum + item.saved, 0), - })), - recallTrend: daily.map((row) => ({ - date: row.date, - hitRatePct: row.hitRatePct, - })), - activityHeatmap: [ - { day: "Mon", hour: 9, count: 4 }, - { day: "Mon", hour: 10, count: 6 }, - { day: "Tue", hour: 14, count: 7 }, - { day: "Wed", hour: 15, count: 8 }, - { day: "Thu", hour: 20, count: 5 }, - { day: "Fri", hour: 11, count: 6 }, - { day: "Sat", hour: 13, count: 3 }, - ], - byAgent: [ - { agent: "Codex", saved: 718_000, served: 1_140_000, percent: 39, boots: 18 }, - { agent: "Claude", saved: 904_000, served: 1_420_000, percent: 39, boots: 22 }, - { agent: "Factory Droid", saved: 532_000, served: 960_000, percent: 36, boots: 12 }, - ], - byOperation: [ - { operation: "boot", saved: 1_430_000, served: 2_180_000, baseline: 3_610_000, events: 52 }, - { operation: "recall", saved: 541_000, served: 884_000, baseline: 1_425_000, events: 94 }, - { operation: "store", saved: 197_000, served: 410_000, baseline: 607_000, events: 40 }, - { operation: "tool", saved: 121_000, served: 215_000, baseline: 336_000, events: 27 }, - ], - recent: [ - { - timestamp: isoMinutesAgo(18), - agent: "Codex", - percent: 42, - served: 36_000, - baseline: 62_000, - saved: 26_000, - admitted: 12, - rejected: 3, - }, - { - timestamp: isoMinutesAgo(62), - agent: "Claude", - percent: 38, - served: 41_000, - baseline: 66_000, - saved: 25_000, - admitted: 14, - rejected: 2, - }, - { - timestamp: isoMinutesAgo(135), - agent: "Factory Droid", - percent: 35, - served: 34_000, - baseline: 52_000, - saved: 18_000, - admitted: 9, - rejected: 1, - }, - ], - }; -} - -function buildFixture() { - const savings = buildSavingsFixture(); - return { - health: { - status: "ok", - embedding_status: "available", - storage_bytes: 12_914_688, - backup_count: 3, - log_bytes: 412_672, - runtime: { - exe_path: "C:/cortex-test/testuser/cortex/.cortex/runtime/mock/cortexd.exe", - pid_path: "C:/cortex-test/testuser/cortex/.cortex/cortex.pid", - token_path: "C:/cortex-test/testuser/cortex/.cortex/cortex.token", - port: 7437, - version: "0.5.0", - }, - budgets: { - configLoaded: true, - config_loaded: true, - enabled: true, - source: "budgets.toml", - error: null, - endpoints: { - recall: { limit: 300, windowSeconds: 60, window_seconds: 60 }, - store: { limit: 180, windowSeconds: 60, window_seconds: 60 }, - mcp: { limit: 120, windowSeconds: 60, window_seconds: 60 }, - }, - recentDenials: 2, - recent_denials: 2, - }, - stats: { - memories: 239, - decisions: 61, - events: 812, - }, - }, - sessions: { - sessions: [ - { - sessionId: "codex-smoke", - agent: "Codex", - description: "Validating desktop flows", - project: "cortex-control-center", - files: ["src/App.jsx", "src/BrainVisualizer.jsx", ".github/workflows/ci.yml"], - lastHeartbeat: isoMinutesAgo(2), - }, - { - sessionId: "claude-review", - agent: "Claude", - description: "Reviewing analytics polish", - project: "desktop polish", - files: ["src/styles.css"], - lastHeartbeat: isoMinutesAgo(6), - }, - ], - }, - locks: { - locks: [ - { - id: "lock-1", - path: "desktop/cortex-control-center/src/App.jsx", - agent: "Codex", - expiresAt: new Date(Date.now() + 52 * 60_000).toISOString(), - }, - ], - }, - tasks: { - tasks: [ - { - taskId: "task-0", - title: "Wire operator actions into the Work surface", - description: "Claim, complete, abandon, message, unlock, and ack should all work from one operator control.", - status: "pending", - priority: "high", - project: "desktop", - files: ["src/App.jsx", "src/live-surface.js"], - }, - { - taskId: "task-1", - title: "Smoke verify analytics shell", - status: "in_progress", - priority: "high", - claimedBy: "Codex", - summary: "Checking auth bootstrap and expect harness", - claimedAt: isoMinutesAgo(11), - project: "desktop", - }, - { - taskId: "task-2", - title: "Review brain HUD spacing", - status: "done", - priority: "medium", - claimedBy: "Claude", - summary: "CSS overlap pass complete", - completedAt: isoMinutesAgo(37), - project: "desktop", - }, - ], - }, - feed: { - entries: [ - { - id: "feed-1", - kind: "status", - agent: "Codex", - timestamp: isoMinutesAgo(9), - priority: "high", - tokens: 318, - summary: "Browser smoke harness switched to mock-backed auth bootstrap.", - files: ["scripts/mock-cortex-server.mjs"], - }, - { - id: "feed-2", - kind: "review", - agent: "Claude", - timestamp: isoMinutesAgo(31), - summary: "Brain view title updated to Neural topology.", - files: ["src/BrainVisualizer.jsx"], - }, - { - id: "feed-3", - kind: "task_complete", - agent: "Factory Droid", - timestamp: isoMinutesAgo(2), - summary: "Completed live surface review pass for the task queue.", - taskId: "task-9", - files: ["src/live-surface.test.js"], - }, - ], - }, - messages: { - messages: [ - { - id: "msg-1", - from: "Claude", - to: "Codex", - timestamp: isoMinutesAgo(22), - message: "Analytics summary box looks stable in the latest build.", - }, - { - id: "msg-2", - from: "Factory Droid", - to: "Codex", - timestamp: isoMinutesAgo(48), - message: "CI notes ready once the smoke harness lands.", - }, - ], - }, - activity: { - activities: [ - { - id: "activity-1", - agent: "Codex", - timestamp: isoMinutesAgo(5), - description: "Patched CI workflow and local browser smoke scripts.", - files: [".github/workflows/ci.yml", "desktop/cortex-control-center/package.json"], - }, - { - id: "activity-2", - agent: "Claude", - timestamp: isoMinutesAgo(28), - description: "Reviewed brain HUD overlap fix and analytics spacing.", - files: ["src/styles.css", "src/BrainVisualizer.jsx"], - }, - ], - }, - savings, - conflicts: { - pairs: [ - { - left: { - id: 41, - source_agent: "Codex", - created_at: isoMinutesAgo(240), - decision: "Run browser verification from a mocked Cortex API.", - context: "Keeps expect smoke deterministic for source builds and CI.", - confidence: 0.93, - }, - right: { - id: 42, - source_agent: "Claude", - created_at: isoMinutesAgo(238), - decision: "Drive browser verification against a live daemon only.", - context: "Rejected because auth and local state make it flaky.", - confidence: 0.42, - }, - }, - ], - }, - dump: { - memories: [ - { id: 1, source: "memory::browser_smoke_ci", text: "Browser smoke should use a mock daemon on port 7437 for browser fallback.", source_agent: "Codex", score: 5 }, - { id: 2, source: "memory::analytics_projection", text: "Monte Carlo chart now shows a fixed summary box instead of stacked endpoint labels.", source_agent: "Codex", score: 4 }, - { id: 3, source: "memory::brain_title", text: "Brain page headline should use Neural topology, not Jarvis.", source_agent: "Claude", score: 4 }, - { id: 4, source: "memory::activity_stream", text: "Recent activity panel should be populated for smoke coverage.", source_agent: "Factory Droid", score: 3 }, - ], - decisions: [ - { id: 11, decision: "Prefer browser-harness for deterministic browser verification.", context: "Avoids Playwright-based browser tooling in local dev and CI.", source_agent: "Codex", score: 5, status: "active" }, - { id: 12, decision: "Use browser bootstrap query params for smoke auth.", context: "Avoids cookie extraction and live daemon token reads.", source_agent: "Codex", score: 4, status: "active" }, - { id: 13, decision: "Keep CI smoke opt-in until an agent provider secret is configured.", context: "Hosted runners do not come with agent auth by default.", source_agent: "Claude", score: 4, status: "active", disputes_id: 14 }, - { id: 14, decision: "Require live daemon auth for browser smoke.", context: "Superseded by deterministic mock-backed auth.", source_agent: "Factory Droid", score: 1, status: "disputed" }, - ], - }, - peek: { - matches: [ - { source: "memory::browser_smoke_ci", relevance: 0.98, method: "keyword" }, - { source: "memory::analytics_projection", relevance: 0.93, method: "semantic" }, - { source: "decision::ci_opt_in", relevance: 0.89, method: "keyword" }, - ], - }, - recall: { - results: [ - { - source: "memory::browser_smoke_ci", - excerpt: "Expect smoke should use a mock daemon on port 7437 so browser fallback auth is deterministic in CI and source builds.", - relevance: 0.98, - method: "keyword", - }, - { - source: "memory::analytics_projection", - excerpt: "Monte Carlo projection labels were moved into a summary box to avoid overlap with the endpoint marker.", - relevance: 0.93, - method: "semantic", - }, - ], - }, - permissions: { - grants: [ - { - client: "codex", - permission: "admin", - scope: "*", - granted_by: "expect-smoke", - granted_at: isoMinutesAgo(30), - }, - { - client: "claude", - permission: "read", - scope: "cortex_recall", - granted_by: "expect-smoke", - granted_at: isoMinutesAgo(45), - }, - ], - }, - feedAcks: new Map(), - }; -} - -async function readJsonBody(request) { - const chunks = []; - for await (const chunk of request) { - chunks.push(chunk); - } - if (!chunks.length) return {}; - - try { - return JSON.parse(Buffer.concat(chunks).toString("utf8")); - } catch { - return {}; - } -} - -function buildFeedResponse(fixture, url) { - let entries = [...fixture.feed.entries].sort( - (left, right) => new Date(left.timestamp).getTime() - new Date(right.timestamp).getTime(), - ); - - const kind = url.searchParams.get("kind"); - if (kind && kind !== "all") { - entries = entries.filter((entry) => entry.kind === kind); - } - - const unreadOnly = url.searchParams.get("unread") === "true"; - const agent = (url.searchParams.get("agent") || "").trim(); - if (unreadOnly && agent) { - const lastSeenId = fixture.feedAcks.get(agent); - if (lastSeenId) { - const ackIndex = entries.findIndex((entry) => entry.id === lastSeenId); - if (ackIndex >= 0) { - entries = entries.slice(ackIndex + 1); - } - } - entries = entries.filter((entry) => entry.agent !== agent); - } - - return { entries }; -} - -function buildMessagesResponse(fixture, url) { - const agent = (url.searchParams.get("agent") || "").trim(); - const messages = agent - ? fixture.messages.messages.filter((entry) => entry.to === agent) - : fixture.messages.messages; - - return { messages }; -} - -function sendJson(response, statusCode, body) { - response.writeHead(statusCode, { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Headers": "Authorization, Content-Type, X-Cortex-Request", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Cache-Control": "no-store", - "Content-Type": "application/json; charset=utf-8", - }); - response.end(JSON.stringify(body)); -} - -function isAuthorized(request, token) { - return request.headers.authorization === `Bearer ${token}`; -} - -export async function startMockCortexServer({ host = "127.0.0.1", port = 7437, token = DEFAULT_TOKEN } = {}) { - const fixture = buildFixture(); - const protectedPrefixes = [ - "/sessions", - "/locks", - "/tasks", - "/feed", - "/messages", - "/activity", - "/savings", - "/conflicts", - "/permissions", - "/dump", - "/peek", - "/recall", - "/resolve", - ]; - - const server = http.createServer(async (request, response) => { - const url = new URL(request.url || "/", `http://${request.headers.host || `${host}:${port}`}`); - - if (request.method === "OPTIONS") { - response.writeHead(204, { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Headers": "Authorization, Content-Type, X-Cortex-Request", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - }); - response.end(); - return; - } - - if (protectedPrefixes.some((prefix) => url.pathname.startsWith(prefix)) && !isAuthorized(request, token)) { - sendJson(response, 401, { error: "Unauthorized" }); - return; - } - - if (request.method === "GET" && url.pathname === "/health") { - sendJson(response, 200, fixture.health); - return; - } - if (request.method === "GET" && url.pathname === "/events/stream") { - const streamToken = url.searchParams.get("token") || ""; - if (!isAuthorized(request, token) && streamToken !== token) { - sendJson(response, 401, { error: "Unauthorized" }); - return; - } - response.writeHead(200, { - "Access-Control-Allow-Origin": "*", - "Cache-Control": "no-store", - Connection: "keep-alive", - "Content-Type": "text/event-stream; charset=utf-8", - }); - response.write("event: connected\n"); - response.write('data: {"source":"expect-smoke"}\n\n'); - const heartbeat = setInterval(() => { - response.write("event: feed\n"); - response.write("data: {}\n\n"); - }, 20_000); - request.on("close", () => { - clearInterval(heartbeat); - }); - return; - } - if (request.method === "GET" && url.pathname === "/sessions") { - sendJson(response, 200, fixture.sessions); - return; - } - if (request.method === "GET" && url.pathname === "/locks") { - sendJson(response, 200, fixture.locks); - return; - } - if (request.method === "GET" && url.pathname === "/tasks") { - sendJson(response, 200, fixture.tasks); - return; - } - if (request.method === "GET" && url.pathname === "/feed") { - sendJson(response, 200, buildFeedResponse(fixture, url)); - return; - } - if (request.method === "GET" && url.pathname === "/messages") { - sendJson(response, 200, buildMessagesResponse(fixture, url)); - return; - } - if (request.method === "GET" && url.pathname === "/activity") { - sendJson(response, 200, fixture.activity); - return; - } - if (request.method === "GET" && url.pathname === "/savings") { - sendJson(response, 200, fixture.savings); - return; - } - if (request.method === "GET" && url.pathname === "/conflicts") { - sendJson(response, 200, fixture.conflicts); - return; - } - if (request.method === "GET" && url.pathname === "/permissions") { - sendJson(response, 200, fixture.permissions); - return; - } - if (request.method === "GET" && url.pathname === "/dump") { - sendJson(response, 200, fixture.dump); - return; - } - if (request.method === "GET" && url.pathname === "/peek") { - sendJson(response, 200, fixture.peek); - return; - } - if (request.method === "GET" && url.pathname === "/recall") { - sendJson(response, 200, fixture.recall); - return; - } - if (request.method === "POST" && url.pathname === "/resolve") { - sendJson(response, 200, { ok: true, action: "resolved" }); - return; - } - if (request.method === "POST" && url.pathname === "/permissions/grant") { - const body = await readJsonBody(request); - const grant = { - client: String(body.client || "unknown").trim() || "unknown", - permission: String(body.permission || "read").trim() || "read", - scope: String(body.scope || "*").trim() || "*", - granted_by: String(body.granted_by || "expect-smoke").trim() || "expect-smoke", - granted_at: nowIso(), - }; - fixture.permissions.grants = fixture.permissions.grants.filter( - (entry) => !(entry.client === grant.client && entry.permission === grant.permission && entry.scope === grant.scope), - ); - fixture.permissions.grants.push(grant); - sendJson(response, 200, { ok: true, grant }); - return; - } - if (request.method === "POST" && url.pathname === "/permissions/revoke") { - const body = await readJsonBody(request); - const client = String(body.client || "").trim(); - const permission = String(body.permission || "").trim(); - const scope = String(body.scope || "*").trim() || "*"; - fixture.permissions.grants = fixture.permissions.grants.filter( - (entry) => !(entry.client === client && entry.permission === permission && entry.scope === scope), - ); - sendJson(response, 200, { ok: true, revoked: true }); - return; - } - if (request.method === "POST" && url.pathname === "/tasks/claim") { - const body = await readJsonBody(request); - const task = fixture.tasks.tasks.find((entry) => entry.taskId === body.taskId); - if (!task) { - sendJson(response, 404, { error: "task_not_found" }); - return; - } - if (task.status === "claimed") { - sendJson(response, 409, { error: "task_already_claimed", claimedBy: task.claimedBy }); - return; - } - if (task.status === "completed" || task.status === "done") { - sendJson(response, 409, { error: "task_already_completed" }); - return; - } - task.status = "claimed"; - task.claimedBy = body.agent; - task.claimedAt = nowIso(); - sendJson(response, 200, { claimed: true, taskId: task.taskId }); - return; - } - if (request.method === "POST" && url.pathname === "/tasks/complete") { - const body = await readJsonBody(request); - const task = fixture.tasks.tasks.find((entry) => entry.taskId === body.taskId); - if (!task) { - sendJson(response, 404, { error: "task_not_found" }); - return; - } - if (task.claimedBy !== body.agent) { - sendJson(response, 403, { error: "not_task_holder", claimedBy: task.claimedBy || null }); - return; - } - task.status = "completed"; - task.completedAt = nowIso(); - task.summary = body.summary || task.summary || ""; - fixture.feed.entries.push({ - id: `feed-${fixture.feed.entries.length + 1}`, - kind: "task_complete", - agent: body.agent, - timestamp: task.completedAt, - summary: `Completed: ${task.title}`, - taskId: task.taskId, - files: Array.isArray(task.files) ? task.files : [], - }); - sendJson(response, 200, { completed: true, taskId: task.taskId }); - return; - } - if (request.method === "POST" && url.pathname === "/tasks/abandon") { - const body = await readJsonBody(request); - const task = fixture.tasks.tasks.find((entry) => entry.taskId === body.taskId); - if (!task) { - sendJson(response, 404, { error: "task_not_found" }); - return; - } - if (task.claimedBy !== body.agent) { - sendJson(response, 403, { error: "not_task_holder", claimedBy: task.claimedBy || null }); - return; - } - task.status = "pending"; - delete task.claimedBy; - delete task.claimedAt; - sendJson(response, 200, { abandoned: true, taskId: task.taskId, status: "pending" }); - return; - } - if (request.method === "POST" && url.pathname === "/tasks/delete") { - const body = await readJsonBody(request); - fixture.tasks.tasks = fixture.tasks.tasks.filter((entry) => entry.taskId !== body.taskId); - sendJson(response, 200, { ok: true, deleted: true, taskId: body.taskId || null }); - return; - } - if (request.method === "POST" && url.pathname === "/message") { - const body = await readJsonBody(request); - fixture.messages.messages.push({ - id: `msg-${fixture.messages.messages.length + 1}`, - from: body.from, - to: body.to, - timestamp: nowIso(), - message: body.message, - }); - sendJson(response, 200, { sent: true, messageId: `msg-${fixture.messages.messages.length}` }); - return; - } - if (request.method === "POST" && url.pathname === "/unlock") { - const body = await readJsonBody(request); - fixture.locks.locks = fixture.locks.locks.filter( - (entry) => !(entry.path === body.path && entry.agent === body.agent), - ); - sendJson(response, 200, { unlocked: true }); - return; - } - if (request.method === "POST" && url.pathname === "/feed/ack") { - const body = await readJsonBody(request); - fixture.feedAcks.set(body.agent, body.lastSeenId); - sendJson(response, 200, { acked: true }); - return; - } - - sendJson(response, 404, { error: `No mock handler for ${request.method} ${url.pathname}` }); - }); - - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(port, host, resolve); - }); - - return { - token, - baseUrl: `http://${host}:${port}`, - close: () => - new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }), - }; -} - -if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { - const host = process.env.EXPECT_SMOKE_HOST || "127.0.0.1"; - const port = Number(process.env.EXPECT_SMOKE_API_PORT || "7438"); - const server = await startMockCortexServer({ host, port, token: DEFAULT_TOKEN }); - console.log(`[expect-smoke] mock Cortex listening at ${server.baseUrl}`); -} diff --git a/desktop/cortex-control-center/scripts/rebuild-hooks.mjs b/desktop/cortex-control-center/scripts/rebuild-hooks.mjs deleted file mode 100644 index 175ab8b8..00000000 --- a/desktop/cortex-control-center/scripts/rebuild-hooks.mjs +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env node -import { execSync } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); -const bakLines = execSync("git show HEAD:desktop/cortex-control-center/src/App.jsx", { cwd: path.join(ROOT, "..") }) - .toString() - .split("\n"); - -const HOOK_HEADER = fs.readFileSync(path.join(SRC, "app/hooks/useDashboardState.js"), "utf8").split("\n").slice(0, 111).join("\n"); - -const FULL_RETURN = fs.readFileSync(path.join(ROOT, "scripts/split-panels-and-hooks.mjs"), "utf8") - .match(/const HOOK_RETURN_KEYS = `([\s\S]*?)`;/)[1]; - -const STATE_RETURN = `browserBootstrap, - isTauriRuntime, - panel, - setPanel, - brainPanelMounted, - setBrainPanelMounted, - panelMotionDirection, - setPanelMotionDirection, - daemonState, - setDaemonState, - healthMeta, - setHealthMeta, - stats, - setStats, - sessions, - setSessions, - tasks, - setTasks, - locks, - setLocks, - feedEntries, - setFeedEntries, - messageEntries, - setMessageEntries, - activityEntries, - setActivityEntries, - sidebarCollapsed, - setSidebarCollapsed, - isNarrowViewport, - setIsNarrowViewport, - savings, - setSavings, - memoryQuery, - setMemoryQuery, - memoryResults, - setMemoryResults, - memorySearching, - setMemorySearching, - feedFilters, - setFeedFilters, - selectedOperator, - setSelectedOperator, - messageTarget, - setMessageTarget, - messageDraft, - setMessageDraft, - taskCompletionDrafts, - setTaskCompletionDrafts, - completionTaskId, - setCompletionTaskId, - busyActionKey, - setBusyActionKey, - activitySince, - setActivitySince, - feedbackMessage, - setFeedbackMessage, - daemonTimeoutStaleSummary, - setDaemonTimeoutStaleSummary, - conflictPairs, - setConflictPairs, - resolveDrafts, - setResolveDrafts, - conflictLoading, - setConflictLoading, - permissionGrants, - setPermissionGrants, - permissionLoading, - setPermissionLoading, - permissionAccessDenied, - setPermissionAccessDenied, - permissionsEndpointAvailable, - setPermissionsEndpointAvailable, - permissionDraft, - setPermissionDraft, - editorSetup, - setEditorSetup, - editorDetections, - setEditorDetections, - selectedEditorIds, - setSelectedEditorIds, - cortexBase, - setCortexBase, - showConnectionDialog, - setShowConnectionDialog, - showEditorSetupWizard, - setShowEditorSetupWizard, - availableUpdate, - setAvailableUpdate, - updateInstalling, - setUpdateInstalling, - restartingDaemon, - setRestartingDaemon, - restartError, - setRestartError, - showMissionMetricLegend, - setShowMissionMetricLegend, - showMissionCompactUnits, - setShowMissionCompactUnits, - hasVisitedAnalytics, - setHasVisitedAnalytics, - analyticsReady, - setAnalyticsReady, - startupCoreReadyState, - setStartupCoreReadyState, - isSettingUpEditors, - setIsSettingUpEditors, - controlSettings, - setControlSettings, - budgetConfigStatus, - setBudgetConfigStatus, - budgetDraft, - setBudgetDraft, - budgetDraftDirty, - setBudgetDraftDirty, - budgetConfigBusy, - setBudgetConfigBusy, - budgetConfigMessage, - setBudgetConfigMessage, - ipcAvailable, - setIpcAvailable, - osReducedMotion, - setOsReducedMotion, - currency, - setCurrency, - analyticsMode, - setAnalyticsMode, - effectiveReducedMotion, - invokeRef, - tokenRef, - refreshAllRef, - refreshAllInFlightRef, - refreshAllQueuedRef, - daemonTransitionRef, - recoveryRetryTimerRef, - startupRetryStateRef, - startupCoreReadyRef, - lastCoreRefreshAtRef, - lastSecondaryRefreshAtRef, - startupSecondaryRefreshInFlightRef, - skipInitialFeedRefreshRef, - skipInitialMessagesRefreshRef, - skipInitialActivityRefreshRef, - connectionDialogRef, - connectionDialogTriggerRef, - editorSetupDialogRef, - editorSetupTriggerRef, - topbarRef, - analyticsPanelRef, - brainPanelRef, - analyticsTabRefs, - sessionsRef, - daemonStateRef, - streamConnectedAtRef, - streamDisconnectedAtRef, - streamSessionEventCountRef, - devVerificationStartedRef, - permissionsEndpointAvailableRef, - browserHealthProbeRef, - connectionDialogAutoPromptSuppressedRef, - budgetConfigLoadAttemptedRef, - restoreFocusToTrigger, - openConnectionDialog, - dismissConnectionDialog, - closeConnectionDialog, - closeEditorSetupWizard, - updateControlSetting, - changePanel, - normalizedSessions, - knownAgents, - editorSetupSummary, - editorDetectionSummary, - setupCommandPath, - manualMcpSnippet, - selectedOperatorName, - messageTargetName, - safeCurrency, - currencyRate, - activeBudgetStatus, - budgetSummary, - budgetDraftError, - budgetDraftEndpoints, - memoryLoad, - currencyFormatter, - formatCurrency, - savingsEstimateLegend, - formatMissionTokenValue, - clearTransientFeedback, - setSecondaryAvailabilityFeedback, - clearRecoveryRetry, - scheduleRecoveryRetry, - resetStartupRetryState, - scheduleStartupRecoveryRetry, - clearDisconnectedData`; - -function sliceByLine(startLine, endLine) { - return bakLines.slice(startLine - 1, endLine).join("\n"); -} - -function definedSymbols(block) { - const names = new Set(); - for (const match of block.matchAll(/^ (?:async function|function|const) ([A-Za-z_$][\w$]*)/gm)) { - names.add(match[1]); - } - return names; -} - -function destructureList(exclude = new Set()) { - return FULL_RETURN.split(",\n") - .map((line) => line.trim()) - .filter(Boolean) - .filter((name) => !exclude.has(name)) - .join(",\n "); -} - -function writeHook(file, fn, param, block, ret, { skipDestructure = false } = {}) { - const exclude = definedSymbols(block); - const destructure = skipDestructure || !param - ? "" - : ` const { - ${destructureList(exclude)}, - } = ${param}; - -`; - fs.writeFileSync( - path.join(SRC, `app/hooks/${file}`), - `${HOOK_HEADER} - -export function ${fn}(${param}) { -${destructure}${block} -${ret} -} -`, - ); -} - -writeHook("useDashboardState.js", "useDashboardState", "", sliceByLine(1459, 1906), ` return { - ${STATE_RETURN}, - };`, { skipDestructure: true }); - -writeHook("useRefreshOrchestration.js", "useRefreshOrchestration", "ctx", sliceByLine(1907, 2589), " return { ...ctx };"); -writeHook("useRefreshAll.js", "useRefreshAll", "ctx", sliceByLine(2590, 2852), " return { ...ctx };"); -writeHook("useDashboardEffects.js", "useDashboardEffects", "ctx", `${sliceByLine(2853, 3099)}\n${sliceByLine(3231, 3639)}`, " return ctx;"); -writeHook("useSseStream.js", "useSseStream", "ctx", sliceByLine(3100, 3229), " return ctx;"); -writeHook("useDaemonConnection.js", "useDaemonConnection", "ctx", sliceByLine(3640, 3773), " return ctx;"); -writeHook("useDashboardHandlers.js", "useDashboardHandlers", "ctx", sliceByLine(3775, 4292), ` return { - ...ctx, - ${FULL_RETURN}, - };`); - - -// useDashboardState written above with skipDestructure -fs.writeFileSync( - path.join(SRC, "app/hooks/useDashboardHooks.js"), - `import { useDashboardState } from "./useDashboardState.js"; -import { useRefreshOrchestration } from "./useRefreshOrchestration.js"; -import { useRefreshAll } from "./useRefreshAll.js"; -import { useDashboardEffects } from "./useDashboardEffects.js"; -import { useSseStream } from "./useSseStream.js"; -import { useDaemonConnection } from "./useDaemonConnection.js"; -import { useDashboardHandlers } from "./useDashboardHandlers.js"; - -export function useDashboardHooks() { - let ctx = useDashboardState(); - ctx = useRefreshOrchestration(ctx); - ctx = useRefreshAll(ctx); - ctx = useDashboardEffects(ctx); - ctx = useSseStream(ctx); - ctx = useDaemonConnection(ctx); - return useDashboardHandlers(ctx); -} -`, -); - -console.log("Rebuilt hooks from git source"); diff --git a/desktop/cortex-control-center/scripts/run-dev-lifecycle-verification.mjs b/desktop/cortex-control-center/scripts/run-dev-lifecycle-verification.mjs index 71e45f2d..ba530287 100644 --- a/desktop/cortex-control-center/scripts/run-dev-lifecycle-verification.mjs +++ b/desktop/cortex-control-center/scripts/run-dev-lifecycle-verification.mjs @@ -1,122 +1 @@ -import { spawn } from "node:child_process"; -import { once } from "node:events"; -import { access, readFile } from "node:fs/promises"; -import { constants as fsConstants } from "node:fs"; -import os from "node:os"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { cleanupDevRuntime } from "./cleanup-dev-runtime.mjs"; - -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const projectDir = resolve(scriptDir, ".."); -const npmCommand = process.platform === "win32" ? (process.env.ComSpec || "cmd.exe") : "npm"; -const reportPath = join( - os.tmpdir(), - `cortex-dev-restart-reconnect-${Date.now()}-${process.pid}.json`, -); -const timeoutMs = Number(process.env.CORTEX_DEV_VERIFY_TIMEOUT_MS || "240000"); - -function wait(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function waitForReportFile(path, child, timeout) { - const deadline = Date.now() + timeout; - while (Date.now() < deadline) { - if (child.exitCode !== null) { - throw new Error(`tauri dev exited before verification completed (code ${child.exitCode}).`); - } - try { - await access(path, fsConstants.F_OK); - return; - } catch { - await wait(500); - } - } - throw new Error(`Timed out waiting for verification report at ${path}.`); -} - -function spawnProcess(command, args, label, extraEnv = {}) { - const child = spawn(command, args, { - cwd: projectDir, - env: { - ...process.env, - ...extraEnv, - FORCE_COLOR: process.env.FORCE_COLOR || "1", - }, - stdio: "inherit", - windowsHide: true, - }); - - child.once("error", (error) => { - console.error(`[dev-verify] ${label} failed to start: ${error.message}`); - }); - - return child; -} - -async function stopChild(child, label) { - if (!child) return; - - if (process.platform === "win32" && child.pid) { - const killer = spawn("taskkill.exe", ["/PID", String(child.pid), "/T", "/F"], { - stdio: "ignore", - windowsHide: true, - }); - await once(killer, "exit").catch(() => {}); - } else if (!child.killed && child.exitCode === null) { - child.kill("SIGTERM"); - const exited = Promise.race([ - once(child, "exit"), - wait(10000).then(() => null), - ]); - const result = await exited; - if (result === null && child.exitCode === null) { - child.kill("SIGKILL"); - await once(child, "exit").catch(() => {}); - } - } - console.log(`[dev-verify] stopped ${label}`); -} - -function printSummary(report) { - console.log(`[dev-verify] report: ${report.reportPath || reportPath}`); - console.log(`[dev-verify] status: ${report.success ? "passed" : "failed"}`); - if (report.agent) { - console.log(`[dev-verify] agent: ${report.agent}`); - } - for (const step of Array.isArray(report.steps) ? report.steps : []) { - console.log(`[dev-verify] step: ${step.name}`); - } - if (report.error) { - console.error(`[dev-verify] error: ${report.error}`); - } -} - -const npmArgs = process.platform === "win32" - ? ["/d", "/s", "/c", "npm run dev"] - : ["run", "dev"]; - -cleanupDevRuntime(); - -const child = spawnProcess( - npmCommand, - npmArgs, - "tauri dev", - { - CORTEX_DEV_VERIFY_REPORT_PATH: reportPath, - VITE_CORTEX_DEV_VERIFY_RESTART: "1", - }, -); - -try { - await waitForReportFile(reportPath, child, timeoutMs); - const report = JSON.parse(await readFile(reportPath, "utf8")); - printSummary(report); - if (!report.success) { - throw new Error(report.error || "Restart/reconnect verification failed."); - } -} finally { - await stopChild(child, "tauri dev"); - cleanupDevRuntime({ quiet: true }); -} +import{spawn}from"node:child_process";import{once}from"node:events";import{access,readFile}from"node:fs/promises";import{constants as fsConstants}from"node:fs";import os from"node:os";import{dirname,join,resolve}from"node:path";import{fileURLToPath}from"node:url";import{cleanupDevRuntime}from"./cleanup-dev-runtime.mjs";const scriptDir=dirname(fileURLToPath(import.meta.url)),projectDir=resolve(scriptDir,".."),npmCommand=process.platform==="win32"?process.env.ComSpec||"cmd.exe":"npm",reportPath=join(os.tmpdir(),`cortex-dev-restart-reconnect-${Date.now()}-${process.pid}.json`),timeoutMs=Number(process.env.CORTEX_DEV_VERIFY_TIMEOUT_MS||"240000");function wait(ms){return new Promise(resolve2=>setTimeout(resolve2,ms))}async function waitForReportFile(path,child2,timeout){const deadline=Date.now()+timeout;for(;Date.now(){console.error(`[dev-verify] ${label} failed to start: ${error.message}`)}),child2}async function stopChild(child2,label){if(child2){if(process.platform==="win32"&&child2.pid){const killer=spawn("taskkill.exe",["/PID",String(child2.pid),"/T","/F"],{stdio:"ignore",windowsHide:!0});await once(killer,"exit").catch(()=>{})}else!child2.killed&&child2.exitCode===null&&(child2.kill("SIGTERM"),await Promise.race([once(child2,"exit"),wait(1e4).then(()=>null)])===null&&child2.exitCode===null&&(child2.kill("SIGKILL"),await once(child2,"exit").catch(()=>{})));console.log(`[dev-verify] stopped ${label}`)}}function printSummary(report){console.log(`[dev-verify] report: ${report.reportPath||reportPath}`),console.log(`[dev-verify] status: ${report.success?"passed":"failed"}`),report.agent&&console.log(`[dev-verify] agent: ${report.agent}`);for(const step of Array.isArray(report.steps)?report.steps:[])console.log(`[dev-verify] step: ${step.name}`);report.error&&console.error(`[dev-verify] error: ${report.error}`)}const npmArgs=process.platform==="win32"?["/d","/s","/c","npm run dev"]:["run","dev"];cleanupDevRuntime();const child=spawnProcess(npmCommand,npmArgs,"tauri dev",{CORTEX_DEV_VERIFY_REPORT_PATH:reportPath,VITE_CORTEX_DEV_VERIFY_RESTART:"1"});try{await waitForReportFile(reportPath,child,timeoutMs);const report=JSON.parse(await readFile(reportPath,"utf8"));if(printSummary(report),!report.success)throw new Error(report.error||"Restart/reconnect verification failed.")}finally{await stopChild(child,"tauri dev"),cleanupDevRuntime({quiet:!0})} diff --git a/desktop/cortex-control-center/scripts/run-tauri-build.mjs b/desktop/cortex-control-center/scripts/run-tauri-build.mjs index 102c21f1..7b739c50 100644 --- a/desktop/cortex-control-center/scripts/run-tauri-build.mjs +++ b/desktop/cortex-control-center/scripts/run-tauri-build.mjs @@ -1,153 +1 @@ -import { spawn } from "node:child_process"; -import { constants as fsConstants } from "node:fs"; -import { access, readFile } from "node:fs/promises"; -import { dirname, isAbsolute, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const projectDir = resolve(scriptDir, ".."); - -const defaultKeyFile = resolve(projectDir, ".secrets", "tauri", "updater-private.key"); -const defaultKeyPasswordFile = resolve(projectDir, ".secrets", "tauri", "updater-private.key.password"); - -const keyPathEnvNames = ["TAURI_SIGNING_PRIVATE_KEY_FILE", "TAURI_SIGNING_PRIVATE_KEY_PATH"]; -const passwordPathEnvNames = [ - "TAURI_SIGNING_PRIVATE_KEY_PASSWORD_FILE", - "TAURI_SIGNING_PRIVATE_KEY_PASS_FILE", -]; - -function resolvePath(input) { - if (!input || !input.trim()) return ""; - const value = input.trim(); - return isAbsolute(value) ? value : resolve(projectDir, value); -} - -async function canRead(path) { - if (!path) return false; - try { - await access(path, fsConstants.R_OK); - return true; - } catch { - return false; - } -} - -async function resolveTauriCliPath() { - const candidates = [ - resolve(projectDir, "node_modules", "@tauri-apps", "cli", "tauri.js"), - resolve(projectDir, "node_modules", "@tauri-apps", "cli", "bin", "tauri.js"), - ]; - - for (const candidate of candidates) { - if (await canRead(candidate)) { - return candidate; - } - } - - return ""; -} - -async function loadKey() { - const inlineValue = process.env.TAURI_SIGNING_PRIVATE_KEY || ""; - if (inlineValue.trim()) { - const possiblePath = resolvePath(inlineValue); - if (await canRead(possiblePath)) { - const key = await readFile(possiblePath, "utf8"); - return { key, source: possiblePath }; - } - return { key: inlineValue, source: "TAURI_SIGNING_PRIVATE_KEY (inline value)" }; - } - - const candidates = [ - ...keyPathEnvNames.map((name) => ({ name, path: resolvePath(process.env[name] || "") })), - { name: "default", path: defaultKeyFile }, - ]; - - for (const candidate of candidates) { - if (await canRead(candidate.path)) { - const key = await readFile(candidate.path, "utf8"); - return { key, source: candidate.path }; - } - } - - return null; -} - -async function loadPassword() { - if (process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD) { - return process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD; - } - - const candidates = [ - ...passwordPathEnvNames.map((name) => resolvePath(process.env[name] || "")), - defaultKeyPasswordFile, - ]; - - for (const candidate of candidates) { - if (await canRead(candidate)) { - return (await readFile(candidate, "utf8")).trim(); - } - } - - return ""; -} - -function printMissingKeyError() { - console.error("[desktop:build] Missing Tauri updater signing key."); - console.error("[desktop:build] Checked:"); - console.error(" - TAURI_SIGNING_PRIVATE_KEY (inline value or file path)"); - console.error(" - TAURI_SIGNING_PRIVATE_KEY_FILE"); - console.error(" - TAURI_SIGNING_PRIVATE_KEY_PATH"); - console.error(` - ${defaultKeyFile}`); - console.error("[desktop:build] Configure one of those locations, then retry."); -} - -async function main() { - const tauriCli = await resolveTauriCliPath(); - if (!tauriCli) { - console.error("[desktop:build] Missing Tauri CLI. Run npm ci first."); - process.exit(1); - } - - const loadedKey = await loadKey(); - if (!loadedKey || !loadedKey.key.trim()) { - printMissingKeyError(); - process.exit(1); - } - - const signingPassword = await loadPassword(); - const args = [tauriCli, "build", ...process.argv.slice(2)]; - const env = { - ...process.env, - TAURI_SIGNING_PRIVATE_KEY: loadedKey.key, - TAURI_SIGNING_PRIVATE_KEY_PASSWORD: - signingPassword || process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD || "", - }; - - console.log(`[desktop:build] Using updater signing key from ${loadedKey.source}`); - - const child = spawn(process.execPath, args, { - cwd: projectDir, - env, - stdio: "inherit", - windowsHide: true, - }); - - child.once("error", (error) => { - console.error(`[desktop:build] Failed to start Tauri build: ${error.message}`); - process.exit(1); - }); - - child.once("exit", (code, signal) => { - if (signal) { - console.error(`[desktop:build] Tauri build terminated by signal ${signal}`); - process.exit(1); - } - process.exit(code ?? 1); - }); -} - -main().catch((error) => { - console.error(`[desktop:build] ${error.message}`); - process.exit(1); -}); +import{spawn}from"node:child_process";import{constants as fsConstants}from"node:fs";import{access,readFile}from"node:fs/promises";import{dirname,isAbsolute,resolve}from"node:path";import{fileURLToPath}from"node:url";const scriptDir=dirname(fileURLToPath(import.meta.url)),projectDir=resolve(scriptDir,".."),defaultKeyFile=resolve(projectDir,".secrets","tauri","updater-private.key"),defaultKeyPasswordFile=resolve(projectDir,".secrets","tauri","updater-private.key.password"),keyPathEnvNames=["TAURI_SIGNING_PRIVATE_KEY_FILE","TAURI_SIGNING_PRIVATE_KEY_PATH"],passwordPathEnvNames=["TAURI_SIGNING_PRIVATE_KEY_PASSWORD_FILE","TAURI_SIGNING_PRIVATE_KEY_PASS_FILE"];function resolvePath(input){if(!input||!input.trim())return"";const value=input.trim();return isAbsolute(value)?value:resolve(projectDir,value)}async function canRead(path){if(!path)return!1;try{return await access(path,fsConstants.R_OK),!0}catch{return!1}}async function resolveTauriCliPath(){const candidates=[resolve(projectDir,"node_modules","@tauri-apps","cli","tauri.js"),resolve(projectDir,"node_modules","@tauri-apps","cli","bin","tauri.js")];for(const candidate of candidates)if(await canRead(candidate))return candidate;return""}async function loadKey(){const inlineValue=process.env.TAURI_SIGNING_PRIVATE_KEY||"";if(inlineValue.trim()){const possiblePath=resolvePath(inlineValue);return await canRead(possiblePath)?{key:await readFile(possiblePath,"utf8"),source:possiblePath}:{key:inlineValue,source:"TAURI_SIGNING_PRIVATE_KEY (inline value)"}}const candidates=[...keyPathEnvNames.map(name=>({name,path:resolvePath(process.env[name]||"")})),{name:"default",path:defaultKeyFile}];for(const candidate of candidates)if(await canRead(candidate.path))return{key:await readFile(candidate.path,"utf8"),source:candidate.path};return null}async function loadPassword(){if(process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD)return process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD;const candidates=[...passwordPathEnvNames.map(name=>resolvePath(process.env[name]||"")),defaultKeyPasswordFile];for(const candidate of candidates)if(await canRead(candidate))return(await readFile(candidate,"utf8")).trim();return""}function printMissingKeyError(){console.error("[desktop:build] Missing Tauri updater signing key."),console.error("[desktop:build] Checked:"),console.error(" - TAURI_SIGNING_PRIVATE_KEY (inline value or file path)"),console.error(" - TAURI_SIGNING_PRIVATE_KEY_FILE"),console.error(" - TAURI_SIGNING_PRIVATE_KEY_PATH"),console.error(` - ${defaultKeyFile}`),console.error("[desktop:build] Configure one of those locations, then retry.")}async function main(){const tauriCli=await resolveTauriCliPath();tauriCli||(console.error("[desktop:build] Missing Tauri CLI. Run npm ci first."),process.exit(1));const loadedKey=await loadKey();(!loadedKey||!loadedKey.key.trim())&&(printMissingKeyError(),process.exit(1));const signingPassword=await loadPassword(),args=[tauriCli,"build",...process.argv.slice(2)],env={...process.env,TAURI_SIGNING_PRIVATE_KEY:loadedKey.key,TAURI_SIGNING_PRIVATE_KEY_PASSWORD:signingPassword||process.env.TAURI_SIGNING_PRIVATE_KEY_PASSWORD||""};console.log(`[desktop:build] Using updater signing key from ${loadedKey.source}`);const child=spawn(process.execPath,args,{cwd:projectDir,env,stdio:"inherit",windowsHide:!0});child.once("error",error=>{console.error(`[desktop:build] Failed to start Tauri build: ${error.message}`),process.exit(1)}),child.once("exit",(code,signal)=>{signal&&(console.error(`[desktop:build] Tauri build terminated by signal ${signal}`),process.exit(1)),process.exit(code??1)})}main().catch(error=>{console.error(`[desktop:build] ${error.message}`),process.exit(1)}); diff --git a/desktop/cortex-control-center/scripts/split-hooks.mjs b/desktop/cortex-control-center/scripts/split-hooks.mjs deleted file mode 100644 index b3cb5af5..00000000 --- a/desktop/cortex-control-center/scripts/split-hooks.mjs +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env node -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); -const hookPath = path.join(SRC, "app/hooks/useDashboardHooks.js"); -const lines = fs.readFileSync(hookPath, "utf8").split("\n"); - -const imports = lines.slice(0, 111).join("\n"); -const body = lines.slice(112, -1); // drop closing } - -const HOOK_RETURN = fs.readFileSync(path.join(ROOT, "scripts/split-panels-and-hooks.mjs"), "utf8") - .match(/const HOOK_RETURN_KEYS = `([\s\S]*?)`;/)[1]; - -function sliceBody(start, end) { - return body.slice(start, end).join("\n"); -} - -const stateBody = sliceBody(0, 805 - 113); -const refreshBody = sliceBody(805 - 113, 1506 - 113); -const effectsBody = sliceBody(1508 - 113, 2530 - 113); -const handlersBody = sliceBody(2531 - 113, 2947 - 113); - -function writeHook(name, fnName, param, extraImport, block, ret) { - const content = `${imports} -${extraImport} -export function ${fnName}(${param}) { -${block} -${ret} -} -`; - fs.writeFileSync(path.join(SRC, `app/hooks/${name}`), content); -} - -writeHook( - "useDashboardState.js", - "useDashboardState", - "", - "", - stateBody, - ` return { - ${HOOK_RETURN}, - };`, -); - -writeHook( - "useRefreshOrchestration.js", - "useRefreshOrchestration", - "ctx", - "", - refreshBody.replace(/^ /gm, " "), - ` return { - ...ctx, - ${HOOK_RETURN}, - };`, -); - -writeHook( - "useSseStream.js", - "useSseStream", - "ctx", - "", - sliceBody(1755 - 113, 1870 - 113), - " return ctx;", -); - -writeHook( - "useDashboardEffects.js", - "useDashboardEffects", - "ctx", - "", - effectsBody.replace(/ useEffect\(\(\) => \{\n let stream = null;[\s\S]*? \}, \[cortexBase, refreshAllRef\]\);\n\n/m, ""), - " return ctx;", -); - -writeHook( - "useDashboardHandlers.js", - "useDashboardHandlers", - "ctx", - "", - handlersBody, - ` return { - ...ctx, - ${HOOK_RETURN}, - };`, -); - -const composer = `import { useDashboardState } from "./useDashboardState.js"; -import { useRefreshOrchestration } from "./useRefreshOrchestration.js"; -import { useSseStream } from "./useSseStream.js"; -import { useDashboardEffects } from "./useDashboardEffects.js"; -import { useDashboardHandlers } from "./useDashboardHandlers.js"; - -export function useDashboardHooks() { - let ctx = useDashboardState(); - ctx = useRefreshOrchestration(ctx); - ctx = useSseStream(ctx); - ctx = useDashboardEffects(ctx); - return useDashboardHandlers(ctx); -} -`; - -fs.writeFileSync(hookPath, composer); -console.log("Split useDashboardHooks into sub-hooks"); diff --git a/desktop/cortex-control-center/scripts/split-panels-and-hooks.mjs b/desktop/cortex-control-center/scripts/split-panels-and-hooks.mjs deleted file mode 100644 index 39e97a39..00000000 --- a/desktop/cortex-control-center/scripts/split-panels-and-hooks.mjs +++ /dev/null @@ -1,620 +0,0 @@ -#!/usr/bin/env node -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); - -function read(rel) { - return fs.readFileSync(path.join(SRC, rel), "utf8"); -} - -function write(rel, content) { - const full = path.join(SRC, rel); - fs.mkdirSync(path.dirname(full), { recursive: true }); - fs.writeFileSync(full, content.endsWith("\n") ? content : `${content}\n`); -} - -const HOOK_RETURN_KEYS = `panel, - setPanel, - brainPanelMounted, - panelMotionDirection, - daemonState, - healthMeta, - stats, - sessions, - tasks, - locks, - feedEntries, - messageEntries, - activityEntries, - sidebarCollapsed, - setSidebarCollapsed, - isNarrowViewport, - savings, - memoryQuery, - setMemoryQuery, - memoryResults, - memorySearching, - feedFilters, - setFeedFilters, - selectedOperator, - setSelectedOperator, - messageTarget, - setMessageTarget, - messageDraft, - setMessageDraft, - taskCompletionDrafts, - setTaskCompletionDrafts, - completionTaskId, - setCompletionTaskId, - busyActionKey, - activitySince, - setActivitySince, - feedbackMessage, - daemonTimeoutStaleSummary, - conflictPairs, - resolveDrafts, - conflictLoading, - permissionGrants, - permissionLoading, - permissionAccessDenied, - permissionsEndpointAvailable, - permissionDraft, - setPermissionDraft, - editorSetup, - editorDetections, - selectedEditorIds, - cortexBase, - setCortexBase, - showConnectionDialog, - showEditorSetupWizard, - availableUpdate, - updateInstalling, - setUpdateInstalling, - setFeedbackMessage, - restartingDaemon, - restartError, - showMissionMetricLegend, - setShowMissionMetricLegend, - showMissionCompactUnits, - setShowMissionCompactUnits, - hasVisitedAnalytics, - analyticsReady, - isSettingUpEditors, - controlSettings, - budgetConfigStatus, - budgetDraft, - budgetDraftDirty, - budgetConfigBusy, - budgetConfigMessage, - ipcAvailable, - currency, - setCurrency, - analyticsMode, - setAnalyticsMode, - effectiveReducedMotion, - invokeRef, - refreshAllRef, - tokenRef, - connectionDialogRef, - connectionDialogTriggerRef, - editorSetupDialogRef, - editorSetupTriggerRef, - topbarRef, - analyticsPanelRef, - brainPanelRef, - analyticsTabRefs, - isTauriRuntime, - changePanel, - normalizedSessions, - knownAgents, - editorSetupSummary, - editorDetectionSummary, - manualMcpSnippet, - selectedOperatorName, - messageTargetName, - safeCurrency, - budgetSummary, - budgetDraftError, - budgetDraftEndpoints, - memoryLoad, - formatCurrency, - savingsEstimateLegend, - formatMissionTokenValue, - runRefreshAll, - openConnectionDialog, - dismissConnectionDialog, - closeConnectionDialog, - closeEditorSetupWizard, - updateControlSetting, - restoreFocusToTrigger, - toggleEditorSelection, - openEditorSetupWizard, - applyEditorSetup, - reloadBudgetConfigDraft, - saveBudgetConfigDraft, - updateBudgetDraftRoot, - updateBudgetEndpointDraft, - handleStartDaemon, - handleStopDaemon, - handleRestartDaemon, - handleTaskClaim, - handleTaskAbandon, - handleTaskComplete, - handleTaskDelete, - handleUnlock, - handleSendMessage, - handleFeedAck, - handleMemorySearch, - handleMemoryExpand, - handleResolveConflict, - handleResolveDraftChange, - handleGrantPermission, - handleRevokePermission, - refreshMessages, - refreshActivity, - refreshFeed, - refreshConflicts, - refreshPermissions, - refreshSavings, - reportSurfaceError, - readAuthToken, - api, - postApi, - call, - pill, - utilityPill, - sidebarUtilityStats, - daemonRecoveryHint, - daemonStatusBadge, - daemonSysStatus, - pendingTasks, - claimedTasks, - completedTasks, - monteCarloProjection, - bootSavingsMomentum, - latestRecallHitRate, - recallWindowAverage, - recallWindowSpread, - topActivityEntries, - topFeedEntries, - recentOverviewTasks, - firstRunReadiness, - handleFirstRunAction, - activePanelLabel, - connectionEndpoint, - hostLabel, - handleAnalyticsTabKey, - effectiveSidebarCollapsed, - canStartDaemon, - canStopDaemon, - canSetupEditors, - operationRows, - operationMaxSaved, - topSavingsByAgent`; - -const PANEL_IMPORTS = `import { AppIcon } from "../../ui-icons.jsx"; -import { CURRENCY_OPTIONS, SAVINGS_OPERATION_LABELS, SAVINGS_USD_PER_MILLION, SAVINGS_HISTORY_DAYS, timeAgo, MISSION_METRIC_LEGEND, CONTROL_CENTER_VERSION, ANALYTICS_METRIC_LEGEND } from "../../constants.js"; -import { BUDGET_ENDPOINT_DEFINITIONS } from "../../settings/settings-state.js"; -import { handleKeyboardActivation } from "../../keyboard-access.js"; -import { sameAgent } from "../../live-surface.js"; -import { normalizeCurrencyCode, formatDaemonEndpoint } from "../utils/format.js"; -import { conflictBadgeClass } from "../normalize/conflicts.js"; -import { agentColor } from "../utils/agent-color.js"; -import { AnimatedNumber } from "../components/AnimatedNumber.jsx"; -import { Sparkline } from "../components/Sparkline.jsx"; -import { MonteCarloProjectionChart } from "../components/MonteCarloProjectionChart.jsx"; -import { EmptyItem } from "../components/common.jsx"; -import { AgentItem } from "../components/AgentItem.jsx"; -import { OperatorSelector } from "../components/OperatorSelector.jsx"; -import { TaskItem } from "../components/TaskItem.jsx"; -import { LockItem } from "../components/LockItem.jsx"; -import { FeedItem } from "../components/FeedItem.jsx"; -import { MessageItem } from "../components/MessageItem.jsx"; -import { ActivityItem } from "../components/ActivityItem.jsx"; -import { ConflictPairCard } from "../components/ConflictPairCard.jsx"; -import { PANEL_SEQUENCE } from "../constants.js"; -import { BrainVisualizerPanel } from "../components/BrainVisualizerPanel.jsx"; -`; - -function makePanelWrapper(name, jsxBody) { - return `${PANEL_IMPORTS} -export function ${name}(p) { - const { - ${HOOK_RETURN_KEYS}, - } = p; - - return ( -${jsxBody} - ); -} -`; -} - -function extractPanelStageSection(content, startNeedle, endNeedle) { - const start = content.indexOf(startNeedle); - const end = content.indexOf(endNeedle, start); - if (start < 0 || end < 0) throw new Error(`Failed to extract ${startNeedle}`); - return content.slice(start, end).split("\n").map((line) => line.replace(/^ {12}/, " ")).join("\n"); -} - -function splitPanels() { - const stage = read("app/panels/panel-stage.jsx"); - const innerStart = stage.indexOf('
\n );"); - const inner = stage.slice(innerStart, innerEnd); - - write( - "app/panels/SettingsPanel.jsx", - makePanelWrapper( - "SettingsPanel", - extractPanelStageSection(inner, '\n\n {panel === \"overview\"") - .replace(/^ \{panel === "overview".*\n/m, ""), - ), - ); - - write( - "app/panels/OverviewPanel.jsx", - makePanelWrapper( - "OverviewPanel", - extractPanelStageSection(inner, '{panel === "overview" ? (', " ) : null}\n\n\n {panel === \"agents\"") - .replace(/^\s*\{panel === "overview" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/AgentsPanel.jsx", - makePanelWrapper( - "AgentsPanel", - extractPanelStageSection(inner, '{panel === "agents" ? (', " ) : null}\n\n {panel === \"work\"") - .replace(/^\s*\{panel === "agents" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/WorkPanel.jsx", - makePanelWrapper( - "WorkPanel", - extractPanelStageSection(inner, '{panel === "work" ? (', " ) : null}\n\n {panel === \"memory\"") - .replace(/^\s*\{panel === "work" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/MemoryPanel.jsx", - makePanelWrapper( - "MemoryPanel", - extractPanelStageSection(inner, '{panel === "memory" ? (', " ) : null}\n\n\n\n\n\n\n {panel === \"analytics\"") - .replace(/^\s*\{panel === "memory" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/AnalyticsPanel.jsx", - makePanelWrapper( - "AnalyticsPanel", - extractPanelStageSection(inner, '{panel === "analytics" || hasVisitedAnalytics ? (', " ) : null}\n\n\n {brainPanelMounted") - .replace(/^\s*\{panel === "analytics" \|\| hasVisitedAnalytics \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/ConflictsPanel.jsx", - makePanelWrapper( - "ConflictsPanel", - extractPanelStageSection(inner, '{panel === "conflicts" ? (', " ) : null}\n\n {panel === \"about\"") - .replace(/^\s*\{panel === "conflicts" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/AboutPanel.jsx", - makePanelWrapper( - "AboutPanel", - extractPanelStageSection(inner, '{panel === "about" ? (', " ) : null}\n\n
") - .replace(/^\s*\{panel === "about" \? \(\n/, "") - .replace(/\n\s*\) : null\}\s*$/, ""), - ), - ); - - write( - "app/panels/panel-stage.jsx", - `${PANEL_IMPORTS} -import { SettingsPanel } from "./SettingsPanel.jsx"; -import { OverviewPanel } from "./OverviewPanel.jsx"; -import { AgentsPanel } from "./AgentsPanel.jsx"; -import { WorkPanel } from "./WorkPanel.jsx"; -import { MemoryPanel } from "./MemoryPanel.jsx"; -import { AnalyticsPanel } from "./AnalyticsPanel.jsx"; -import { ConflictsPanel } from "./ConflictsPanel.jsx"; -import { AboutPanel } from "./AboutPanel.jsx"; - -export function PanelStage(p) { - return ( -
- - - - - - - - - -
- ); -} -`, - ); -} - -function splitHook() { - const hookLines = read("app/hooks/useDashboardHooks.js").split("\n"); - const imports = hookLines.slice(0, 111).join("\n"); - const bodyLines = hookLines.slice(112, -2); // exclude closing brace and empty - - const splitAt = (predicate) => { - const idx = bodyLines.findIndex(predicate); - if (idx < 0) throw new Error("split point not found"); - return idx; - }; - - const idxEffects = splitAt((line) => line.trim().startsWith("useEffect(() => {") && line.includes("localStorage.setItem(CORTEX_BASE_STORAGE_KEY")); - const idxHandlers = splitAt((line) => line.trim() === "async function handleMemorySearch(event) {"); - - const stateBlock = bodyLines.slice(0, idxEffects).join("\n"); - const effectsBlock = bodyLines.slice(idxEffects, idxHandlers).join("\n"); - const handlersBlock = bodyLines.slice(idxHandlers).join("\n"); - - const sharedImports = imports.replace("export function useDashboardHooks() {", "").trim(); - - write( - "app/hooks/useDashboardState.js", - `${sharedImports} - -export function useDashboardState() { -${stateBlock} - return { - ${HOOK_RETURN_KEYS}, - }; -} -`, - ); - - write( - "app/hooks/useDashboardEffects.js", - `${sharedImports} -import { useDashboardState } from "./useDashboardState.js"; - -export function useDashboardEffects(state) { -${effectsBlock} - return state; -} -`, - ); - - // handlers block includes pre-return computed values - keep in orchestrator file - write( - "app/hooks/useDashboardHandlers.js", - `${sharedImports} - -export function useDashboardHandlers(state) { -${handlersBlock} - return { - ${HOOK_RETURN_KEYS}, - }; -} -`, - ); - - write( - "app/hooks/useDashboardHooks.js", - `import { useDashboardState } from "./useDashboardState.js"; -import { useDashboardEffects } from "./useDashboardEffects.js"; -import { useDashboardHandlers } from "./useDashboardHandlers.js"; - -export function useDashboardHooks() { - const state = useDashboardState(); - useDashboardEffects(state); - return useDashboardHandlers(state); -} -`, - ); -} - -function splitOverridesCss() { - const css = read("styles/overrides-2026.css").split("\n"); - const mid = Math.floor(css.length / 2); - write("styles/overrides-2026-a.css", css.slice(0, mid).join("\n")); - write("styles/overrides-2026-b.css", css.slice(mid).join("\n")); - fs.unlinkSync(path.join(SRC, "styles/overrides-2026.css")); - - const index = read("styles/index.css") - .replace('@import "./overrides-2026.css";', '@import "./overrides-2026-a.css";\n@import "./overrides-2026-b.css";'); - write("styles/index.css", index); -} - -function writeCssTestHelper() { - write( - "test/read-styles.js", - `import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const SRC = path.dirname(fileURLToPath(import.meta.url)); - -function walkCss(dir) { - const entries = fs.readdirSync(dir, { withFileTypes: true }); - let files = []; - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) files = files.concat(walkCss(full)); - else if (entry.name.endsWith(".css")) files.push(full); - } - return files.sort(); -} - -export function readBundledStyles() { - const stylesDir = path.join(SRC, "styles"); - return walkCss(stylesDir).map((file) => fs.readFileSync(file, "utf8")).join("\\n"); -} -`, - ); - - write("styles.css", '@import "./styles/index.css";\n'); -} - -function updateCssTests() { - for (const rel of [ - "brain-visualizer.test.js", - "contrast-tokens.test.js", - "reflow-layout.test.js", - "sidebar-collapse.test.js", - "panel-transition.test.js", - "design/motion.test.js", - ]) { - let content = read(rel); - if (content.includes("readBundledStyles")) continue; - content = content.replace( - 'import { readFileSync } from "node:fs";\n', - 'import { readFileSync } from "node:fs";\nimport { readBundledStyles } from "./test/read-styles.js";\n', - ); - content = content.replace( - /const css = readFileSync\(new URL\("\.\.?\/styles\.css", import\.meta\.url\), "utf8"\);/, - "const css = readBundledStyles();", - ); - write(rel, content); - } -} - -function updatePanelNavigationTest() { - write( - "panel-navigation.test.js", - `import { readFileSync, readdirSync } from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; - -const SRC_DIR = path.dirname(fileURLToPath(import.meta.url)); - -function listAppSources(dir = path.join(SRC_DIR, "app")) { - const entries = readdirSync(dir, { withFileTypes: true }); - const files = []; - for (const entry of entries) { - const absolutePath = path.join(dir, entry.name); - if (entry.isDirectory()) { - files.push(...listAppSources(absolutePath)); - continue; - } - if (/\\.(js|jsx)$/.test(entry.name)) files.push(absolutePath); - } - return files; -} - -const appSource = listAppSources().map((file) => readFileSync(file, "utf8")).join("\\n"); - -function readBlock(source, needle) { - const start = source.indexOf(needle); - expect(start, \`missing source block \${needle}\`).toBeGreaterThanOrEqual(0); - - const bodyStart = source.indexOf("{", start); - expect(bodyStart, \`missing source body for \${needle}\`).toBeGreaterThanOrEqual(0); - - let depth = 1; - for (let index = bodyStart + 1; index < source.length; index += 1) { - if (source[index] === "{") { - depth += 1; - } else if (source[index] === "}") { - depth -= 1; - } - - if (depth === 0) { - return source.slice(bodyStart + 1, index); - } - } - - throw new Error(\`unterminated source block \${needle}\`); -} - -describe("panel navigation scheduling", () => { - it("updates the active panel urgently after recording motion direction", () => { - const changePanel = readBlock(appSource, "const changePanel = useCallback"); - - expect(changePanel).toContain("setPanelMotionDirection("); - expect(changePanel).toContain("setPanel(nextPanel);"); - expect(changePanel).not.toContain("startTransition(() => setPanel(nextPanel))"); - }); - - it("keeps the settings panel mounted while inactive", () => { - expect(appSource).toContain( - 'className={\`panel settings-panel \${panel === "settings" ? "active" : "panel-hidden"}\`}', - ); - expect(appSource).toContain('aria-hidden={panel === "settings" ? undefined : true}'); - }); - - it("exposes a keyboard skip link to the main content landmark", () => { - const skipLinkIndex = appSource.indexOf(''); - const sidebarIndex = appSource.indexOf("'); - - expect(skipLinkIndex, "missing skip link").toBeGreaterThanOrEqual(0); - expect(mainIndex, "missing skip target main landmark").toBeGreaterThanOrEqual(0); - expect(skipLinkIndex, "skip link should be the first focusable shell control").toBeLessThan(sidebarIndex); - }); - - it("gives placeholder-only task and permission controls accessible names", () => { - expect(appSource).toContain('aria-label={\`Completion summary for \${task.title}\`}'); - expect(appSource).toContain('aria-label="Client id for permission grant"'); - expect(appSource).toContain(': "Operator message body"'); - }); - - it("announces budget validation and load errors as alerts", () => { - expect(appSource).toContain( - '{budgetSummary.error ?

{budgetSummary.error}

: null}', - ); - expect(appSource).toContain( - '{budgetDraftError ?

{budgetDraftError}

: null}', - ); - }); - - it("does not load desktop budget state during the settings panel entry animation", () => { - expect(appSource).toContain("const budgetReloadTimer = window.setTimeout(() => {"); - expect(appSource).toContain("}, effectiveReducedMotion ? 0 : MOTION_MS.panel);"); - expect(appSource).toContain("window.clearTimeout(budgetReloadTimer);"); - }); -}); -`, - ); -} - -function updateMainCssImport() { - let main = read("main.jsx"); - main = main.replace('import "./styles.css";', 'import "./styles/index.css";'); - write("main.jsx", main); -} - -splitPanels(); -// splitHook(); // too risky without validation - do manual split below -splitOverridesCss(); -writeCssTestHelper(); -updateCssTests(); -updatePanelNavigationTest(); -updateMainCssImport(); - -console.log("Split panels and updated tests"); diff --git a/desktop/cortex-control-center/scripts/split-refactor.mjs b/desktop/cortex-control-center/scripts/split-refactor.mjs deleted file mode 100644 index 0e64648b..00000000 --- a/desktop/cortex-control-center/scripts/split-refactor.mjs +++ /dev/null @@ -1,963 +0,0 @@ -#!/usr/bin/env node -/** - * One-time refactor script: splits App.jsx and styles.css into modules. - */ -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SRC = path.join(ROOT, "src"); -const APP = path.join(SRC, "app"); -const STYLES = path.join(SRC, "styles"); - -function readLines(file) { - return fs.readFileSync(file, "utf8").split("\n"); -} - -function sliceLines(lines, start, end) { - return lines.slice(start - 1, end).join("\n"); -} - -function writeFile(relPath, content) { - const full = path.join(SRC, relPath); - fs.mkdirSync(path.dirname(full), { recursive: true }); - fs.writeFileSync(full, content.endsWith("\n") ? content : `${content}\n`); -} - -function splitApp() { - const lines = readLines(path.join(SRC, "App.jsx")); - const total = lines.length; - - // --- Extracted module bodies (1-indexed line ranges from original App.jsx) --- - const extractions = { - "app/constants.js": { start: 74, end: 158, header: `// Matches daemon-rs/src/main.rs:DEFAULT_CORTEX_PORT. Bump both simultaneously.\n` }, - "app/browser-bootstrap.js": { start: 160, end: 302, imports: `import {\n CORTEX_AUTH_STORAGE_KEY,\n CORTEX_BASE_STORAGE_KEY,\n CORTEX_PANEL_STORAGE_KEY,\n DEFAULT_CORTEX_BASE,\n LEGACY_CORTEX_AUTH_STORAGE_KEYS,\n PANEL_SEQUENCE_KEYS,\n} from "./constants.js";\n\n` }, - "app/utils/format.js": { start: 304, end: 323, imports: `import { DEFAULT_CORTEX_PORT } from "../constants.js";\nimport { FEED_KIND_LABEL } from "../constants.js";\n\n`, exports: true }, - "app/components/AnimatedNumber.jsx": { start: 325, end: 363, imports: `import { useEffect, useRef, useState } from "react";\nimport { MOTION_MS, easeOutCubic } from "../../design/motion.js";\n\n`, exports: "AnimatedNumber" }, - "app/components/sparkline-utils.js": { start: 365, end: 387, imports: "", exports: true }, - "app/components/Sparkline.jsx": { start: 389, end: 435, imports: `import { useState } from "react";\nimport { buildLineGeometry } from "./sparkline-utils.js";\n\n`, exports: "Sparkline" }, - "app/components/MonteCarloProjectionChart.jsx": { start: 437, end: 526, imports: `import { formatSignedCompactNumber } from "../../number-format.js";\n\n`, exports: "MonteCarloProjectionChart" }, - "app/components/common.jsx": { start: 528, end: 545, imports: `import { AppIcon } from "../../ui-icons.jsx";\n\n`, exports: "ComingSoon, EmptyItem" }, - "app/utils/agent-color.js": { start: 547, end: 555, exports: true }, - "app/normalize/conflicts.js": { start: 557, end: 872, imports: `import { timeAgo } from "../../constants.js";\n\n`, exports: true }, - "app/normalize/permissions.js": { start: 874, end: 893, imports: `function pickDefined(...values) {\n for (const value of values) {\n if (value !== undefined && value !== null && value !== "") {\n return value;\n }\n }\n return null;\n}\n\n`, exports: true }, - "app/components/AgentItem.jsx": { start: 895, end: 919, imports: `import { timeAgo } from "../../constants.js";\nimport { agentColor } from "../utils/agent-color.js";\n\n`, exports: "AgentItem" }, - "app/components/OperatorSelector.jsx": { start: 921, end: 940, imports: `import { useId } from "react";\n\n`, exports: "OperatorSelector" }, - "app/components/TaskItem.jsx": { start: 942, end: 1065, imports: `import { canClaimTask, canFinalizeTask } from "../../live-surface.js";\nimport { timeAgo } from "../../constants.js";\n\n`, exports: "TaskItem" }, - "app/components/LockItem.jsx": { start: 1067, end: 1096, imports: `import { canUnlockLock } from "../../live-surface.js";\n\n`, exports: "LockItem" }, - "app/components/FeedItem.jsx": { start: 1098, end: 1124, imports: `import { timeAgo } from "../../constants.js";\nimport { feedKindLabel } from "../utils/format.js";\n\n`, exports: "FeedItem" }, - "app/components/MessageItem.jsx": { start: 1126, end: 1142, imports: `import { timeAgo } from "../../constants.js";\nimport { AppIcon } from "../../ui-icons.jsx";\nimport { agentColor } from "../utils/agent-color.js";\n\n`, exports: "MessageItem" }, - "app/components/ActivityItem.jsx": { start: 1144, end: 1165, imports: `import { timeAgo } from "../../constants.js";\n\n`, exports: "ActivityItem" }, - "app/components/ConflictPairCard.jsx": { start: 1167, end: 1343, imports: `import { timeAgo } from "../../constants.js";\nimport {\n conflictBadgeClass,\n formatConfidencePercent,\n formatTimestamp,\n formatTrustScore,\n} from "../normalize/conflicts.js";\nimport { agentColor } from "../utils/agent-color.js";\n\n`, exports: "ConflictPairCard" }, - "app/normalize/sessions.js": { start: 1345, end: 1381, imports: `import { sameAgent } from "../../live-surface.js";\n\n`, exports: true }, - "app/utils/daemon.js": { start: 1383, end: 1456, exports: true }, - }; - - for (const [relPath, spec] of Object.entries(extractions)) { - let body = sliceLines(lines, spec.start, spec.end); - if (spec.header) body = spec.header + body; - const imports = spec.imports || ""; - let exportSuffix = ""; - if (spec.exports === true) { - // export all top-level functions/consts - exportSuffix = "\n// auto-exported\n"; - } else if (typeof spec.exports === "string") { - exportSuffix = ""; - body = body.replace(/^function (\w+)/gm, "export function $1"); - } - // Prefix export on functions and const class in normalize/utils files - if (spec.exports === true) { - body = body - .replace(/^function /gm, "export function ") - .replace(/^const CONFLICT_/gm, "export const CONFLICT_") - .replace(/^const EMPTY_/gm, "export const EMPTY_"); - if (relPath.includes("constants.js")) { - body = body.replace(/^const /gm, "export const ").replace(/^function /gm, "export function "); - } - if (relPath.includes("browser-bootstrap.js")) { - body = body.replace(/^function /gm, "export function "); - } - if (relPath.includes("format.js")) { - body = body.replace(/^function /gm, "export function "); - } - if (relPath.includes("agent-color.js")) { - body = body.replace(/^function /gm, "export function "); - } - if (relPath.includes("permissions.js")) { - body = body.replace(/^function /gm, "export function "); - } - if (relPath.includes("sparkline-utils.js")) { - body = body.replace(/^let /gm, "export let ").replace(/^function /gm, "export function "); - } - } - writeFile(relPath, imports + body); - } - - // BrainErrorBoundary + LazyBrainVisualizer - writeFile( - "app/components/BrainVisualizerPanel.jsx", - `import { Component, lazy, Suspense } from "react"; -import { AppIcon } from "../../ui-icons.jsx"; - -const LazyBrainVisualizer = lazy(() => - import("../../BrainVisualizer.jsx").then((module) => ({ default: module.BrainVisualizer })), -); - -class BrainErrorBoundary extends Component { - constructor(props) { super(props); this.state = { crashed: false, error: "" }; } - static getDerivedStateFromError(err) { return { crashed: true, error: err?.message || "Unknown error" }; } - render() { - if (this.state.crashed) return ( -
-
-

Brain visualizer crashed: {this.state.error}

- -
- ); - return this.props.children; - } -} - -export function BrainVisualizerPanel({ brainPanelRef, panel, brainPanelMounted, api, cortexBase, authToken, effectiveReducedMotion }) { - if (!brainPanelMounted) return null; - return ( -
- - -
-

Loading brain visualizer…

- - )} - > - -
-
-
- ); -} -`, - ); - - // Hook body: lines 1459-4293 (inside App function, excluding export line and return) - const hookBody = sliceLines(lines, 1459, 4293); - - // Panel JSX blocks - const panelStageInner = sliceLines(lines, 4605, 6467); - - writeFile( - "app/hooks/useDashboardHooks.js", - `import { startTransition, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { checkForUpdates, installUpdate } from "../../updater.js"; -import { MOTION_MS } from "../../design/motion.js"; -import { - createApi, - createPostApi, - isAuthFailure, - settledCollectErrors, - summarizeDashboardErrors, -} from "../../api-client.js"; -import { - CURRENCY_OPTIONS, - USD_TO_CURRENCY_RATE, - SAVINGS_OPERATION_LABELS, - timeAgo, -} from "../../constants.js"; -import { - buildKnownAgents, - filterFeedEntries, - isTransportSession, - nextFeedAckId, - normalizeTask, - resolveAgentName, - sameAgent, -} from "../../live-surface.js"; -import { - buildFirstRunReadiness, - computeStartupRetryStep, - daemonStatusPill, - daemonSystemStatus, - daemonUtilityPill, - isDaemonStartingState, - shouldContinueStartupRecovery, - isTransientDaemonFeedback, -} from "../../daemon-startup.js"; -import { buildMonteCarloProjection } from "../../analytics-projection.js"; -import { summarizeBootThroughput } from "../../analytics-metrics.js"; -import { formatCompactNumber, formatSignedCompactNumber } from "../../number-format.js"; -import { handleKeyboardActivation, shouldIgnoreGlobalShortcut, trapFocusInContainer } from "../../keyboard-access.js"; -import { - BUDGET_ENDPOINT_DEFINITIONS, - createBudgetDraftFromStatus, - readControlCenterSettings, - resolveEffectiveReducedMotion, - serializeBudgetDraftForSave, - summarizeBudgetStatus, - validateBudgetDraft, - writeControlCenterSettings, -} from "../../settings/settings-state.js"; -import { - ANALYTICS_METRIC_LEGEND, - ANALYTICS_REFRESH_MS, - CONTROL_CENTER_VERSION, - CORE_REFRESH_MIN_INTERVAL_MS, - CORTEX_BASE_STORAGE_KEY, - CORTEX_OPERATOR_STORAGE_KEY, - CORTEX_PANEL_STORAGE_KEY, - DAEMON_START_POLL_INTERVAL_MS, - DAEMON_START_STILL_STARTING_GRACE_MS, - DAEMON_START_WAIT_TIMEOUT_MS, - DAEMON_STOP_HANG_TIMEOUT_MS, - DAEMON_STOP_WAIT_TIMEOUT_MS, - DEFAULT_CORTEX_BASE, - DEV_RESTART_VERIFY_ENABLED, - DEV_RESTART_VERIFY_TIMEOUT_MS, - EMPTY_DAEMON, - EMPTY_HEALTH_META, - FALLBACK_REFRESH_MS, - MISSION_METRIC_LEGEND, - PANEL_SEQUENCE, - PANEL_SEQUENCE_KEYS, - PANEL_SEQUENCE_LABEL, - RECALL_HEADLINE_MIN_QUERIES, - SAVINGS_HISTORY_DAYS, - SAVINGS_USD_PER_MILLION, - SECONDARY_REFRESH_MIN_INTERVAL_MS, - SIDEBAR_COLLAPSE_BREAKPOINT_PX, - SSE_RECONNECT_BASE_MS, - SSE_RECONNECT_MAX_MS, - SSE_REFRESH_THROTTLE_MS, - panelIndex, -} from "../constants.js"; -import { - readBrowserBootstrap, - readLocalStorageValue, - readPersistedBrowserAuthToken, - readTauriInvoke, - persistBrowserAuthToken, -} from "../browser-bootstrap.js"; -import { normalizeCurrencyCode, formatDaemonEndpoint, getOsReducedMotionPreference } from "../utils/format.js"; -import { - normalizeConflictPairsPayload, -} from "../normalize/conflicts.js"; -import { - normalizePermissionPayload, -} from "../normalize/permissions.js"; -import { - normalizeSession, - sessionMatchesAgent, -} from "../normalize/sessions.js"; -import { - extractMcpToolError, - isDaemonOfflineErrorMessage, - isDaemonSuppressibleErrorMessage, - isDaemonTimeoutErrorMessage, - isReadyReadinessPayload, - isReachableHealthPayload, - parseMcpToolResult, - setElementInert, -} from "../utils/daemon.js"; - -export function useDashboardHooks() { -${hookBody} - return { - panel, - setPanel, - brainPanelMounted, - panelMotionDirection, - daemonState, - healthMeta, - stats, - sessions, - tasks, - locks, - feedEntries, - messageEntries, - activityEntries, - sidebarCollapsed, - setSidebarCollapsed, - isNarrowViewport, - savings, - memoryQuery, - setMemoryQuery, - memoryResults, - memorySearching, - feedFilters, - setFeedFilters, - selectedOperator, - setSelectedOperator, - messageTarget, - setMessageTarget, - messageDraft, - setMessageDraft, - taskCompletionDrafts, - setTaskCompletionDrafts, - completionTaskId, - setCompletionTaskId, - busyActionKey, - activitySince, - setActivitySince, - feedbackMessage, - daemonTimeoutStaleSummary, - conflictPairs, - resolveDrafts, - conflictLoading, - permissionGrants, - permissionLoading, - permissionAccessDenied, - permissionsEndpointAvailable, - permissionDraft, - setPermissionDraft, - editorSetup, - editorDetections, - selectedEditorIds, - cortexBase, - setCortexBase, - showConnectionDialog, - showEditorSetupWizard, - availableUpdate, - updateInstalling, - restartingDaemon, - restartError, - showMissionMetricLegend, - setShowMissionMetricLegend, - showMissionCompactUnits, - setShowMissionCompactUnits, - hasVisitedAnalytics, - analyticsReady, - isSettingUpEditors, - controlSettings, - budgetConfigStatus, - budgetDraft, - budgetDraftDirty, - budgetConfigBusy, - budgetConfigMessage, - ipcAvailable, - currency, - setCurrency, - analyticsMode, - setAnalyticsMode, - effectiveReducedMotion, - invokeRef, - tokenRef, - connectionDialogRef, - connectionDialogTriggerRef, - editorSetupDialogRef, - editorSetupTriggerRef, - topbarRef, - analyticsPanelRef, - brainPanelRef, - analyticsTabRefs, - isTauriRuntime, - changePanel, - normalizedSessions, - knownAgents, - editorSetupSummary, - editorDetectionSummary, - manualMcpSnippet, - selectedOperatorName, - messageTargetName, - safeCurrency, - budgetSummary, - budgetDraftError, - budgetDraftEndpoints, - memoryLoad, - formatCurrency, - savingsEstimateLegend, - formatMissionTokenValue, - runRefreshAll, - openConnectionDialog, - dismissConnectionDialog, - closeConnectionDialog, - closeEditorSetupWizard, - updateControlSetting, - restoreFocusToTrigger, - toggleEditorSelection, - openEditorSetupWizard, - applyEditorSetup, - reloadBudgetConfigDraft, - saveBudgetConfigDraft, - updateBudgetDraftRoot, - updateBudgetEndpointDraft, - handleStartDaemon, - handleStopDaemon, - handleRestartDaemon, - handleTaskClaim, - handleTaskAbandon, - handleTaskComplete, - handleTaskDelete, - handleUnlock, - handleSendMessage, - handleFeedAck, - handleMemorySearch, - handleMemoryExpand, - handleResolveConflict, - handleResolveDraftChange, - handleGrantPermission, - handleRevokePermission, - refreshMessages, - refreshActivity, - refreshFeed, - refreshConflicts, - refreshPermissions, - refreshSavings, - reportSurfaceError, - readAuthToken, - api, - postApi, - call, - pill, - utilityPill, - sidebarUtilityStats, - daemonRecoveryHint, - daemonStatusBadge, - daemonSysStatus, - pendingTasks, - claimedTasks, - completedTasks, - monteCarloProjection, - bootSavingsMomentum, - latestRecallHitRate, - recallWindowAverage, - recallWindowSpread, - topActivityEntries, - topFeedEntries, - recentOverviewTasks, - firstRunReadiness, - handleFirstRunAction, - activePanelLabel, - connectionEndpoint, - hostLabel, - handleAnalyticsTabKey, - effectiveSidebarCollapsed, - canStartDaemon, - canStopDaemon, - canSetupEditors, - operationRows, - operationMaxSaved, - topSavingsByAgent, - }; -} -`, - ); - - // Fix hook: the original ends with pre-return vars, not return. Remove duplicate return block artifacts. - // The hook body already contains everything up to handleAnalyticsTabKey. - - writeFile( - "app/panels/panel-stage.jsx", - `/* eslint-disable react/jsx-max-depth */ -import { AppIcon } from "../../ui-icons.jsx"; -import { CURRENCY_OPTIONS, SAVINGS_OPERATION_LABELS, SAVINGS_USD_PER_MILLION, SAVINGS_HISTORY_DAYS, timeAgo, MISSION_METRIC_LEGEND, CONTROL_CENTER_VERSION } from "../../constants.js"; -import { BUDGET_ENDPOINT_DEFINITIONS } from "../../settings/settings-state.js"; -import { MOTION_MS } from "../../design/motion.js"; -import { handleKeyboardActivation } from "../../keyboard-access.js"; -import { sameAgent } from "../../live-surface.js"; -import { DEFAULT_CORTEX_BASE } from "../constants.js"; -import { persistBrowserAuthToken } from "../browser-bootstrap.js"; -import { normalizeCurrencyCode, formatDaemonEndpoint } from "../utils/format.js"; -import { conflictBadgeClass } from "../normalize/conflicts.js"; -import { agentColor } from "../utils/agent-color.js"; -import { AnimatedNumber } from "../components/AnimatedNumber.jsx"; -import { Sparkline } from "../components/Sparkline.jsx"; -import { MonteCarloProjectionChart } from "../components/MonteCarloProjectionChart.jsx"; -import { EmptyItem } from "../components/common.jsx"; -import { AgentItem } from "../components/AgentItem.jsx"; -import { OperatorSelector } from "../components/OperatorSelector.jsx"; -import { TaskItem } from "../components/TaskItem.jsx"; -import { LockItem } from "../components/LockItem.jsx"; -import { FeedItem } from "../components/FeedItem.jsx"; -import { MessageItem } from "../components/MessageItem.jsx"; -import { ActivityItem } from "../components/ActivityItem.jsx"; -import { ConflictPairCard } from "../components/ConflictPairCard.jsx"; -import { PANEL_SEQUENCE } from "../constants.js"; -import { BrainVisualizerPanel } from "../components/BrainVisualizerPanel.jsx"; - -export function PanelStage(props) { - const d = props; - const { - panel, panelMotionDirection, hasVisitedAnalytics, analyticsReady, brainPanelMounted, - } = d; - - return ( -${panelStageInner.split("\n").map((line) => (line ? ` ${line}` : "")).join("\n")} - ); -} -`, - ); - - writeFile( - "app/AppShell.jsx", - `import { useEffect } from "react"; -import { installUpdate } from "../updater.js"; -import { AppIcon } from "../ui-icons.jsx"; -import { PANEL_SEQUENCE } from "./constants.js"; -import { PanelStage } from "./panels/panel-stage.jsx"; - -export function AppShell(d) { - const { - effectiveSidebarCollapsed, - panel, - changePanel, - pill, - utilityPill, - sidebarUtilityStats, - activePanelLabel, - daemonState, - daemonRecoveryHint, - handleRestartDaemon, - restartingDaemon, - invokeRef, - handleStartDaemon, - handleStopDaemon, - canStartDaemon, - canStopDaemon, - restartError, - availableUpdate, - updateInstalling, - setUpdateInstalling, - setFeedbackMessage, - feedbackMessage, - setSidebarCollapsed, - topbarRef, - stats, - normalizedSessions, - openConnectionDialog, - hostLabel, - daemonStatusBadge, - showEditorSetupWizard, - isSettingUpEditors, - closeEditorSetupWizard, - editorSetupDialogRef, - editorDetectionSummary, - selectedEditorIds, - toggleEditorSelection, - manualMcpSnippet, - applyEditorSetup, - showConnectionDialog, - dismissConnectionDialog, - connectionDialogRef, - connectionDialogTriggerRef, - isTauriRuntime, - connectionEndpoint, - closeConnectionDialog, - setCortexBase, - tokenRef, - persistBrowserAuthToken, - readAuthToken, - refreshAllRef, - DEFAULT_CORTEX_BASE, - trapFocusInContainer, - restoreFocusToTrigger, - } = d; - - useEffect(() => { - if (!showConnectionDialog || !connectionDialogRef.current) return undefined; - return trapFocusInContainer(connectionDialogRef.current); - }, [showConnectionDialog]); - - useEffect(() => { - if (!showEditorSetupWizard || !editorSetupDialogRef.current) return undefined; - return trapFocusInContainer(editorSetupDialogRef.current); - }, [showEditorSetupWizard]); - - return ( -
- Skip to main content - - -
-

- {feedbackMessage} -

-
-
- CORTEX - / - {activePanelLabel.toUpperCase()} -
-
- MEM {stats.memories} - DEC {stats.decisions} - EVT {stats.events} - AGENTS {normalizedSessions.length} - - - {daemonStatusBadge.label} - -
-
- - {showEditorSetupWizard && ( -
!isSettingUpEditors && closeEditorSetupWizard()}> -
e.stopPropagation()} - > -
-
- Shared MCP Registration -

Setup MCP

-
- - {editorDetectionSummary.detected}/{editorDetectionSummary.results.length} - -
-

- Choose which supported clients should receive the shared Cortex attach-only MCP entry. Every client points at the same - app-owned daemon command. -

-
- {editorDetectionSummary.results.map((entry) => { - const tone = !entry.detected ? "idle" : entry.registered ? "ok" : "warn"; - const stateLabel = !entry.detected ? "Not detected" : entry.registered ? "Configured" : "Detected"; - const selected = selectedEditorIds.includes(entry.id); - return ( - - ); - })} -
-
- Manual Fallback -

If a client is missing from the supported list, register this MCP server manually or paste it into that AI's setup flow:

-
{manualMcpSnippet}
-

Replace codex with that AI's agent ID (for example: claude, cursor, gemini).

-
-
- - -
-
-
- )} - - {showConnectionDialog && ( -
-
e.stopPropagation()} - > -
-

Connection Settings

- -
-

- {isTauriRuntime - ? "Desktop app mode uses the local app-managed Cortex daemon only." - : "Connect to a local or remote Cortex daemon"} -

-
{ - e.preventDefault(); - if (isTauriRuntime) { - setCortexBase(DEFAULT_CORTEX_BASE); - tokenRef.current = ""; - persistBrowserAuthToken(""); - closeConnectionDialog(); - queueMicrotask(() => refreshAllRef.current()); - return; - } - const fd = new FormData(e.target); - const host = fd.get("host")?.toString().trim() || "127.0.0.1"; - const port = fd.get("port")?.toString().trim() || "7437"; - const token = fd.get("token")?.toString().trim(); - setCortexBase(\`http://\${host}:\${port}\`); - tokenRef.current = token || ""; - persistBrowserAuthToken(token || ""); - closeConnectionDialog(); - queueMicrotask(() => refreshAllRef.current()); - }}> - - - -
- - -
- -
-
- )} - - -
-
- ); -} -`, - ); - - writeFile( - "App.jsx", - `export { App } from "./app/App.jsx"; -`, - ); - - writeFile( - "app/App.jsx", - `import { useEffect } from "react"; -import { DEFAULT_CORTEX_BASE } from "./constants.js"; -import { persistBrowserAuthToken } from "./browser-bootstrap.js"; -import { useDashboardHooks } from "./hooks/useDashboardHooks.js"; -import { AppShell } from "./AppShell.jsx"; - -export function App() { - const dashboard = useDashboardHooks(); - const { refreshAllRef, runRefreshAll } = dashboard; - - useEffect(() => { - refreshAllRef.current = runRefreshAll; - }, [refreshAllRef, runRefreshAll]); - - return ; -} -`, - ); - - console.log(`Split App.jsx (${total} lines) into app/ modules`); -} - -function splitStyles() { - const cssPath = path.join(SRC, "styles.css"); - const lines = readFilesLines(cssPath); - const sections = [ - { file: "styles/base.css", start: 1, end: 194 }, - { file: "styles/layout.css", start: 195, end: 673 }, - { file: "styles/components.css", start: 674, end: 1298 }, - { file: "styles/topbar.css", start: 1299, end: 1518 }, - { file: "styles/animations.css", start: 1519, end: 1783 }, - { file: "styles/charts.css", start: 1784, end: 1932 }, - { file: "styles/panels/analytics.css", start: 1933, end: 2828 }, - { file: "styles/panels/coming-soon.css", start: 2829, end: 2872 }, - { file: "styles/panels/brain.css", start: 2873, end: 3240 }, - { file: "styles/overrides-2026.css", start: 3241, end: 4491 }, - { file: "styles/sidebar-collapse.css", start: 4492, end: 4790 }, - { file: "styles/connection-dialog.css", start: 4791, end: 4977 }, - { file: "styles/panels/conflicts.css", start: 4978, end: 5279 }, - { file: "styles/accessibility.css", start: 5280, end: 5520 }, - ]; - - for (const section of sections) { - writeFile(section.file, sliceLines(lines, section.start, section.end)); - } - - const imports = sections.map((s) => `@import "./${s.file.replace(/^styles\//, "")}";`).join("\n"); - writeFile("styles/index.css", imports + "\n"); - - writeFile("styles.css", '@import "./styles/index.css";\n'); - - console.log(`Split styles.css (${lines.length} lines) into styles/ modules`); -} - -function readFilesLines(file) { - return fs.readFileSync(file, "utf8").split("\n"); -} - -splitApp(); -splitStyles(); -console.log("Done."); diff --git a/desktop/cortex-control-center/src-tauri/build.rs b/desktop/cortex-control-center/src-tauri/build.rs index 1760c396..3e11bddc 100644 --- a/desktop/cortex-control-center/src-tauri/build.rs +++ b/desktop/cortex-control-center/src-tauri/build.rs @@ -22,11 +22,7 @@ fn copy_sidecar_binary() { let binaries_dir = manifest_dir.join("binaries"); let _ = fs::create_dir_all(&binaries_dir); - let ext = if target_triple.contains("windows") { - ".exe" - } else { - "" - }; + let ext = if target_triple.contains("windows") { ".exe" } else { "" }; let dest = binaries_dir.join(format!("cortex-{target_triple}{ext}")); let profile = env::var("PROFILE").unwrap_or_default(); @@ -37,75 +33,29 @@ fn copy_sidecar_binary() { candidates.push(PathBuf::from(sidecar_override)); } - if let Some(repo_root) = manifest_dir - .parent() - .and_then(|p| p.parent()) - .and_then(|p| p.parent()) - { + if let Some(repo_root) = manifest_dir.parent().and_then(|p| p.parent()).and_then(|p| p.parent()) { let daemon_root = repo_root.join("daemon-rs"); if profile != "release" { - candidates.push( - daemon_root - .join(DEV_DAEMON_TARGET_DIR) - .join("debug") - .join(format!("cortex{ext}")), - ); - candidates.push( - daemon_root - .join("target") - .join("debug") - .join(format!("cortex{ext}")), - ); - candidates.push( - daemon_root - .join(RELEASE_DAEMON_TARGET_DIR) - .join("release") - .join(format!("cortex{ext}")), - ); + candidates.push(daemon_root.join(DEV_DAEMON_TARGET_DIR).join("debug").join(format!("cortex{ext}"))); + candidates.push(daemon_root.join("target").join("debug").join(format!("cortex{ext}"))); + candidates.push(daemon_root.join(RELEASE_DAEMON_TARGET_DIR).join("release").join(format!("cortex{ext}"))); } else { - candidates.push( - daemon_root - .join(RELEASE_DAEMON_TARGET_DIR) - .join("release") - .join(format!("cortex{ext}")), - ); + candidates.push(daemon_root.join(RELEASE_DAEMON_TARGET_DIR).join("release").join(format!("cortex{ext}"))); } - candidates.push( - daemon_root - .join("target") - .join("release") - .join(format!("cortex{ext}")), - ); + candidates.push(daemon_root.join("target").join("release").join(format!("cortex{ext}"))); } - let home = env::var_os("USERPROFILE") - .or_else(|| env::var_os("HOME")) - .map(PathBuf::from); + let home = env::var_os("USERPROFILE").or_else(|| env::var_os("HOME")).map(PathBuf::from); if let Some(home) = home { - candidates.push( - home.join(".cortex") - .join("bin") - .join(format!("cortex{ext}")), - ); - candidates.push( - home.join("cortex") - .join("daemon-rs") - .join("target") - .join("release") - .join(format!("cortex{ext}")), - ); + candidates.push(home.join(".cortex").join("bin").join(format!("cortex{ext}"))); + candidates.push(home.join("cortex").join("daemon-rs").join("target").join("release").join(format!("cortex{ext}"))); } for src in candidates { if src.exists() { if let Err(err) = copy_if_changed(&src, &dest) { - println!( - "cargo:warning=Failed to copy Cortex sidecar from {} to {}: {}", - src.display(), - dest.display(), - err - ); + println!("cargo:warning=Failed to copy Cortex sidecar from {} to {}: {}", src.display(), dest.display(), err); } return; } diff --git a/desktop/cortex-control-center/src-tauri/rustfmt.toml b/desktop/cortex-control-center/src-tauri/rustfmt.toml new file mode 100644 index 00000000..40d36bf9 --- /dev/null +++ b/desktop/cortex-control-center/src-tauri/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 160 +use_small_heuristics = "Max" diff --git a/desktop/cortex-control-center/src-tauri/src/budget/mod.rs b/desktop/cortex-control-center/src-tauri/src/budget/mod.rs index 1baf90be..966c1da3 100644 --- a/desktop/cortex-control-center/src-tauri/src/budget/mod.rs +++ b/desktop/cortex-control-center/src-tauri/src/budget/mod.rs @@ -88,36 +88,19 @@ struct BudgetTomlEndpoint { } pub fn budget_config_path() -> Result { - let home = resolved_cortex_paths() - .home - .or_else(|| default_cortex_dir().ok()) - .ok_or_else(|| "Could not resolve Cortex home path".to_string())?; - fs::create_dir_all(&home) - .map_err(|err| format!("Failed to create Cortex home {}: {err}", home.display()))?; - let canonical_home = fs::canonicalize(&home) - .map_err(|err| format!("Failed to resolve Cortex home {}: {err}", home.display()))?; + let home = resolved_cortex_paths().home.or_else(|| default_cortex_dir().ok()).ok_or_else(|| "Could not resolve Cortex home path".to_string())?; + fs::create_dir_all(&home).map_err(|err| format!("Failed to create Cortex home {}: {err}", home.display()))?; + let canonical_home = fs::canonicalize(&home).map_err(|err| format!("Failed to resolve Cortex home {}: {err}", home.display()))?; Ok(canonical_home.join(BUDGETS_FILE_NAME)) } -fn budget_error( - code: &str, - message: impl Into, - endpoint: Option, - field: Option<&str>, -) -> BudgetConfigErrorSnapshot { - BudgetConfigErrorSnapshot { - code: code.to_string(), - message: message.into(), - endpoint, - field: field.map(str::to_string), - } +fn budget_error(code: &str, message: impl Into, endpoint: Option, field: Option<&str>) -> BudgetConfigErrorSnapshot { + BudgetConfigErrorSnapshot { code: code.to_string(), message: message.into(), endpoint, field: field.map(str::to_string) } } fn parse_budget_endpoint_name(name: &str) -> Option { let normalized = name.trim().to_ascii_lowercase(); - BUDGET_ENDPOINT_NAMES - .contains(&normalized.as_str()) - .then_some(normalized) + BUDGET_ENDPOINT_NAMES.contains(&normalized.as_str()).then_some(normalized) } fn budget_source_label() -> String { @@ -125,14 +108,7 @@ fn budget_source_label() -> String { } fn empty_budget_snapshot(_path: &Path) -> BudgetConfigSnapshot { - BudgetConfigSnapshot { - config_loaded: false, - enabled: false, - source: budget_source_label(), - error: None, - endpoints: BTreeMap::new(), - recent_denials: 0, - } + BudgetConfigSnapshot { config_loaded: false, enabled: false, source: budget_source_label(), error: None, endpoints: BTreeMap::new(), recent_denials: 0 } } pub fn budget_snapshot_from_contents(_path: &Path, contents: &str) -> BudgetConfigSnapshot { @@ -143,22 +119,14 @@ pub fn budget_snapshot_from_contents(_path: &Path, contents: &str) -> BudgetConf config_loaded: true, enabled: false, source: budget_source_label(), - error: Some(budget_error( - "parse_error", - format!("failed to parse budgets.toml: {err}"), - None, - None, - )), + error: Some(budget_error("parse_error", format!("failed to parse budgets.toml: {err}"), None, None)), endpoints: BTreeMap::new(), recent_denials: 0, }; } }; - let enabled = parsed - .defaults - .and_then(|defaults| defaults.enabled) - .unwrap_or(true); + let enabled = parsed.defaults.and_then(|defaults| defaults.enabled).unwrap_or(true); let mut endpoints = BTreeMap::new(); for (raw_name, raw_budget) in parsed.endpoints.unwrap_or_default() { let Some(endpoint) = parse_budget_endpoint_name(&raw_name) else { @@ -166,12 +134,7 @@ pub fn budget_snapshot_from_contents(_path: &Path, contents: &str) -> BudgetConf config_loaded: true, enabled: false, source: budget_source_label(), - error: Some(budget_error( - "unknown_endpoint", - format!("unknown budget endpoint: {raw_name}"), - Some(raw_name), - None, - )), + error: Some(budget_error("unknown_endpoint", format!("unknown budget endpoint: {raw_name}"), Some(raw_name), None)), endpoints: BTreeMap::new(), recent_denials: 0, }; @@ -181,12 +144,7 @@ pub fn budget_snapshot_from_contents(_path: &Path, contents: &str) -> BudgetConf config_loaded: true, enabled: false, source: budget_source_label(), - error: Some(budget_error( - "missing_limit", - format!("budget endpoint {endpoint} is missing limit"), - Some(endpoint), - Some("limit"), - )), + error: Some(budget_error("missing_limit", format!("budget endpoint {endpoint} is missing limit"), Some(endpoint), Some("limit"))), endpoints: BTreeMap::new(), recent_denials: 0, }; @@ -238,23 +196,10 @@ pub fn budget_snapshot_from_contents(_path: &Path, contents: &str) -> BudgetConf }; } - endpoints.insert( - endpoint, - BudgetEndpointSnapshot { - limit: limit as u64, - window_seconds: window_seconds as u64, - }, - ); + endpoints.insert(endpoint, BudgetEndpointSnapshot { limit: limit as u64, window_seconds: window_seconds as u64 }); } - BudgetConfigSnapshot { - config_loaded: true, - enabled, - source: budget_source_label(), - error: None, - endpoints, - recent_denials: 0, - } + BudgetConfigSnapshot { config_loaded: true, enabled, source: budget_source_label(), error: None, endpoints, recent_denials: 0 } } pub fn read_budget_config_snapshot(path: &Path) -> Result { @@ -265,12 +210,7 @@ pub fn read_budget_config_snapshot(path: &Path) -> Result Result MAX_BUDGET_INTEGER { - return Err(format!( - "Budget endpoint {endpoint} limit must be between 1 and {MAX_BUDGET_INTEGER}" - )); + return Err(format!("Budget endpoint {endpoint} limit must be between 1 and {MAX_BUDGET_INTEGER}")); } - let window_seconds = raw - .window_seconds - .ok_or_else(|| format!("Budget endpoint {endpoint} is missing window_seconds"))?; + let window_seconds = raw.window_seconds.ok_or_else(|| format!("Budget endpoint {endpoint} is missing window_seconds"))?; if window_seconds == 0 || window_seconds > MAX_BUDGET_INTEGER { - return Err(format!( - "Budget endpoint {endpoint} window_seconds must be between 1 and {MAX_BUDGET_INTEGER}" - )); + return Err(format!("Budget endpoint {endpoint} window_seconds must be between 1 and {MAX_BUDGET_INTEGER}")); } - endpoints.insert( - endpoint, - BudgetTomlEndpoint { - limit, - window_seconds, - }, - ); + endpoints.insert(endpoint, BudgetTomlEndpoint { limit, window_seconds }); } - Ok(BudgetTomlFile { - defaults: BudgetTomlDefaults { - enabled: draft.enabled, - }, - endpoints, - }) + Ok(BudgetTomlFile { defaults: BudgetTomlDefaults { enabled: draft.enabled }, endpoints }) } pub fn write_budget_config_file(path: &Path, contents: &str) -> Result<(), String> { - let parent = path - .parent() - .ok_or_else(|| format!("Invalid budget config path: {}", path.display()))?; - fs::create_dir_all(parent) - .map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; + let parent = path.parent().ok_or_else(|| format!("Invalid budget config path: {}", path.display()))?; + fs::create_dir_all(parent).map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; let temp_path = parent.join(format!(".{}.{}.tmp", BUDGETS_FILE_NAME, std::process::id())); { - let mut file = File::create(&temp_path) - .map_err(|err| format!("Failed to create {}: {err}", temp_path.display()))?; - file.write_all(contents.as_bytes()) - .map_err(|err| format!("Failed to write {}: {err}", temp_path.display()))?; - file.sync_all() - .map_err(|err| format!("Failed to flush {}: {err}", temp_path.display()))?; + let mut file = File::create(&temp_path).map_err(|err| format!("Failed to create {}: {err}", temp_path.display()))?; + file.write_all(contents.as_bytes()).map_err(|err| format!("Failed to write {}: {err}", temp_path.display()))?; + file.sync_all().map_err(|err| format!("Failed to flush {}: {err}", temp_path.display()))?; } if path.exists() { - fs::remove_file(path) - .map_err(|err| format!("Failed to replace {}: {err}", path.display()))?; + fs::remove_file(path).map_err(|err| format!("Failed to replace {}: {err}", path.display()))?; } fs::rename(&temp_path, path).map_err(|err| { let _ = fs::remove_file(&temp_path); @@ -354,8 +267,7 @@ pub fn write_budget_config_file(path: &Path, contents: &str) -> Result<(), Strin pub fn save_budget_from_draft(draft: BudgetConfigDraft) -> Result { let path = budget_config_path()?; let config = validate_budget_draft(draft)?; - let contents = toml::to_string_pretty(&config) - .map_err(|err| format!("Failed to serialize budget config: {err}"))?; + let contents = toml::to_string_pretty(&config).map_err(|err| format!("Failed to serialize budget config: {err}"))?; write_budget_config_file(&path, &contents)?; read_budget_config_snapshot(&path) } diff --git a/desktop/cortex-control-center/src-tauri/src/budget/tests.rs b/desktop/cortex-control-center/src-tauri/src/budget/tests.rs index f1099c6d..8f177f15 100644 --- a/desktop/cortex-control-center/src-tauri/src/budget/tests.rs +++ b/desktop/cortex-control-center/src-tauri/src/budget/tests.rs @@ -2,12 +2,11 @@ use super::*; use std::fs; use std::path::Path; - - #[test] - fn budget_editor_snapshot_parses_valid_config() { - let snapshot = budget_snapshot_from_contents( - Path::new("C:/cortex-test/testuser/.cortex/budgets.toml"), - r#" +#[test] +fn budget_editor_snapshot_parses_valid_config() { + let snapshot = budget_snapshot_from_contents( + Path::new("C:/cortex-test/testuser/.cortex/budgets.toml"), + r#" [defaults] enabled = true @@ -15,117 +14,80 @@ enabled = true limit = 300 window_seconds = 60 "#, - ); - - assert!(snapshot.config_loaded); - assert!(snapshot.enabled); - assert_eq!( - snapshot.error.as_ref().map(|error| error.code.as_str()), - None - ); - assert_eq!(snapshot.source, "budgets.toml"); - assert_eq!(snapshot.endpoints["recall"].limit, 300); - assert_eq!(snapshot.endpoints["recall"].window_seconds, 60); - } - - - #[test] - fn budget_editor_snapshot_returns_structured_errors() { - let snapshot = budget_snapshot_from_contents( - Path::new("C:/cortex-test/testuser/.cortex/budgets.toml"), - r#" + ); + + assert!(snapshot.config_loaded); + assert!(snapshot.enabled); + assert_eq!(snapshot.error.as_ref().map(|error| error.code.as_str()), None); + assert_eq!(snapshot.source, "budgets.toml"); + assert_eq!(snapshot.endpoints["recall"].limit, 300); + assert_eq!(snapshot.endpoints["recall"].window_seconds, 60); +} + +#[test] +fn budget_editor_snapshot_returns_structured_errors() { + let snapshot = budget_snapshot_from_contents( + Path::new("C:/cortex-test/testuser/.cortex/budgets.toml"), + r#" [endpoints.unknown] limit = 1 window_seconds = 60 "#, - ); - - let error = snapshot.error.expect("unknown endpoint should be invalid"); - assert_eq!(error.code, "unknown_endpoint"); - assert_eq!(error.endpoint.as_deref(), Some("unknown")); - assert!(!snapshot.enabled); - } - - - #[test] - fn budget_editor_draft_serializes_only_enabled_endpoints() { - let config = validate_budget_draft(BudgetConfigDraft { - enabled: true, - endpoints: vec![ - BudgetEndpointDraft { - endpoint: "store".to_string(), - enabled: false, - limit: Some(120), - window_seconds: Some(60), - }, - BudgetEndpointDraft { - endpoint: "recall".to_string(), - enabled: true, - limit: Some(42), - window_seconds: Some(15), - }, - ], - }) - .expect("draft should validate"); - - assert!(config.defaults.enabled); - assert_eq!(config.endpoints.len(), 1); - assert_eq!(config.endpoints["recall"].limit, 42); - } - - - #[test] - fn budget_editor_rejects_duplicate_or_invalid_endpoint_drafts() { - let duplicate = validate_budget_draft(BudgetConfigDraft { - enabled: true, - endpoints: vec![ - BudgetEndpointDraft { - endpoint: "recall".to_string(), - enabled: true, - limit: Some(1), - window_seconds: Some(60), - }, - BudgetEndpointDraft { - endpoint: "recall".to_string(), - enabled: true, - limit: Some(2), - window_seconds: Some(60), - }, - ], - }) - .unwrap_err(); - assert!(duplicate.contains("Duplicate budget endpoint")); - - let invalid = validate_budget_draft(BudgetConfigDraft { - enabled: true, - endpoints: vec![BudgetEndpointDraft { - endpoint: "recall".to_string(), - enabled: true, - limit: Some(0), - window_seconds: Some(60), - }], - }) - .unwrap_err(); - assert!(invalid.contains("limit must be between 1")); - } - - - #[test] - fn budget_editor_write_replaces_file_atomically() { - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time") - .as_nanos(); - let dir = std::env::temp_dir().join(format!("cortex-budget-editor-{unique}")); - fs::create_dir_all(&dir).expect("create temp dir"); - let path = dir.join("budgets.toml"); - - write_budget_config_file(&path, "[defaults]\nenabled = true\n") - .expect("write initial file"); - write_budget_config_file(&path, "[defaults]\nenabled = false\n") - .expect("replace existing file"); - - let contents = fs::read_to_string(&path).expect("read replaced file"); - assert!(contents.contains("enabled = false")); - let _ = fs::remove_dir_all(&dir); - } + ); + + let error = snapshot.error.expect("unknown endpoint should be invalid"); + assert_eq!(error.code, "unknown_endpoint"); + assert_eq!(error.endpoint.as_deref(), Some("unknown")); + assert!(!snapshot.enabled); +} + +#[test] +fn budget_editor_draft_serializes_only_enabled_endpoints() { + let config = validate_budget_draft(BudgetConfigDraft { + enabled: true, + endpoints: vec![ + BudgetEndpointDraft { endpoint: "store".to_string(), enabled: false, limit: Some(120), window_seconds: Some(60) }, + BudgetEndpointDraft { endpoint: "recall".to_string(), enabled: true, limit: Some(42), window_seconds: Some(15) }, + ], + }) + .expect("draft should validate"); + + assert!(config.defaults.enabled); + assert_eq!(config.endpoints.len(), 1); + assert_eq!(config.endpoints["recall"].limit, 42); +} + +#[test] +fn budget_editor_rejects_duplicate_or_invalid_endpoint_drafts() { + let duplicate = validate_budget_draft(BudgetConfigDraft { + enabled: true, + endpoints: vec![ + BudgetEndpointDraft { endpoint: "recall".to_string(), enabled: true, limit: Some(1), window_seconds: Some(60) }, + BudgetEndpointDraft { endpoint: "recall".to_string(), enabled: true, limit: Some(2), window_seconds: Some(60) }, + ], + }) + .unwrap_err(); + assert!(duplicate.contains("Duplicate budget endpoint")); + + let invalid = validate_budget_draft(BudgetConfigDraft { + enabled: true, + endpoints: vec![BudgetEndpointDraft { endpoint: "recall".to_string(), enabled: true, limit: Some(0), window_seconds: Some(60) }], + }) + .unwrap_err(); + assert!(invalid.contains("limit must be between 1")); +} + +#[test] +fn budget_editor_write_replaces_file_atomically() { + let unique = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).expect("system time").as_nanos(); + let dir = std::env::temp_dir().join(format!("cortex-budget-editor-{unique}")); + fs::create_dir_all(&dir).expect("create temp dir"); + let path = dir.join("budgets.toml"); + + write_budget_config_file(&path, "[defaults]\nenabled = true\n").expect("write initial file"); + write_budget_config_file(&path, "[defaults]\nenabled = false\n").expect("replace existing file"); + + let contents = fs::read_to_string(&path).expect("read replaced file"); + assert!(contents.contains("enabled = false")); + let _ = fs::remove_dir_all(&dir); +} diff --git a/desktop/cortex-control-center/src-tauri/src/commands/app.rs b/desktop/cortex-control-center/src-tauri/src/commands/app.rs index 233b7a7e..082da583 100644 --- a/desktop/cortex-control-center/src-tauri/src/commands/app.rs +++ b/desktop/cortex-control-center/src-tauri/src/commands/app.rs @@ -25,17 +25,14 @@ pub fn write_dev_verification_report(content: String) -> Result return Err("Dev verification reporting is only available in debug builds.".to_string()); } - let report_path = env::var("CORTEX_DEV_VERIFY_REPORT_PATH") - .map(PathBuf::from) - .map_err(|_| "CORTEX_DEV_VERIFY_REPORT_PATH is not configured.".to_string())?; + let report_path = + env::var("CORTEX_DEV_VERIFY_REPORT_PATH").map(PathBuf::from).map_err(|_| "CORTEX_DEV_VERIFY_REPORT_PATH is not configured.".to_string())?; if let Some(parent) = report_path.parent() { - fs::create_dir_all(parent) - .map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; + fs::create_dir_all(parent).map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; } - fs::write(&report_path, content) - .map_err(|err| format!("Failed to write {}: {err}", report_path.display()))?; + fs::write(&report_path, content).map_err(|err| format!("Failed to write {}: {err}", report_path.display()))?; Ok(report_path.display().to_string()) } diff --git a/desktop/cortex-control-center/src-tauri/src/commands/budget.rs b/desktop/cortex-control-center/src-tauri/src/commands/budget.rs index a1fcc8e5..b8df481d 100644 --- a/desktop/cortex-control-center/src-tauri/src/commands/budget.rs +++ b/desktop/cortex-control-center/src-tauri/src/commands/budget.rs @@ -1,7 +1,4 @@ -use crate::budget::{ - budget_config_path, read_budget_config_snapshot, save_budget_from_draft, BudgetConfigDraft, - BudgetConfigSnapshot, -}; +use crate::budget::{budget_config_path, read_budget_config_snapshot, save_budget_from_draft, BudgetConfigDraft, BudgetConfigSnapshot}; #[tauri::command] pub fn read_budget_config() -> Result { diff --git a/desktop/cortex-control-center/src-tauri/src/commands/cortex.rs b/desktop/cortex-control-center/src-tauri/src/commands/cortex.rs index c991382c..ad804cd3 100644 --- a/desktop/cortex-control-center/src-tauri/src/commands/cortex.rs +++ b/desktop/cortex-control-center/src-tauri/src/commands/cortex.rs @@ -1,28 +1,15 @@ use crate::cortex_http::{send_cortex_request, FetchCortexResponse}; #[tauri::command] -pub async fn fetch_cortex( - path: String, - auth_token: String, - timeout_ms: Option, -) -> Result { - tauri::async_runtime::spawn_blocking(move || { - send_cortex_request("GET", &path, &auth_token, None, timeout_ms) - }) - .await - .map_err(|err| format!("fetch_cortex task failed: {err}"))? +pub async fn fetch_cortex(path: String, auth_token: String, timeout_ms: Option) -> Result { + tauri::async_runtime::spawn_blocking(move || send_cortex_request("GET", &path, &auth_token, None, timeout_ms)) + .await + .map_err(|err| format!("fetch_cortex task failed: {err}"))? } #[tauri::command] -pub async fn post_cortex( - path: String, - auth_token: String, - body: String, - timeout_ms: Option, -) -> Result { - tauri::async_runtime::spawn_blocking(move || { - send_cortex_request("POST", &path, &auth_token, Some(&body), timeout_ms) - }) - .await - .map_err(|err| format!("post_cortex task failed: {err}"))? +pub async fn post_cortex(path: String, auth_token: String, body: String, timeout_ms: Option) -> Result { + tauri::async_runtime::spawn_blocking(move || send_cortex_request("POST", &path, &auth_token, Some(&body), timeout_ms)) + .await + .map_err(|err| format!("post_cortex task failed: {err}"))? } diff --git a/desktop/cortex-control-center/src-tauri/src/commands/daemon.rs b/desktop/cortex-control-center/src-tauri/src/commands/daemon.rs index 207ee8f4..adf7b667 100644 --- a/desktop/cortex-control-center/src-tauri/src/commands/daemon.rs +++ b/desktop/cortex-control-center/src-tauri/src/commands/daemon.rs @@ -1,7 +1,6 @@ use crate::constants::{DAEMON_REACHABILITY_TIMEOUT_MS, DAEMON_STOP_WAIT_MS}; use crate::cortex_http::{ - auth_token_ready, is_cortex_reachable_with_port, probe_cortex_reachability_with_port, - read_auth_token_with_retry, wait_for_reachability, + auth_token_ready, is_cortex_reachable_with_port, probe_cortex_reachability_with_port, read_auth_token_with_retry, wait_for_reachability, }; use crate::daemon::paths::{daemon_port, log_startup_path, service_ensure_fallback_enabled}; use crate::daemon::shutdown::send_http_shutdown; @@ -14,36 +13,22 @@ use tauri::State; pub async fn daemon_status(state: State<'_, DaemonState>) -> Result { let (managed, pid) = state.status()?; let port = daemon_port(); - let probe = tauri::async_runtime::spawn_blocking(move || { - probe_cortex_reachability_with_port(port, DAEMON_REACHABILITY_TIMEOUT_MS) - }) - .await - .map_err(|err| format!("daemon_status reachability task failed: {err}"))?; + let probe = tauri::async_runtime::spawn_blocking(move || probe_cortex_reachability_with_port(port, DAEMON_REACHABILITY_TIMEOUT_MS)) + .await + .map_err(|err| format!("daemon_status reachability task failed: {err}"))?; let reachable = probe.reachable; let starting = probe.starting; let auth_token_ready = if reachable { - tauri::async_runtime::spawn_blocking(auth_token_ready) - .await - .map_err(|err| format!("daemon_status token task failed: {err}"))? + tauri::async_runtime::spawn_blocking(auth_token_ready).await.map_err(|err| format!("daemon_status token task failed: {err}"))? } else { false }; - let mut message = - describe_daemon_state(managed, reachable, starting, auth_token_ready, pid, port); + let mut message = describe_daemon_state(managed, reachable, starting, auth_token_ready, pid, port); if probe.identity_mismatch { - message.push_str( - " Runtime identity metadata mismatch detected; using loose local daemon probe.", - ); + message.push_str(" Runtime identity metadata mismatch detected; using loose local daemon probe."); } - Ok(DaemonCommandResult { - running: managed || reachable || starting, - reachable, - managed, - auth_token_ready, - pid, - message, - }) + Ok(DaemonCommandResult { running: managed || reachable || starting, reachable, managed, auth_token_ready, pid, message }) } #[tauri::command] @@ -64,18 +49,9 @@ pub async fn start_daemon(state: State<'_, DaemonState>) -> Result) -> Result) -> Result) -> Result { - log_startup_path( - "start_daemon", - "service-ensure", - "daemon started or validated via service ensure after app-managed fallback", - ); + log_startup_path("start_daemon", "service-ensure", "daemon started or validated via service ensure after app-managed fallback"); let auth_token_ready = auth_token_ready(); Ok(DaemonCommandResult { running: true, @@ -165,23 +115,15 @@ pub async fn start_daemon(state: State<'_, DaemonState>) -> Result { log_startup_path("start_daemon", "blocked", "app-managed spawn failed"); - Err(format!( - "App-managed local start failed and Windows service ensure was unavailable: {local_err}" - )) + Err(format!("App-managed local start failed and Windows service ensure was unavailable: {local_err}")) } Err(service_err) => { log_startup_path("start_daemon", "blocked", "app-managed spawn failed"); - Err(format!( - "App-managed local start failed: {local_err}. Windows service ensure failed: {service_err}" - )) + Err(format!("App-managed local start failed: {local_err}. Windows service ensure failed: {service_err}")) } } } else { - log_startup_path( - "start_daemon", - "blocked", - "app-managed spawn failed (service fallback disabled)", - ); + log_startup_path("start_daemon", "blocked", "app-managed spawn failed (service fallback disabled)"); Err(format!("App-managed local start failed: {local_err}")) } } @@ -198,15 +140,10 @@ pub async fn stop_daemon(state: State<'_, DaemonState>) -> Result) -> Result, - expected_paths: Option<&ResolvedCortexPaths>, -) -> (bool, bool) { +pub fn health_state_with_identity_fallback(status: u16, body: &str, expected_port: Option, expected_paths: Option<&ResolvedCortexPaths>) -> (bool, bool) { if is_cortex_health_response(status, body, expected_port, expected_paths) { return (true, false); } @@ -51,26 +46,13 @@ pub fn probe_cortex_reachability_with_port(port: u16, timeout_ms: u64) -> Cortex "/readiness", "", None, - RequestTimeouts { - connect: Duration::from_millis(timeout_ms), - read: Duration::from_millis(timeout_ms), - write: Duration::from_millis(timeout_ms), - }, + RequestTimeouts { connect: Duration::from_millis(timeout_ms), read: Duration::from_millis(timeout_ms), write: Duration::from_millis(timeout_ms) }, ); if let Ok(resp) = readiness_response { - let (readiness_state, identity_mismatch) = readiness_state_with_identity_fallback( - resp.status, - &resp.body, - Some(port), - Some(&expected_paths), - ); + let (readiness_state, identity_mismatch) = readiness_state_with_identity_fallback(resp.status, &resp.body, Some(port), Some(&expected_paths)); if let Some(ready) = readiness_state { - return CortexReachabilityProbe { - reachable: ready, - starting: !ready, - identity_mismatch, - }; + return CortexReachabilityProbe { reachable: ready, starting: !ready, identity_mismatch }; } } @@ -80,26 +62,13 @@ pub fn probe_cortex_reachability_with_port(port: u16, timeout_ms: u64) -> Cortex "/health", "", None, - RequestTimeouts { - connect: Duration::from_millis(timeout_ms), - read: Duration::from_millis(timeout_ms), - write: Duration::from_millis(timeout_ms), - }, + RequestTimeouts { connect: Duration::from_millis(timeout_ms), read: Duration::from_millis(timeout_ms), write: Duration::from_millis(timeout_ms) }, ); if let Ok(resp) = health_response { - let (healthy, identity_mismatch) = health_state_with_identity_fallback( - resp.status, - &resp.body, - Some(port), - Some(&expected_paths), - ); + let (healthy, identity_mismatch) = health_state_with_identity_fallback(resp.status, &resp.body, Some(port), Some(&expected_paths)); if healthy { - return CortexReachabilityProbe { - reachable: true, - starting: false, - identity_mismatch, - }; + return CortexReachabilityProbe { reachable: true, starting: false, identity_mismatch }; } } @@ -111,11 +80,7 @@ pub fn is_cortex_reachable_with_port(port: u16, timeout_ms: u64) -> bool { } pub async fn wait_for_reachability(port: u16, target: bool, timeout: Duration) -> bool { - tauri::async_runtime::spawn_blocking(move || { - wait_for_reachability_blocking(port, target, timeout) - }) - .await - .unwrap_or(false) + tauri::async_runtime::spawn_blocking(move || wait_for_reachability_blocking(port, target, timeout)).await.unwrap_or(false) } pub fn wait_for_reachability_blocking(port: u16, target: bool, timeout: Duration) -> bool { @@ -132,8 +97,7 @@ pub fn wait_for_reachability_blocking(port: u16, target: bool, timeout: Duration } pub fn read_auth_token_once() -> Result { let path = token_path()?; - let token = fs::read_to_string(&path) - .map_err(|err| format!("Failed to read token at {}: {err}", path.display()))?; + let token = fs::read_to_string(&path).map_err(|err| format!("Failed to read token at {}: {err}", path.display()))?; Ok(token.trim().to_string()) } @@ -176,11 +140,9 @@ pub fn read_auth_token_with_retry_blocking(timeout: Duration) -> Result Result { - tauri::async_runtime::spawn_blocking(move || { - read_auth_token_with_retry_blocking(Duration::from_millis(AUTH_TOKEN_WAIT_MS)) - }) - .await - .map_err(|err| format!("Auth token wait task failed: {err}"))? + tauri::async_runtime::spawn_blocking(move || read_auth_token_with_retry_blocking(Duration::from_millis(AUTH_TOKEN_WAIT_MS))) + .await + .map_err(|err| format!("Auth token wait task failed: {err}"))? } fn normalize_runtime_path(value: &str) -> String { let mut normalized = value.trim().replace('\\', "/"); @@ -199,18 +161,10 @@ fn health_path_field_matches(value: Option<&serde_json::Value>, expected: Option return true; }; let expected = normalize_runtime_path(&expected.to_string_lossy()); - value - .and_then(|field| field.as_str()) - .map(normalize_runtime_path) - .is_some_and(|actual| actual == expected) + value.and_then(|field| field.as_str()).map(normalize_runtime_path).is_some_and(|actual| actual == expected) } -pub fn cortex_readiness_state( - status: u16, - body: &str, - expected_port: Option, - expected_paths: Option<&ResolvedCortexPaths>, -) -> Option { +pub fn cortex_readiness_state(status: u16, body: &str, expected_port: Option, expected_paths: Option<&ResolvedCortexPaths>) -> Option { let Ok(json) = serde_json::from_str::(body.trim()) else { return None; }; @@ -218,10 +172,7 @@ pub fn cortex_readiness_state( let ready = json.get("ready").and_then(|value| value.as_bool())?; let runtime = json.get("runtime").and_then(|value| value.as_object())?; let stats = json.get("stats").and_then(|value| value.as_object())?; - let runtime_port = runtime - .get("port") - .and_then(|value| value.as_u64()) - .and_then(|value| u16::try_from(value).ok()); + let runtime_port = runtime.get("port").and_then(|value| value.as_u64()).and_then(|value| u16::try_from(value).ok()); if let Some(expected_port) = expected_port { if runtime_port != Some(expected_port) { @@ -262,12 +213,7 @@ pub fn cortex_readiness_state( Some(ready) } -pub fn is_cortex_health_response( - status: u16, - body: &str, - expected_port: Option, - expected_paths: Option<&ResolvedCortexPaths>, -) -> bool { +pub fn is_cortex_health_response(status: u16, body: &str, expected_port: Option, expected_paths: Option<&ResolvedCortexPaths>) -> bool { if !(200..300).contains(&status) { return false; } @@ -279,10 +225,7 @@ pub fn is_cortex_health_response( let health_status = json.get("status").and_then(|value| value.as_str()); let runtime = json.get("runtime").and_then(|value| value.as_object()); let stats = json.get("stats").and_then(|value| value.as_object()); - let runtime_port = runtime - .and_then(|runtime| runtime.get("port")) - .and_then(|value| value.as_u64()) - .and_then(|value| u16::try_from(value).ok()); + let runtime_port = runtime.and_then(|runtime| runtime.get("port")).and_then(|value| value.as_u64()).and_then(|value| u16::try_from(value).ok()); if let Some(expected_port) = expected_port { if runtime_port != Some(expected_port) { @@ -291,26 +234,16 @@ pub fn is_cortex_health_response( } if let Some(paths) = expected_paths { - if !health_path_field_matches(stats.and_then(|obj| obj.get("home")), paths.home.as_deref()) - { + if !health_path_field_matches(stats.and_then(|obj| obj.get("home")), paths.home.as_deref()) { return false; } - if !health_path_field_matches( - runtime.and_then(|obj| obj.get("token_path")), - paths.token.as_deref(), - ) { + if !health_path_field_matches(runtime.and_then(|obj| obj.get("token_path")), paths.token.as_deref()) { return false; } - if !health_path_field_matches( - runtime.and_then(|obj| obj.get("db_path")), - paths.db.as_deref(), - ) { + if !health_path_field_matches(runtime.and_then(|obj| obj.get("db_path")), paths.db.as_deref()) { return false; } - if !health_path_field_matches( - runtime.and_then(|obj| obj.get("pid_path")), - paths.pid.as_deref(), - ) { + if !health_path_field_matches(runtime.and_then(|obj| obj.get("pid_path")), paths.pid.as_deref()) { return false; } } diff --git a/desktop/cortex-control-center/src-tauri/src/cortex_http/request.rs b/desktop/cortex-control-center/src-tauri/src/cortex_http/request.rs index 5a582474..34cae9bf 100644 --- a/desktop/cortex-control-center/src-tauri/src/cortex_http/request.rs +++ b/desktop/cortex-control-center/src-tauri/src/cortex_http/request.rs @@ -17,18 +17,8 @@ pub struct FetchCortexResponse { pub status: u16, pub body: String, } -pub fn send_cortex_request( - method: &str, - path: &str, - auth_token: &str, - body: Option<&str>, - timeout_ms: Option, -) -> Result { - let read_timeout = Duration::from_millis( - timeout_ms - .unwrap_or(DAEMON_READ_TIMEOUT_MS) - .clamp(DAEMON_MIN_REQUEST_TIMEOUT_MS, DAEMON_MAX_REQUEST_TIMEOUT_MS), - ); +pub fn send_cortex_request(method: &str, path: &str, auth_token: &str, body: Option<&str>, timeout_ms: Option) -> Result { + let read_timeout = Duration::from_millis(timeout_ms.unwrap_or(DAEMON_READ_TIMEOUT_MS).clamp(DAEMON_MIN_REQUEST_TIMEOUT_MS, DAEMON_MAX_REQUEST_TIMEOUT_MS)); send_cortex_request_with_port( daemon_port(), method, @@ -47,10 +37,7 @@ pub fn should_use_partial_response_on_read_timeout(err: &std::io::Error, respons return false; } - if matches!( - err.kind(), - std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock - ) { + if matches!(err.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) { return true; } @@ -85,17 +72,11 @@ pub fn send_cortex_request_with_port( validate_cortex_request_path(path)?; let mut stream = - TcpStream::connect_timeout(&SocketAddr::from(([127, 0, 0, 1], port)), timeouts.connect) - .map_err(|e| format!("Cannot connect to daemon: {e}"))?; - stream - .set_read_timeout(Some(timeouts.read)) - .map_err(|e| format!("Cannot set read timeout: {e}"))?; - stream - .set_write_timeout(Some(timeouts.write)) - .map_err(|e| format!("Cannot set write timeout: {e}"))?; - - let mut request = - format!("{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nX-Cortex-Request: true\r\n"); + TcpStream::connect_timeout(&SocketAddr::from(([127, 0, 0, 1], port)), timeouts.connect).map_err(|e| format!("Cannot connect to daemon: {e}"))?; + stream.set_read_timeout(Some(timeouts.read)).map_err(|e| format!("Cannot set read timeout: {e}"))?; + stream.set_write_timeout(Some(timeouts.write)).map_err(|e| format!("Cannot set write timeout: {e}"))?; + + let mut request = format!("{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nX-Cortex-Request: true\r\n"); if !auth_token.is_empty() { request.push_str(&format!("Authorization: Bearer {auth_token}\r\n")); } @@ -108,9 +89,7 @@ pub fn send_cortex_request_with_port( request.push_str(payload); } - stream - .write_all(request.as_bytes()) - .map_err(|e| format!("Write failed: {e}"))?; + stream.write_all(request.as_bytes()).map_err(|e| format!("Write failed: {e}"))?; let mut response = Vec::new(); if let Err(err) = stream.read_to_end(&mut response) { @@ -131,17 +110,9 @@ pub fn send_cortex_request_with_port( }); // Check for chunked transfer encoding - let body_bytes = if chunked { - decode_chunked_bytes(body)? - } else { - body.to_vec() - }; - let body_text = String::from_utf8(body_bytes) - .map_err(|e| format!("Response body is not valid UTF-8: {e}"))?; - Ok(FetchCortexResponse { - status, - body: body_text, - }) + let body_bytes = if chunked { decode_chunked_bytes(body)? } else { body.to_vec() }; + let body_text = String::from_utf8(body_bytes).map_err(|e| format!("Response body is not valid UTF-8: {e}"))?; + Ok(FetchCortexResponse { status, body: body_text }) } else { Err("Invalid HTTP response".to_string()) } @@ -151,22 +122,13 @@ fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { if needle.is_empty() || haystack.len() < needle.len() { return None; } - haystack - .windows(needle.len()) - .position(|window| window == needle) + haystack.windows(needle.len()).position(|window| window == needle) } fn parse_status_code(headers: &str) -> Result { - let status_line = headers - .lines() - .next() - .ok_or_else(|| "Missing HTTP status line".to_string())?; - let code = status_line - .split_whitespace() - .nth(1) - .ok_or_else(|| format!("Invalid HTTP status line: {status_line}"))?; - code.parse::() - .map_err(|e| format!("Invalid HTTP status code '{code}': {e}")) + let status_line = headers.lines().next().ok_or_else(|| "Missing HTTP status line".to_string())?; + let code = status_line.split_whitespace().nth(1).ok_or_else(|| format!("Invalid HTTP status line: {status_line}"))?; + code.parse::().map_err(|e| format!("Invalid HTTP status code '{code}': {e}")) } fn decode_chunked_bytes(body: &[u8]) -> Result, String> { @@ -174,14 +136,10 @@ fn decode_chunked_bytes(body: &[u8]) -> Result, String> { let mut remaining = body; loop { - let line_end = find_bytes(remaining, b"\r\n").ok_or_else(|| { - "Invalid chunked encoding: missing chunk size line ending".to_string() - })?; - let size_line = std::str::from_utf8(&remaining[..line_end]) - .map_err(|e| format!("Invalid chunk size line UTF-8: {e}"))?; + let line_end = find_bytes(remaining, b"\r\n").ok_or_else(|| "Invalid chunked encoding: missing chunk size line ending".to_string())?; + let size_line = std::str::from_utf8(&remaining[..line_end]).map_err(|e| format!("Invalid chunk size line UTF-8: {e}"))?; let size_hex = size_line.split(';').next().unwrap_or("").trim(); - let size = usize::from_str_radix(size_hex, 16) - .map_err(|e| format!("Invalid chunk size '{size_hex}': {e}"))?; + let size = usize::from_str_radix(size_hex, 16).map_err(|e| format!("Invalid chunk size '{size_hex}': {e}"))?; let data_start = line_end + 2; if data_start > remaining.len() { diff --git a/desktop/cortex-control-center/src-tauri/src/cortex_http/tests.rs b/desktop/cortex-control-center/src-tauri/src/cortex_http/tests.rs index b0eb12bc..dc6bbb40 100644 --- a/desktop/cortex-control-center/src-tauri/src/cortex_http/tests.rs +++ b/desktop/cortex-control-center/src-tauri/src/cortex_http/tests.rs @@ -1,211 +1,156 @@ use super::{ - cortex_readiness_state, health_state_with_identity_fallback, is_cortex_health_response, - readiness_state_with_identity_fallback, should_use_partial_response_on_read_timeout, - validate_cortex_request_path, FetchCortexResponse, + cortex_readiness_state, health_state_with_identity_fallback, is_cortex_health_response, readiness_state_with_identity_fallback, + should_use_partial_response_on_read_timeout, validate_cortex_request_path, FetchCortexResponse, }; use crate::daemon::paths::ResolvedCortexPaths; use crate::daemon::shutdown::extract_error_detail; use std::path::PathBuf; - - #[test] - fn validate_cortex_request_path_rejects_absolute_urls_and_injection() { - assert!(validate_cortex_request_path("/health").is_ok()); - assert!(validate_cortex_request_path("/sessions?agent=foo").is_ok()); - assert!(validate_cortex_request_path("http://127.0.0.1:7437/sessions").is_err()); - assert!(validate_cortex_request_path("/bad path").is_err()); - assert!(validate_cortex_request_path("/bad\r\nInjected: true").is_err()); +#[test] +fn validate_cortex_request_path_rejects_absolute_urls_and_injection() { + assert!(validate_cortex_request_path("/health").is_ok()); + assert!(validate_cortex_request_path("/sessions?agent=foo").is_ok()); + assert!(validate_cortex_request_path("http://127.0.0.1:7437/sessions").is_err()); + assert!(validate_cortex_request_path("/bad path").is_err()); + assert!(validate_cortex_request_path("/bad\r\nInjected: true").is_err()); +} + +#[test] +fn partial_response_timeout_only_applies_when_bytes_exist() { + let timeout = std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out"); + let would_block = std::io::Error::new(std::io::ErrorKind::WouldBlock, "would block"); + let reset = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"); + + assert!(should_use_partial_response_on_read_timeout(&timeout, 8)); + assert!(should_use_partial_response_on_read_timeout(&would_block, 8)); + #[cfg(windows)] + { + let winsock_timeout = std::io::Error::from_raw_os_error(10060); + assert!(should_use_partial_response_on_read_timeout(&winsock_timeout, 8)); } - - - #[test] - fn partial_response_timeout_only_applies_when_bytes_exist() { - let timeout = std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out"); - let would_block = std::io::Error::new(std::io::ErrorKind::WouldBlock, "would block"); - let reset = std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"); - - assert!(should_use_partial_response_on_read_timeout(&timeout, 8)); - assert!(should_use_partial_response_on_read_timeout(&would_block, 8)); - #[cfg(windows)] - { - let winsock_timeout = std::io::Error::from_raw_os_error(10060); - assert!(should_use_partial_response_on_read_timeout( - &winsock_timeout, - 8 - )); - } - assert!(!should_use_partial_response_on_read_timeout(&timeout, 0)); - assert!(!should_use_partial_response_on_read_timeout(&reset, 8)); - } - - - #[test] - fn extract_error_detail_prefers_json_error_field() { - let detail = extract_error_detail("{\"error\":\"Unauthorized\"}").unwrap(); - assert_eq!(detail, "Unauthorized"); - } - - - #[test] - fn cortex_health_probe_accepts_healthy_response_shape() { - assert!(is_cortex_health_response( + assert!(!should_use_partial_response_on_read_timeout(&timeout, 0)); + assert!(!should_use_partial_response_on_read_timeout(&reset, 8)); +} + +#[test] +fn extract_error_detail_prefers_json_error_field() { + let detail = extract_error_detail("{\"error\":\"Unauthorized\"}").unwrap(); + assert_eq!(detail, "Unauthorized"); +} + +#[test] +fn cortex_health_probe_accepts_healthy_response_shape() { + assert!(is_cortex_health_response(200, r#"{"status":"ok","runtime":{"version":"0.5.0"},"stats":{"memories":1}}"#, None, None)); + assert!(is_cortex_health_response(200, r#"{"status":"degraded","runtime":{"version":"0.5.0"},"stats":{"memories":1}}"#, None, None)); +} + +#[test] +fn cortex_health_probe_rejects_non_cortex_responses() { + assert!(!is_cortex_health_response(200, "ok", None, None)); + assert!(!is_cortex_health_response(200, r#"{"status":"ok"}"#, None, None)); + assert!(!is_cortex_health_response(200, r#"{"status":"ok","runtime":{"version":"0.5.0"}}"#, None, None)); + assert!(!is_cortex_health_response(503, r#"{"status":"ok","runtime":{}}"#, None, None)); +} + +#[test] +fn cortex_readiness_probe_accepts_ready_and_starting_payloads() { + assert_eq!( + cortex_readiness_state( 200, - r#"{"status":"ok","runtime":{"version":"0.5.0"},"stats":{"memories":1}}"#, - None, - None - )); - assert!(is_cortex_health_response( - 200, - r#"{"status":"degraded","runtime":{"version":"0.5.0"},"stats":{"memories":1}}"#, - None, - None - )); - } - - - #[test] - fn cortex_health_probe_rejects_non_cortex_responses() { - assert!(!is_cortex_health_response( - 200, - "ok", - None, - None - )); - assert!(!is_cortex_health_response( - 200, - r#"{"status":"ok"}"#, - None, - None - )); - assert!(!is_cortex_health_response( - 200, - r#"{"status":"ok","runtime":{"version":"0.5.0"}}"#, - None, - None - )); - assert!(!is_cortex_health_response( - 503, - r#"{"status":"ok","runtime":{}}"#, - None, - None - )); - } - - - #[test] - fn cortex_readiness_probe_accepts_ready_and_starting_payloads() { - assert_eq!( - cortex_readiness_state( - 200, - r#"{"status":"ready","ready":true,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, - Some(7437), - None - ), - Some(true) - ); - assert_eq!( - cortex_readiness_state( - 503, - r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, - Some(7437), - None - ), - Some(false) - ); - } - - - #[test] - fn cortex_readiness_probe_rejects_invalid_payloads() { - assert_eq!( - cortex_readiness_state( - 200, - r#"{"status":"ready","runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, - Some(7437), - None - ), - None - ); - assert_eq!( - cortex_readiness_state( - 500, - r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, - Some(7437), - None - ), - None - ); - } - - - #[test] - fn cortex_health_probe_rejects_identity_mismatch() { - let expected = ResolvedCortexPaths { - home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), - token: Some(PathBuf::from( - "C:/cortex-test/testuser/.cortex/cortex.token", - )), - db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), - pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), - port: Some(7437), - bind: Some("127.0.0.1".to_string()), - }; - assert!(!is_cortex_health_response( - 200, - r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/cortex-test/testuser/.cortex/cortex.db","pid_path":"C:/cortex-test/testuser/.cortex/cortex.pid"},"stats":{"home":"C:/cortex-test/testuser/.cortex","memories":1}}"#, - Some(7437), - Some(&expected) - )); - assert!(is_cortex_health_response( - 200, - r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/cortex-test/testuser/.cortex/cortex.token","db_path":"C:/cortex-test/testuser/.cortex/cortex.db","pid_path":"C:/cortex-test/testuser/.cortex/cortex.pid"},"stats":{"home":"C:/cortex-test/testuser/.cortex","memories":1}}"#, + r#"{"status":"ready","ready":true,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, Some(7437), - Some(&expected) - )); - } - - - #[test] - fn readiness_identity_fallback_classifies_starting_payload_on_path_mismatch() { - let expected = ResolvedCortexPaths { - home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), - token: Some(PathBuf::from( - "C:/cortex-test/testuser/.cortex/cortex.token", - )), - db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), - pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), - port: Some(7437), - bind: Some("127.0.0.1".to_string()), - }; - let (state, mismatch) = readiness_state_with_identity_fallback( + None + ), + Some(true) + ); + assert_eq!( + cortex_readiness_state( 503, - r#"{"status":"starting","ready":false,"runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/other/cortex.db","pid_path":"C:/other/cortex.pid"},"stats":{"home":"C:/other","memories":1}}"#, + r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, Some(7437), - Some(&expected), - ); - assert_eq!(state, Some(false)); - assert!(mismatch); - } - - - #[test] - fn health_identity_fallback_detects_reachable_payload_on_path_mismatch() { - let expected = ResolvedCortexPaths { - home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), - token: Some(PathBuf::from( - "C:/cortex-test/testuser/.cortex/cortex.token", - )), - db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), - pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), - port: Some(7437), - bind: Some("127.0.0.1".to_string()), - }; - let (healthy, mismatch) = health_state_with_identity_fallback( - 200, - r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/other/cortex.db","pid_path":"C:/other/cortex.pid"},"stats":{"home":"C:/other","memories":1}}"#, + None + ), + Some(false) + ); +} + +#[test] +fn cortex_readiness_probe_rejects_invalid_payloads() { + assert_eq!( + cortex_readiness_state(200, r#"{"status":"ready","runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, Some(7437), None), + None + ); + assert_eq!( + cortex_readiness_state( + 500, + r#"{"status":"starting","ready":false,"runtime":{"port":7437},"stats":{"home":"C:/cortex-test/testuser/.cortex"}}"#, Some(7437), - Some(&expected), - ); - assert!(healthy); - assert!(mismatch); - } + None + ), + None + ); +} + +#[test] +fn cortex_health_probe_rejects_identity_mismatch() { + let expected = ResolvedCortexPaths { + home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), + token: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.token")), + db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), + pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), + port: Some(7437), + bind: Some("127.0.0.1".to_string()), + }; + assert!(!is_cortex_health_response( + 200, + r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/cortex-test/testuser/.cortex/cortex.db","pid_path":"C:/cortex-test/testuser/.cortex/cortex.pid"},"stats":{"home":"C:/cortex-test/testuser/.cortex","memories":1}}"#, + Some(7437), + Some(&expected) + )); + assert!(is_cortex_health_response( + 200, + r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/cortex-test/testuser/.cortex/cortex.token","db_path":"C:/cortex-test/testuser/.cortex/cortex.db","pid_path":"C:/cortex-test/testuser/.cortex/cortex.pid"},"stats":{"home":"C:/cortex-test/testuser/.cortex","memories":1}}"#, + Some(7437), + Some(&expected) + )); +} + +#[test] +fn readiness_identity_fallback_classifies_starting_payload_on_path_mismatch() { + let expected = ResolvedCortexPaths { + home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), + token: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.token")), + db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), + pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), + port: Some(7437), + bind: Some("127.0.0.1".to_string()), + }; + let (state, mismatch) = readiness_state_with_identity_fallback( + 503, + r#"{"status":"starting","ready":false,"runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/other/cortex.db","pid_path":"C:/other/cortex.pid"},"stats":{"home":"C:/other","memories":1}}"#, + Some(7437), + Some(&expected), + ); + assert_eq!(state, Some(false)); + assert!(mismatch); +} + +#[test] +fn health_identity_fallback_detects_reachable_payload_on_path_mismatch() { + let expected = ResolvedCortexPaths { + home: Some(PathBuf::from("C:/cortex-test/testuser/.cortex")), + token: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.token")), + db: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.db")), + pid: Some(PathBuf::from("C:/cortex-test/testuser/.cortex/cortex.pid")), + port: Some(7437), + bind: Some("127.0.0.1".to_string()), + }; + let (healthy, mismatch) = health_state_with_identity_fallback( + 200, + r#"{"status":"ok","runtime":{"port":7437,"token_path":"C:/other/cortex.token","db_path":"C:/other/cortex.db","pid_path":"C:/other/cortex.pid"},"stats":{"home":"C:/other","memories":1}}"#, + Some(7437), + Some(&expected), + ); + assert!(healthy); + assert!(mismatch); +} diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/paths.rs b/desktop/cortex-control-center/src-tauri/src/daemon/paths.rs index cd856565..5e7c717d 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/paths.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/paths.rs @@ -28,15 +28,11 @@ pub fn default_cortex_dir() -> Result { Ok(cortex_home()?.join(".cortex")) } pub(crate) fn token_path() -> Result { - resolved_cortex_paths() - .token - .ok_or_else(|| "Could not resolve Cortex token path".to_string()) + resolved_cortex_paths().token.ok_or_else(|| "Could not resolve Cortex token path".to_string()) } pub fn cortex_db_path() -> Result { - resolved_cortex_paths() - .db - .ok_or_else(|| "Could not resolve Cortex database path".to_string()) + resolved_cortex_paths().db.ok_or_else(|| "Could not resolve Cortex database path".to_string()) } pub fn daemon_port() -> u16 { @@ -51,9 +47,7 @@ fn cortex_binary_name() -> &'static str { } fn normalized_path_for_guard(path: &Path) -> String { - path.to_string_lossy() - .replace('\\', "/") - .to_ascii_lowercase() + path.to_string_lossy().replace('\\', "/").to_ascii_lowercase() } fn path_is_under_root(path: &Path, root: &Path) -> bool { @@ -62,8 +56,7 @@ fn path_is_under_root(path: &Path, root: &Path) -> bool { if !normalized_root.ends_with('/') { normalized_root.push('/'); } - normalized_path == normalized_root.trim_end_matches('/') - || normalized_path.starts_with(&normalized_root) + normalized_path == normalized_root.trim_end_matches('/') || normalized_path.starts_with(&normalized_root) } fn is_allowed_isolated_target_dir(segment: &str) -> bool { @@ -105,31 +98,18 @@ fn is_non_runtime_test_artifact_path(path: &Path) -> bool { } fn is_shared_workspace_debug_runtime_path(path: &Path) -> bool { - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); + let file_name = path.file_name().and_then(|name| name.to_str()).unwrap_or_default().to_ascii_lowercase(); if file_name != cortex_binary_name().to_ascii_lowercase() { return false; } - let segments: Vec = path - .components() - .map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()) - .collect(); - segments - .windows(3) - .any(|window| window == ["daemon-rs", "target", "debug"]) + let segments: Vec = path.components().map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()).collect(); + segments.windows(3).any(|window| window == ["daemon-rs", "target", "debug"]) } pub fn is_disallowed_daemon_binary_path(path: &Path) -> bool { let normalized = normalized_path_for_guard(path); - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); + let file_name = path.file_name().and_then(|name| name.to_str()).unwrap_or_default().to_ascii_lowercase(); if file_name.starts_with("cortex-daemon-run") { return true; @@ -151,25 +131,14 @@ pub fn is_disallowed_daemon_binary_path(path: &Path) -> bool { if let Ok(tmp) = std::env::var("TMP") { temp_roots.push(PathBuf::from(tmp)); } - temp_roots - .iter() - .any(|root| !root.as_os_str().is_empty() && path_is_under_root(path, root)) + temp_roots.iter().any(|root| !root.as_os_str().is_empty() && path_is_under_root(path, root)) } pub fn workspace_binary_candidates(home: &Path, prefer_debug: bool) -> Vec { let daemon_root = home.join("cortex").join("daemon-rs"); - let release_path = daemon_root - .join("target") - .join("release") - .join(cortex_binary_name()); - let isolated_release_path = daemon_root - .join(RELEASE_DAEMON_TARGET_DIR) - .join("release") - .join(cortex_binary_name()); - let isolated_debug_path = daemon_root - .join(DEV_DAEMON_TARGET_DIR) - .join("debug") - .join(cortex_binary_name()); + let release_path = daemon_root.join("target").join("release").join(cortex_binary_name()); + let isolated_release_path = daemon_root.join(RELEASE_DAEMON_TARGET_DIR).join("release").join(cortex_binary_name()); + let isolated_debug_path = daemon_root.join(DEV_DAEMON_TARGET_DIR).join("debug").join(cortex_binary_name()); if prefer_debug { vec![isolated_debug_path, isolated_release_path, release_path] @@ -192,11 +161,7 @@ fn resolve_binary_on_path(binary_name: &str) -> Option { .filter_map(|line| { let candidate = PathBuf::from(line); if is_disallowed_daemon_binary_path(&candidate) { - log_startup_path( - "resolve-binary-on-path", - "reject-disallowed", - &candidate.display().to_string(), - ); + log_startup_path("resolve-binary-on-path", "reject-disallowed", &candidate.display().to_string()); None } else { Some(candidate) @@ -219,56 +184,32 @@ fn path_binary_fallback_enabled() -> bool { } pub fn service_ensure_fallback_enabled() -> bool { - path_binary_fallback_enabled_from_value( - std::env::var(SERVICE_ENSURE_FALLBACK_ENV).ok().as_deref(), - ) + path_binary_fallback_enabled_from_value(std::env::var(SERVICE_ENSURE_FALLBACK_ENV).ok().as_deref()) } fn parse_paths_json(output: &[u8]) -> Result { - let json: serde_json::Value = serde_json::from_slice(output) - .map_err(|err| format!("Invalid JSON from `cortex paths --json`: {err}"))?; + let json: serde_json::Value = serde_json::from_slice(output).map_err(|err| format!("Invalid JSON from `cortex paths --json`: {err}"))?; let port = json .get("port") .and_then(|value| value.as_u64()) - .map(|value| { - u16::try_from(value).map_err(|err| format!("Port value out of range ({value}): {err}")) - }) + .map(|value| u16::try_from(value).map_err(|err| format!("Port value out of range ({value}): {err}"))) .transpose()?; Ok(ResolvedCortexPaths { - home: json - .get("home") - .and_then(|value| value.as_str()) - .map(PathBuf::from), - token: json - .get("token") - .and_then(|value| value.as_str()) - .map(PathBuf::from), - db: json - .get("db") - .and_then(|value| value.as_str()) - .map(PathBuf::from), - pid: json - .get("pid") - .and_then(|value| value.as_str()) - .map(PathBuf::from), + home: json.get("home").and_then(|value| value.as_str()).map(PathBuf::from), + token: json.get("token").and_then(|value| value.as_str()).map(PathBuf::from), + db: json.get("db").and_then(|value| value.as_str()).map(PathBuf::from), + pid: json.get("pid").and_then(|value| value.as_str()).map(PathBuf::from), port, - bind: json - .get("bind") - .and_then(|value| value.as_str()) - .map(|value| value.to_string()), + bind: json.get("bind").and_then(|value| value.as_str()).map(|value| value.to_string()), }) } -fn resolve_paths_with_binary( - binary: impl AsRef, -) -> Result, String> { +fn resolve_paths_with_binary(binary: impl AsRef) -> Result, String> { let mut command = Command::new(binary); command.args(["paths", "--json"]); apply_hidden_process_flags(&mut command); - let output = command - .output() - .map_err(|err| format!("Failed to execute `cortex paths --json`: {err}"))?; + let output = command.output().map_err(|err| format!("Failed to execute `cortex paths --json`: {err}"))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); if stderr.is_empty() { @@ -280,10 +221,7 @@ fn resolve_paths_with_binary( } fn fallback_cortex_paths() -> ResolvedCortexPaths { - let cortex_dir = env::var("CORTEX_HOME") - .ok() - .map(PathBuf::from) - .or_else(|| default_cortex_dir().ok()); + let cortex_dir = env::var("CORTEX_HOME").ok().map(PathBuf::from).or_else(|| default_cortex_dir().ok()); let port = match env::var("CORTEX_PORT") { Ok(value) => match value.parse::() { @@ -306,11 +244,7 @@ fn fallback_cortex_paths() -> ResolvedCortexPaths { db: cortex_dir.as_ref().map(|dir| dir.join("cortex.db")), pid: cortex_dir.as_ref().map(|dir| dir.join("cortex.pid")), port, - bind: env::var("CORTEX_BIND") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .or_else(|| Some("127.0.0.1".to_string())), + bind: env::var("CORTEX_BIND").ok().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()).or_else(|| Some("127.0.0.1".to_string())), } } @@ -341,9 +275,7 @@ fn resolve_daemon_port() -> u16 { } pub fn log_startup_path(context: &str, decision: &str, detail: &str) { - eprintln!( - "[cortex-control-center] startup-path context={context} decision={decision} detail={detail}" - ); + eprintln!("[cortex-control-center] startup-path context={context} decision={decision} detail={detail}"); } pub fn installed_plugin_binary_path(home: &Path) -> PathBuf { @@ -352,27 +284,20 @@ pub fn installed_plugin_binary_path(home: &Path) -> PathBuf { pub fn copy_if_changed(src: &Path, dest: &Path) -> Result<(), String> { let needs_copy = match fs::read(dest) { - Ok(existing) => { - existing != fs::read(src).map_err(|e| format!("read {}: {e}", src.display()))? - } + Ok(existing) => existing != fs::read(src).map_err(|e| format!("read {}: {e}", src.display()))?, Err(err) if err.kind() == std::io::ErrorKind::NotFound => true, Err(err) => return Err(format!("read {}: {err}", dest.display())), }; if needs_copy { - fs::copy(src, dest) - .map_err(|e| format!("copy {} -> {}: {e}", src.display(), dest.display()))?; + fs::copy(src, dest).map_err(|e| format!("copy {} -> {}: {e}", src.display(), dest.display()))?; } Ok(()) } pub fn find_cortex_binary() -> Option { - let sidecar_candidate = env::current_exe().ok().and_then(|exe| { - exe.parent() - .map(|dir| dir.join(cortex_binary_name())) - .filter(|path| path.exists()) - }); + let sidecar_candidate = env::current_exe().ok().and_then(|exe| exe.parent().map(|dir| dir.join(cortex_binary_name())).filter(|path| path.exists())); if let Ok(home) = cortex_home() { let plugin_path = home.join(".cortex").join("bin").join(cortex_binary_name()); @@ -387,11 +312,7 @@ pub fn find_cortex_binary() -> Option { continue; } if is_disallowed_daemon_binary_path(&candidate) { - log_startup_path( - "find-cortex-binary", - "reject-disallowed", - &candidate.display().to_string(), - ); + log_startup_path("find-cortex-binary", "reject-disallowed", &candidate.display().to_string()); continue; } return Some(candidate); @@ -413,20 +334,14 @@ pub fn find_cortex_binary() -> Option { continue; } if is_disallowed_daemon_binary_path(&candidate) { - log_startup_path( - "find-cortex-binary", - "reject-disallowed", - &candidate.display().to_string(), - ); + log_startup_path("find-cortex-binary", "reject-disallowed", &candidate.display().to_string()); continue; } return Some(candidate); } } - if let Some(sidecar) = - sidecar_candidate.filter(|candidate| !is_disallowed_daemon_binary_path(candidate)) - { + if let Some(sidecar) = sidecar_candidate.filter(|candidate| !is_disallowed_daemon_binary_path(candidate)) { return Some(sidecar); } diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/shutdown.rs b/desktop/cortex-control-center/src-tauri/src/daemon/shutdown.rs index 52d18d2d..b8267e35 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/shutdown.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/shutdown.rs @@ -1,8 +1,5 @@ use crate::constants::*; -use crate::cortex_http::readiness::{ - is_cortex_reachable_with_port, read_auth_token_once, read_auth_token_with_retry_blocking, - wait_for_reachability_blocking, -}; +use crate::cortex_http::readiness::{is_cortex_reachable_with_port, read_auth_token_once, read_auth_token_with_retry_blocking, wait_for_reachability_blocking}; use crate::cortex_http::request::{send_cortex_request, FetchCortexResponse}; use crate::daemon::paths::{cortex_db_path, daemon_port}; use crate::daemon::state::DaemonState; @@ -16,8 +13,7 @@ pub fn shutdown_daemon(app: &tauri::AppHandle) { let port = daemon_port(); if managed && is_cortex_reachable_with_port(port, DAEMON_REACHABILITY_TIMEOUT_MS) { let _ = send_http_shutdown(); - let _ = - wait_for_reachability_blocking(port, false, Duration::from_millis(DAEMON_STOP_WAIT_MS)); + let _ = wait_for_reachability_blocking(port, false, Duration::from_millis(DAEMON_STOP_WAIT_MS)); } if managed { let _ = daemon_state.stop(); @@ -31,20 +27,9 @@ fn flush_cortex_db_on_shutdown() -> Result<(), String> { return Ok(()); } - let conn = Connection::open(&db_path).map_err(|err| { - format!( - "Failed to open DB for shutdown flush {}: {err}", - db_path.display() - ) - })?; - configure_shutdown_flush_connection(&conn).map_err(|err| { - format!( - "Failed to flush WAL on shutdown {}: {err}", - db_path.display() - ) - })?; - conn.close() - .map_err(|(_, err)| format!("Failed to close DB after shutdown flush: {err}"))?; + let conn = Connection::open(&db_path).map_err(|err| format!("Failed to open DB for shutdown flush {}: {err}", db_path.display()))?; + configure_shutdown_flush_connection(&conn).map_err(|err| format!("Failed to flush WAL on shutdown {}: {err}", db_path.display()))?; + conn.close().map_err(|(_, err)| format!("Failed to close DB after shutdown flush: {err}"))?; Ok(()) } @@ -63,24 +48,10 @@ pub fn configure_shutdown_flush_connection(conn: &Connection) -> rusqlite::Resul pub(crate) fn send_http_shutdown() -> Result<(), String> { let token = read_auth_token_once().unwrap_or_default(); let initial = send_cortex_request("POST", "/shutdown", &token, Some("{}"), None); - if matches!( - initial, - Ok(FetchCortexResponse { - status: 401 | 403, - .. - }) - ) { - if let Ok(refreshed_token) = - read_auth_token_with_retry_blocking(Duration::from_millis(AUTH_TOKEN_WAIT_MS)) - { + if matches!(initial, Ok(FetchCortexResponse { status: 401 | 403, .. })) { + if let Ok(refreshed_token) = read_auth_token_with_retry_blocking(Duration::from_millis(AUTH_TOKEN_WAIT_MS)) { if !refreshed_token.is_empty() && refreshed_token != token { - return interpret_shutdown_response(send_cortex_request( - "POST", - "/shutdown", - &refreshed_token, - Some("{}"), - None, - )); + return interpret_shutdown_response(send_cortex_request("POST", "/shutdown", &refreshed_token, Some("{}"), None)); } } } @@ -88,19 +59,14 @@ pub(crate) fn send_http_shutdown() -> Result<(), String> { interpret_shutdown_response(initial) } -pub fn interpret_shutdown_response( - response: Result, -) -> Result<(), String> { +pub fn interpret_shutdown_response(response: Result) -> Result<(), String> { match response { Ok(resp) if (200..300).contains(&resp.status) => Ok(()), - Ok(resp) if resp.status == 401 || resp.status == 403 => Err( - "Shutdown rejected by daemon authentication. Refresh the token or restart the daemon from Control Center." - .to_string(), - ), + Ok(resp) if resp.status == 401 || resp.status == 403 => { + Err("Shutdown rejected by daemon authentication. Refresh the token or restart the daemon from Control Center.".to_string()) + } Ok(resp) => { - let detail = extract_error_detail(&resp.body) - .map(|value| format!(" ({value})")) - .unwrap_or_default(); + let detail = extract_error_detail(&resp.body).map(|value| format!(" ({value})")).unwrap_or_default(); Err(format!("Daemon shutdown failed: HTTP {}{detail}", resp.status)) } Err(err) if err.starts_with("Cannot connect to daemon") => Ok(()), diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/spawn.rs b/desktop/cortex-control-center/src-tauri/src/daemon/spawn.rs index c89554b9..bcd93cb5 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/spawn.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/spawn.rs @@ -1,10 +1,12 @@ use crate::constants::*; -use crate::cortex_http::readiness::{is_cortex_reachable_with_port, probe_cortex_reachability_with_port, wait_for_reachability_blocking, CortexReachabilityProbe}; +use crate::cortex_http::readiness::{ + is_cortex_reachable_with_port, probe_cortex_reachability_with_port, wait_for_reachability_blocking, CortexReachabilityProbe, +}; use crate::daemon::paths::find_cortex_binary; use crate::daemon::process::apply_hidden_process_flags; use crate::daemon::state::DaemonState; -use std::time::Duration; use std::process::Command; +use std::time::Duration; fn command_output_summary(output: &std::process::Output) -> String { let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); @@ -29,45 +31,23 @@ pub fn try_service_ensure(port: u16) -> Result { let mut command = Command::new(&cortex_bin); command.args(["service", "ensure"]); apply_hidden_process_flags(&mut command); - let output = command.output().map_err(|err| { - format!( - "Failed to run `{}` service ensure: {err}", - cortex_bin.display() - ) - })?; + let output = command.output().map_err(|err| format!("Failed to run `{}` service ensure: {err}", cortex_bin.display()))?; if !output.status.success() { - return Err(format!( - "`cortex service ensure` failed: {}", - command_output_summary(&output) - )); + return Err(format!("`cortex service ensure` failed: {}", command_output_summary(&output))); } if is_cortex_reachable_with_port(port, DAEMON_REACHABILITY_TIMEOUT_MS) { return Ok(true); } - Ok(wait_for_reachability_blocking( - port, - true, - Duration::from_millis(SERVICE_ENSURE_WAIT_MS), - )) + Ok(wait_for_reachability_blocking(port, true, Duration::from_millis(SERVICE_ENSURE_WAIT_MS))) } -pub fn try_local_app_managed_ensure( - state: &DaemonState, - port: u16, -) -> Result { +pub fn try_local_app_managed_ensure(state: &DaemonState, port: u16) -> Result { state.ensure_local_daemon()?; - if wait_for_reachability_blocking( - port, - true, - Duration::from_millis(LOCAL_DAEMON_START_WAIT_MS), - ) { - return Ok(probe_cortex_reachability_with_port( - port, - DAEMON_REACHABILITY_TIMEOUT_MS, - )); + if wait_for_reachability_blocking(port, true, Duration::from_millis(LOCAL_DAEMON_START_WAIT_MS)) { + return Ok(probe_cortex_reachability_with_port(port, DAEMON_REACHABILITY_TIMEOUT_MS)); } // Re-probe once after the initial wait before declaring "still starting". @@ -85,11 +65,7 @@ pub fn local_probe_allows_starting_retry(probe: &CortexReachabilityProbe) -> boo probe.reachable || probe.starting } -pub fn local_app_managed_start_timeout_message( - state: &DaemonState, - pid: Option, - port: u16, -) -> String { +pub fn local_app_managed_start_timeout_message(state: &DaemonState, pid: Option, port: u16) -> String { let base = if let Some(pid) = pid { format!("App-managed daemon spawned (pid {pid}) but never became reachable on :{port}.") } else { @@ -98,8 +74,6 @@ pub fn local_app_managed_start_timeout_message( match state.stop() { Ok(()) => format!("{base} Control Center cleared the stale app-managed startup state."), - Err(err) => format!( - "{base} Control Center could not clear the stale app-managed startup state: {err}" - ), + Err(err) => format!("{base} Control Center could not clear the stale app-managed startup state: {err}"), } } diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/state.rs b/desktop/cortex-control-center/src-tauri/src/daemon/state.rs index 60bdbdf0..8e52641a 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/state.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/state.rs @@ -1,6 +1,6 @@ use crate::constants::{CONTROL_CENTER_LOCK_FILE, CONTROL_CENTER_OWNER_TAG, DEFAULT_DAEMON_PORT, LOCAL_DAEMON_LOCK_WAIT_SECS}; use crate::daemon::paths::{default_cortex_dir, is_disallowed_daemon_binary_path, resolved_cortex_paths}; -use crate::daemon::process::{apply_hidden_daemon_process_flags}; +use crate::daemon::process::apply_hidden_daemon_process_flags; use fs2::FileExt; use serde::Serialize; use std::fs; @@ -22,11 +22,7 @@ pub struct DaemonState { impl DaemonState { pub fn new(exe_path: Option) -> Self { - Self { - exe_path, - child: Mutex::new(None), - intentional_stop: AtomicBool::new(false), - } + Self { exe_path, child: Mutex::new(None), intentional_stop: AtomicBool::new(false) } } pub fn supervisor_paused(&self) -> bool { @@ -34,10 +30,7 @@ impl DaemonState { } pub fn status(&self) -> Result<(bool, Option), String> { - let mut child = self - .child - .lock() - .map_err(|_| "Failed to lock managed daemon state.".to_string())?; + let mut child = self.child.lock().map_err(|_| "Failed to lock managed daemon state.".to_string())?; let Some(managed_child) = child.as_mut() else { return Ok((false, None)); }; @@ -49,9 +42,7 @@ impl DaemonState { } Ok(None) => Ok((true, Some(managed_child.id()))), Err(err) => { - eprintln!( - "[cortex-control-center] failed to poll managed daemon process; clearing stale handle: {err}" - ); + eprintln!("[cortex-control-center] failed to poll managed daemon process; clearing stale handle: {err}"); *child = None; Ok((false, None)) } @@ -59,10 +50,7 @@ impl DaemonState { } pub fn ensure_local_daemon(&self) -> Result, String> { - let mut child = self - .child - .lock() - .map_err(|_| "Failed to lock managed daemon state.".to_string())?; + let mut child = self.child.lock().map_err(|_| "Failed to lock managed daemon state.".to_string())?; if let Some(existing) = child.as_mut() { match existing.try_wait() { Ok(Some(_)) => { @@ -72,31 +60,20 @@ impl DaemonState { return Ok(Some(existing.id())); } Err(err) => { - eprintln!( - "[cortex-control-center] failed to poll existing managed daemon before spawn; clearing stale handle: {err}" - ); + eprintln!("[cortex-control-center] failed to poll existing managed daemon before spawn; clearing stale handle: {err}"); *child = None; } } } - let exe_path = self.exe_path.clone().ok_or_else(|| { - "Could not resolve Cortex daemon binary for app-managed local mode.".to_string() - })?; + let exe_path = self.exe_path.clone().ok_or_else(|| "Could not resolve Cortex daemon binary for app-managed local mode.".to_string())?; if is_disallowed_daemon_binary_path(&exe_path) { - return Err(format!( - "Refusing to launch app-managed daemon from disallowed path: {}", - exe_path.display() - )); + return Err(format!("Refusing to launch app-managed daemon from disallowed path: {}", exe_path.display())); } let paths = resolved_cortex_paths(); - let home = paths.home.clone().ok_or_else(|| { - "Could not resolve Cortex home path for app-managed local mode.".to_string() - })?; - let db = paths.db.clone().ok_or_else(|| { - "Could not resolve Cortex database path for app-managed local mode.".to_string() - })?; + let home = paths.home.clone().ok_or_else(|| "Could not resolve Cortex home path for app-managed local mode.".to_string())?; + let db = paths.db.clone().ok_or_else(|| "Could not resolve Cortex database path for app-managed local mode.".to_string())?; // App-managed mode is intentionally local-only. We always bind to loopback // so Control Center can own daemon lifecycle without exposing it on LAN. let bind = "127.0.0.1".to_string(); @@ -117,21 +94,13 @@ impl DaemonState { .env("CORTEX_DAEMON_OWNER_SOURCE", "control-center-app") .env("CORTEX_DAEMON_OWNER_MODE", "app-managed-local") .env("CORTEX_WAIT_FOR_DAEMON_LOCK", "1") - .env( - "CORTEX_DAEMON_LOCK_WAIT_SECS", - LOCAL_DAEMON_LOCK_WAIT_SECS.to_string(), - ) + .env("CORTEX_DAEMON_LOCK_WAIT_SECS", LOCAL_DAEMON_LOCK_WAIT_SECS.to_string()) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); apply_hidden_daemon_process_flags(&mut command); - let spawned = command.spawn().map_err(|err| { - format!( - "Failed to spawn app-managed daemon from {}: {err}", - exe_path.display() - ) - })?; + let spawned = command.spawn().map_err(|err| format!("Failed to spawn app-managed daemon from {}: {err}", exe_path.display()))?; let pid = spawned.id(); *child = Some(spawned); // A successful spawn implicitly arms the supervisor: any later death @@ -144,10 +113,7 @@ impl DaemonState { // Pause the supervisor BEFORE killing the child so the watchdog does // not race in and spawn a new instance during teardown. self.intentional_stop.store(true, Ordering::SeqCst); - let mut child = self - .child - .lock() - .map_err(|_| "Failed to lock managed daemon state.".to_string())?; + let mut child = self.child.lock().map_err(|_| "Failed to lock managed daemon state.".to_string())?; if let Some(managed_child) = child.as_mut() { match managed_child.try_wait() { Ok(Some(_)) => { @@ -162,9 +128,7 @@ impl DaemonState { *child = None; } Err(err) => { - eprintln!( - "[cortex-control-center] failed to poll managed daemon process during stop; clearing stale handle: {err}" - ); + eprintln!("[cortex-control-center] failed to poll managed daemon process during stop; clearing stale handle: {err}"); *child = None; } } @@ -183,9 +147,7 @@ pub struct AppInstanceGuard { impl Default for LifecycleState { fn default() -> Self { - Self { - explicit_quit: AtomicBool::new(false), - } + Self { explicit_quit: AtomicBool::new(false) } } } @@ -203,8 +165,7 @@ impl AppInstanceGuard { pub fn acquire() -> Result, String> { let lock_path = control_center_lock_path()?; if let Some(parent) = lock_path.parent() { - fs::create_dir_all(parent) - .map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; + fs::create_dir_all(parent).map_err(|err| format!("Failed to create {}: {err}", parent.display()))?; } let mut lock_file = OpenOptions::new() .create(true) @@ -232,9 +193,7 @@ impl Drop for AppInstanceGuard { } fn control_center_lock_path() -> Result { - Ok(default_cortex_dir()? - .join("runtime") - .join(CONTROL_CENTER_LOCK_FILE)) + Ok(default_cortex_dir()?.join("runtime").join(CONTROL_CENTER_LOCK_FILE)) } #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -246,42 +205,21 @@ pub struct DaemonCommandResult { pub pid: Option, pub message: String, } -pub fn describe_daemon_state( - managed: bool, - reachable: bool, - starting: bool, - auth_token_ready: bool, - pid: Option, - port: u16, -) -> String { +pub fn describe_daemon_state(managed: bool, reachable: bool, starting: bool, auth_token_ready: bool, pid: Option, port: u16) -> String { if managed && reachable && auth_token_ready { format!("Cortex daemon running (pid {}).", pid.unwrap_or_default()) } else if managed && reachable { - format!( - "Cortex daemon running (pid {}) and reachable, waiting for auth token.", - pid.unwrap_or_default() - ) + format!("Cortex daemon running (pid {}) and reachable, waiting for auth token.", pid.unwrap_or_default()) } else if managed && starting { - format!( - "Cortex daemon running (pid {}) and still starting on :{}.", - pid.unwrap_or_default(), - port - ) + format!("Cortex daemon running (pid {}) and still starting on :{}.", pid.unwrap_or_default(), port) } else if managed { - format!( - "Cortex daemon running (pid {}) but not reachable on :{} yet.", - pid.unwrap_or_default(), - port - ) + format!("Cortex daemon running (pid {}) but not reachable on :{} yet.", pid.unwrap_or_default(), port) } else if reachable && auth_token_ready { "Cortex daemon reachable (external process).".to_string() } else if reachable { "Cortex daemon reachable (external process), waiting for auth token.".to_string() } else if starting { - format!( - "Cortex daemon is responding on :{} and still starting.", - port - ) + format!("Cortex daemon is responding on :{} and still starting.", port) } else { "Cortex daemon is offline.".to_string() } diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/supervisor.rs b/desktop/cortex-control-center/src-tauri/src/daemon/supervisor.rs index b12fff9e..ff03b33a 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/supervisor.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/supervisor.rs @@ -1,5 +1,5 @@ use crate::constants::*; -use crate::cortex_http::readiness::{probe_cortex_reachability_with_port}; +use crate::cortex_http::readiness::probe_cortex_reachability_with_port; use crate::daemon::paths::{daemon_port, log_startup_path, service_ensure_fallback_enabled}; use crate::daemon::spawn::{try_local_app_managed_ensure, try_service_ensure}; use crate::daemon::state::DaemonState; @@ -32,19 +32,13 @@ pub fn supervisor_tick(app_handle: &tauri::AppHandle, consecutive_failures: &Ato let attempt = consecutive_failures.fetch_add(1, Ordering::SeqCst); match try_local_app_managed_ensure(&daemon_state, port) { Ok(_) => { - log_startup_path( - "supervisor", - "respawn", - "daemon was unreachable; supervisor respawned via app-managed local mode", - ); + log_startup_path("supervisor", "respawn", "daemon was unreachable; supervisor respawned via app-managed local mode"); consecutive_failures.store(0, Ordering::SeqCst); } Err(err) => { // Throttle log noise: only log first failure and every 10th retry. if attempt == 0 || attempt % 10 == 0 { - eprintln!( - "[cortex-control-center] supervisor respawn attempt {attempt} failed: {err}" - ); + eprintln!("[cortex-control-center] supervisor respawn attempt {attempt} failed: {err}"); log_startup_path("supervisor", "respawn-failed", &err); } } @@ -97,16 +91,10 @@ pub fn bootstrap_daemon_on_startup(app_handle: &tauri::AppHandle) { if service_ensure_fallback_enabled() { match try_service_ensure(port) { Ok(true) => { - log_startup_path( - "setup", - "service-ensure", - "daemon started or validated via service ensure after app-managed fallback", - ); + log_startup_path("setup", "service-ensure", "daemon started or validated via service ensure after app-managed fallback"); } Ok(false) => { - eprintln!( - "[cortex-control-center] app-managed local start failed at startup and Windows service ensure was unavailable: {local_err}" - ); + eprintln!("[cortex-control-center] app-managed local start failed at startup and Windows service ensure was unavailable: {local_err}"); log_startup_path("setup", "blocked", "app-managed spawn failed"); } Err(service_err) => { @@ -117,9 +105,7 @@ pub fn bootstrap_daemon_on_startup(app_handle: &tauri::AppHandle) { } } } else { - eprintln!( - "[cortex-control-center] app-managed local start failed at startup (service fallback disabled): {local_err}" - ); + eprintln!("[cortex-control-center] app-managed local start failed at startup (service fallback disabled): {local_err}"); log_startup_path("setup", "blocked", "app-managed spawn failed"); } } diff --git a/desktop/cortex-control-center/src-tauri/src/daemon/tests.rs b/desktop/cortex-control-center/src-tauri/src/daemon/tests.rs index 7172f284..20081cfe 100644 --- a/desktop/cortex-control-center/src-tauri/src/daemon/tests.rs +++ b/desktop/cortex-control-center/src-tauri/src/daemon/tests.rs @@ -1,7 +1,4 @@ -use super::paths::{ - is_disallowed_daemon_binary_path, path_binary_fallback_enabled_from_value, - workspace_binary_candidates, -}; +use super::paths::{is_disallowed_daemon_binary_path, path_binary_fallback_enabled_from_value, workspace_binary_candidates}; use super::shutdown::{configure_shutdown_flush_connection, extract_error_detail, interpret_shutdown_response}; use super::spawn::{local_app_managed_start_timeout_message, local_probe_allows_starting_retry}; use super::state::{describe_daemon_state, DaemonState, LifecycleState}; @@ -30,208 +27,142 @@ fn cortex_binary_file_name() -> &'static str { } } - - fn spawn_test_sleep_process() -> Child { - #[cfg(windows)] - { - Command::new("cmd") - .args(["/C", "ping -n 30 127.0.0.1 > NUL"]) - .spawn() - .expect("spawn windows sleep surrogate") - } - - #[cfg(not(windows))] - { - Command::new("sleep") - .arg("30") - .spawn() - .expect("spawn unix sleep") - } - } - - - #[test] - fn shutdown_flush_sets_busy_timeout_before_checkpoint() { - let conn = Connection::open_in_memory().expect("open sqlite"); - configure_shutdown_flush_connection(&conn).expect("configure shutdown flush"); - let busy_timeout_ms: i64 = conn - .query_row("PRAGMA busy_timeout", [], |row| row.get(0)) - .expect("read busy_timeout"); - assert_eq!(busy_timeout_ms, SQLITE_BUSY_TIMEOUT_MS as i64); - } - - - #[test] - fn workspace_binary_candidates_prefers_debug_for_dev_builds() { - let candidates = workspace_binary_candidates(Path::new("C:/cortex-test/testuser"), true); - assert_eq!(candidates.len(), 3); - assert!(path_contains(&candidates[0], "target-control-center-dev/debug")); - assert!(path_contains( - &candidates[1], - "target-control-center-release/release", - )); - assert!(path_contains(&candidates[2], "target/release")); - assert!(candidates - .iter() - .all(|path| !path_contains(path, "target/debug"))); +fn spawn_test_sleep_process() -> Child { + #[cfg(windows)] + { + Command::new("cmd").args(["/C", "ping -n 30 127.0.0.1 > NUL"]).spawn().expect("spawn windows sleep surrogate") } - - #[test] - fn workspace_binary_candidates_prefers_release_for_packaged_builds() { - let candidates = workspace_binary_candidates(Path::new("C:/cortex-test/testuser"), false); - assert_eq!(candidates.len(), 3); - assert!(path_contains( - &candidates[0], - "target-control-center-release/release", - )); - assert!(path_contains(&candidates[1], "target/release")); - assert!(path_contains(&candidates[2], "target-control-center-dev/debug")); - assert!(candidates - .iter() - .all(|path| !path_contains(path, "target/debug"))); - } - - - #[test] - fn path_binary_fallback_requires_explicit_truthy_env_value() { - assert!(!path_binary_fallback_enabled_from_value(None)); - assert!(!path_binary_fallback_enabled_from_value(Some(""))); - assert!(!path_binary_fallback_enabled_from_value(Some("0"))); - assert!(!path_binary_fallback_enabled_from_value(Some("false"))); - assert!(path_binary_fallback_enabled_from_value(Some("1"))); - assert!(path_binary_fallback_enabled_from_value(Some("true"))); - assert!(path_binary_fallback_enabled_from_value(Some("Yes"))); - assert!(path_binary_fallback_enabled_from_value(Some("on"))); + #[cfg(not(windows))] + { + Command::new("sleep").arg("30").spawn().expect("spawn unix sleep") } +} +#[test] +fn shutdown_flush_sets_busy_timeout_before_checkpoint() { + let conn = Connection::open_in_memory().expect("open sqlite"); + configure_shutdown_flush_connection(&conn).expect("configure shutdown flush"); + let busy_timeout_ms: i64 = conn.query_row("PRAGMA busy_timeout", [], |row| row.get(0)).expect("read busy_timeout"); + assert_eq!(busy_timeout_ms, SQLITE_BUSY_TIMEOUT_MS as i64); +} - #[test] - fn disallowed_daemon_binary_path_blocks_wrappers_temp_and_test_artifacts() { - let wrapper = PathBuf::from( - "C:/repo/daemon-rs/target/debug/daemon-lifecycle-runtime/cortex-daemon-run.exe", - ); - assert!(is_disallowed_daemon_binary_path(&wrapper)); - - let wrapper_name_only = PathBuf::from("C:/repo/cortex-daemon-run"); - assert!(is_disallowed_daemon_binary_path(&wrapper_name_only)); +#[test] +fn workspace_binary_candidates_prefers_debug_for_dev_builds() { + let candidates = workspace_binary_candidates(Path::new("C:/cortex-test/testuser"), true); + assert_eq!(candidates.len(), 3); + assert!(path_contains(&candidates[0], "target-control-center-dev/debug")); + assert!(path_contains(&candidates[1], "target-control-center-release/release",)); + assert!(path_contains(&candidates[2], "target/release")); + assert!(candidates.iter().all(|path| !path_contains(path, "target/debug"))); +} - let temp_candidate = std::env::temp_dir().join("cortex").join("cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&temp_candidate)); +#[test] +fn workspace_binary_candidates_prefers_release_for_packaged_builds() { + let candidates = workspace_binary_candidates(Path::new("C:/cortex-test/testuser"), false); + assert_eq!(candidates.len(), 3); + assert!(path_contains(&candidates[0], "target-control-center-release/release",)); + assert!(path_contains(&candidates[1], "target/release")); + assert!(path_contains(&candidates[2], "target-control-center-dev/debug")); + assert!(candidates.iter().all(|path| !path_contains(path, "target/debug"))); +} - let target_tests = PathBuf::from("C:/repo/daemon-rs/target-tests/debug/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&target_tests)); +#[test] +fn path_binary_fallback_requires_explicit_truthy_env_value() { + assert!(!path_binary_fallback_enabled_from_value(None)); + assert!(!path_binary_fallback_enabled_from_value(Some(""))); + assert!(!path_binary_fallback_enabled_from_value(Some("0"))); + assert!(!path_binary_fallback_enabled_from_value(Some("false"))); + assert!(path_binary_fallback_enabled_from_value(Some("1"))); + assert!(path_binary_fallback_enabled_from_value(Some("true"))); + assert!(path_binary_fallback_enabled_from_value(Some("Yes"))); + assert!(path_binary_fallback_enabled_from_value(Some("on"))); +} - let target_test = PathBuf::from("C:/repo/daemon-rs/target-test/release/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&target_test)); +#[test] +fn disallowed_daemon_binary_path_blocks_wrappers_temp_and_test_artifacts() { + let wrapper = PathBuf::from("C:/repo/daemon-rs/target/debug/daemon-lifecycle-runtime/cortex-daemon-run.exe"); + assert!(is_disallowed_daemon_binary_path(&wrapper)); - let nextest = PathBuf::from("C:/repo/daemon-rs/target/nextest/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&nextest)); + let wrapper_name_only = PathBuf::from("C:/repo/cortex-daemon-run"); + assert!(is_disallowed_daemon_binary_path(&wrapper_name_only)); - let target_deps = PathBuf::from("C:/repo/daemon-rs/target/debug/deps/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&target_deps)); + let temp_candidate = std::env::temp_dir().join("cortex").join("cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&temp_candidate)); - let isolated_target_deps = - PathBuf::from("C:/repo/daemon-rs/target-control-center-dev/debug/deps/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&isolated_target_deps)); + let target_tests = PathBuf::from("C:/repo/daemon-rs/target-tests/debug/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&target_tests)); - let shared_workspace_runtime = PathBuf::from(format!( - "C:/repo/daemon-rs/target/debug/{}", - cortex_binary_file_name() - )); - assert!(is_disallowed_daemon_binary_path(&shared_workspace_runtime)); + let target_test = PathBuf::from("C:/repo/daemon-rs/target-test/release/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&target_test)); - let isolated_runtime = PathBuf::from(format!( - "C:/repo/daemon-rs/target-control-center-dev/debug/{}", - cortex_binary_file_name() - )); - assert!(!is_disallowed_daemon_binary_path(&isolated_runtime)); + let nextest = PathBuf::from("C:/repo/daemon-rs/target/nextest/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&nextest)); - let isolated_release_runtime = PathBuf::from(format!( - "C:/repo/daemon-rs/target-control-center-release/release/{}", - cortex_binary_file_name() - )); - assert!(!is_disallowed_daemon_binary_path(&isolated_release_runtime)); + let target_deps = PathBuf::from("C:/repo/daemon-rs/target/debug/deps/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&target_deps)); - let rtk_isolated = PathBuf::from("C:/repo/daemon-rs/target-rtk-isolated/debug/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&rtk_isolated)); + let isolated_target_deps = PathBuf::from("C:/repo/daemon-rs/target-control-center-dev/debug/deps/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&isolated_target_deps)); - let codex_test = PathBuf::from("C:/repo/daemon-rs/target-codex-test/debug/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&codex_test)); + let shared_workspace_runtime = PathBuf::from(format!("C:/repo/daemon-rs/target/debug/{}", cortex_binary_file_name())); + assert!(is_disallowed_daemon_binary_path(&shared_workspace_runtime)); - let target_build_script = PathBuf::from("C:/repo/daemon-rs/target/debug/build/cortex.exe"); - assert!(is_disallowed_daemon_binary_path(&target_build_script)); + let isolated_runtime = PathBuf::from(format!("C:/repo/daemon-rs/target-control-center-dev/debug/{}", cortex_binary_file_name())); + assert!(!is_disallowed_daemon_binary_path(&isolated_runtime)); - let safe = PathBuf::from("C:/cortex-test/testuser/.cortex/bin/cortex.exe"); - assert!(!is_disallowed_daemon_binary_path(&safe)); - } + let isolated_release_runtime = PathBuf::from(format!("C:/repo/daemon-rs/target-control-center-release/release/{}", cortex_binary_file_name())); + assert!(!is_disallowed_daemon_binary_path(&isolated_release_runtime)); + let rtk_isolated = PathBuf::from("C:/repo/daemon-rs/target-rtk-isolated/debug/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&rtk_isolated)); - #[test] - fn local_start_timeout_cleanup_clears_managed_child_state() { - let child = spawn_test_sleep_process(); - let pid = child.id(); - let state = DaemonState { - exe_path: None, - child: Mutex::new(Some(child)), - intentional_stop: AtomicBool::new(false), - }; + let codex_test = PathBuf::from("C:/repo/daemon-rs/target-codex-test/debug/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&codex_test)); - let (managed_before, _) = state.status().expect("initial status"); - assert!(managed_before); + let target_build_script = PathBuf::from("C:/repo/daemon-rs/target/debug/build/cortex.exe"); + assert!(is_disallowed_daemon_binary_path(&target_build_script)); - let message = local_app_managed_start_timeout_message(&state, Some(pid), 7437); - assert!(message.contains("cleared the stale app-managed startup state")); + let safe = PathBuf::from("C:/cortex-test/testuser/.cortex/bin/cortex.exe"); + assert!(!is_disallowed_daemon_binary_path(&safe)); +} - let (managed_after, pid_after) = state.status().expect("post-cleanup status"); - assert!(!managed_after); - assert_eq!(pid_after, None); - } +#[test] +fn local_start_timeout_cleanup_clears_managed_child_state() { + let child = spawn_test_sleep_process(); + let pid = child.id(); + let state = DaemonState { exe_path: None, child: Mutex::new(Some(child)), intentional_stop: AtomicBool::new(false) }; + let (managed_before, _) = state.status().expect("initial status"); + assert!(managed_before); - #[test] - fn daemon_state_description_includes_starting_state() { - let managed_message = describe_daemon_state(true, false, true, false, Some(42), 7437); - assert!(managed_message.contains("still starting")); + let message = local_app_managed_start_timeout_message(&state, Some(pid), 7437); + assert!(message.contains("cleared the stale app-managed startup state")); - let external_message = describe_daemon_state(false, false, true, false, None, 7437); - assert!(external_message.contains("still starting")); - } + let (managed_after, pid_after) = state.status().expect("post-cleanup status"); + assert!(!managed_after); + assert_eq!(pid_after, None); +} +#[test] +fn daemon_state_description_includes_starting_state() { + let managed_message = describe_daemon_state(true, false, true, false, Some(42), 7437); + assert!(managed_message.contains("still starting")); - #[test] - fn local_probe_retry_requires_reachability_or_starting_signal() { - assert!(local_probe_allows_starting_retry( - &CortexReachabilityProbe { - reachable: true, - starting: false, - identity_mismatch: false, - } - )); - assert!(local_probe_allows_starting_retry( - &CortexReachabilityProbe { - reachable: false, - starting: true, - identity_mismatch: false, - } - )); - assert!(!local_probe_allows_starting_retry( - &CortexReachabilityProbe { - reachable: false, - starting: false, - identity_mismatch: false, - } - )); - } + let external_message = describe_daemon_state(false, false, true, false, None, 7437); + assert!(external_message.contains("still starting")); +} +#[test] +fn local_probe_retry_requires_reachability_or_starting_signal() { + assert!(local_probe_allows_starting_retry(&CortexReachabilityProbe { reachable: true, starting: false, identity_mismatch: false })); + assert!(local_probe_allows_starting_retry(&CortexReachabilityProbe { reachable: false, starting: true, identity_mismatch: false })); + assert!(!local_probe_allows_starting_retry(&CortexReachabilityProbe { reachable: false, starting: false, identity_mismatch: false })); +} - #[test] - fn interpret_shutdown_response_surfaces_auth_rejection() { - let err = interpret_shutdown_response(Ok(FetchCortexResponse { - status: 401, - body: "{\"error\":\"Unauthorized\"}".to_string(), - })) - .unwrap_err(); +#[test] +fn interpret_shutdown_response_surfaces_auth_rejection() { + let err = interpret_shutdown_response(Ok(FetchCortexResponse { status: 401, body: "{\"error\":\"Unauthorized\"}".to_string() })).unwrap_err(); - assert!(err.contains("Refresh the token")); - } + assert!(err.contains("Refresh the token")); +} diff --git a/desktop/cortex-control-center/src-tauri/src/editor/mod.rs b/desktop/cortex-control-center/src-tauri/src/editor/mod.rs index 25a05604..82b0f6dc 100644 --- a/desktop/cortex-control-center/src-tauri/src/editor/mod.rs +++ b/desktop/cortex-control-center/src-tauri/src/editor/mod.rs @@ -43,10 +43,7 @@ pub fn editor_args(target: &EditorTarget) -> [&'static str; 3] { } pub fn editor_env_pairs(_target: &EditorTarget) -> [(&'static str, &'static str); 2] { - [ - ("CORTEX_APP_REQUIRED", "1"), - ("CORTEX_DAEMON_OWNER_LOCAL_SPAWN", "0"), - ] + [("CORTEX_APP_REQUIRED", "1"), ("CORTEX_DAEMON_OWNER_LOCAL_SPAWN", "0")] } fn editor_path_detected(path: &Path) -> bool { @@ -68,15 +65,9 @@ pub fn editor_config_path(target: &EditorTarget) -> PathBuf { pub fn cortex_mcp_registration(target: &EditorTarget, cortex_exe: &str) -> serde_json::Value { let mut env = serde_json::Map::new(); for (key, value) in editor_env_pairs(target) { - env.insert( - key.to_string(), - serde_json::Value::String(value.to_string()), - ); + env.insert(key.to_string(), serde_json::Value::String(value.to_string())); } - env.insert( - "CORTEX_APP_CLIENT".to_string(), - serde_json::Value::String(target.agent_name.to_string()), - ); + env.insert("CORTEX_APP_CLIENT".to_string(), serde_json::Value::String(target.agent_name.to_string())); serde_json::json!({ "command": cortex_exe, "args": editor_args(target), @@ -87,25 +78,17 @@ pub fn cortex_mcp_registration(target: &EditorTarget, cortex_exe: &str) -> serde fn claude_desktop_config_path(home: &Path) -> PathBuf { #[cfg(windows)] { - home.join("AppData") - .join("Roaming") - .join("Claude") - .join("claude_desktop_config.json") + home.join("AppData").join("Roaming").join("Claude").join("claude_desktop_config.json") } #[cfg(target_os = "macos")] { - home.join("Library") - .join("Application Support") - .join("Claude") - .join("claude_desktop_config.json") + home.join("Library").join("Application Support").join("Claude").join("claude_desktop_config.json") } #[cfg(all(unix, not(target_os = "macos")))] { - home.join(".config") - .join("Claude") - .join("claude_desktop_config.json") + home.join(".config").join("Claude").join("claude_desktop_config.json") } } @@ -163,24 +146,14 @@ pub fn editor_targets(home: &Path) -> Vec { } fn editor_detected(target: &EditorTarget) -> bool { - editor_path_detected(&target.config_path) - || target - .fallback_config_paths - .iter() - .any(|path| editor_path_detected(path)) + editor_path_detected(&target.config_path) || target.fallback_config_paths.iter().any(|path| editor_path_detected(path)) } fn editor_command_path(cortex_exe: Option<&str>) -> Option { cortex_exe.map(|path| path.to_string()) } -pub fn editor_detection( - target: &EditorTarget, - detected: bool, - registered: bool, - cortex_exe: Option<&str>, - message: String, -) -> EditorDetection { +pub fn editor_detection(target: &EditorTarget, detected: bool, registered: bool, cortex_exe: Option<&str>, message: String) -> EditorDetection { let config_path = editor_config_path(target); EditorDetection { id: target.id.into(), @@ -231,13 +204,7 @@ fn json_args_match(config: &serde_json::Value, expected_args: &[&str]) -> bool { config .get("args") .and_then(|value| value.as_array()) - .map(|args| { - args.len() == expected_args.len() - && args - .iter() - .zip(expected_args.iter()) - .all(|(value, expected)| value.as_str() == Some(*expected)) - }) + .map(|args| args.len() == expected_args.len() && args.iter().zip(expected_args.iter()).all(|(value, expected)| value.as_str() == Some(*expected))) .unwrap_or(false) } @@ -245,13 +212,8 @@ pub fn json_env_match(config: &serde_json::Value, target: &EditorTarget) -> bool let Some(env) = config.get("env").and_then(|value| value.as_object()) else { return false; }; - let policy_matches = editor_env_pairs(target) - .iter() - .all(|(key, expected)| env.get(*key).and_then(|value| value.as_str()) == Some(*expected)); - let client_match = env - .get("CORTEX_APP_CLIENT") - .and_then(|value| value.as_str()) - == Some(target.agent_name); + let policy_matches = editor_env_pairs(target).iter().all(|(key, expected)| env.get(*key).and_then(|value| value.as_str()) == Some(*expected)); + let client_match = env.get("CORTEX_APP_CLIENT").and_then(|value| value.as_str()) == Some(target.agent_name); policy_matches && client_match } @@ -259,13 +221,7 @@ fn toml_args_match(config: &toml::Value, expected_args: &[&str]) -> bool { config .get("args") .and_then(|value| value.as_array()) - .map(|args| { - args.len() == expected_args.len() - && args - .iter() - .zip(expected_args.iter()) - .all(|(value, expected)| value.as_str() == Some(*expected)) - }) + .map(|args| args.len() == expected_args.len() && args.iter().zip(expected_args.iter()).all(|(value, expected)| value.as_str() == Some(*expected))) .unwrap_or(false) } @@ -273,21 +229,12 @@ pub fn toml_env_match(config: &toml::Value, target: &EditorTarget) -> bool { let Some(env) = config.get("env").and_then(|value| value.as_table()) else { return false; }; - let policy_matches = editor_env_pairs(target) - .iter() - .all(|(key, expected)| env.get(*key).and_then(|value| value.as_str()) == Some(*expected)); - let client_match = env - .get("CORTEX_APP_CLIENT") - .and_then(|value| value.as_str()) - == Some(target.agent_name); + let policy_matches = editor_env_pairs(target).iter().all(|(key, expected)| env.get(*key).and_then(|value| value.as_str()) == Some(*expected)); + let client_match = env.get("CORTEX_APP_CLIENT").and_then(|value| value.as_str()) == Some(target.agent_name); policy_matches && client_match } -fn is_editor_registered_at_path( - target: &EditorTarget, - cortex_exe: &str, - config_path: &Path, -) -> Result { +fn is_editor_registered_at_path(target: &EditorTarget, cortex_exe: &str, config_path: &Path) -> Result { if !config_path.exists() { return Ok(false); } @@ -300,11 +247,7 @@ fn is_editor_registered_at_path( .get("mcpServers") .and_then(|value| value.get("cortex")) .map(|value| { - value - .get("command") - .and_then(|command| command.as_str()) - .map(|command| command == cortex_exe) - .unwrap_or(false) + value.get("command").and_then(|command| command.as_str()).map(|command| command == cortex_exe).unwrap_or(false) && json_args_match(value, &expected_args) && json_env_match(value, target) }) @@ -316,11 +259,7 @@ fn is_editor_registered_at_path( .get("mcp_servers") .and_then(|value| value.get("cortex")) .map(|value| { - value - .get("command") - .and_then(|command| command.as_str()) - .map(|command| command == cortex_exe) - .unwrap_or(false) + value.get("command").and_then(|command| command.as_str()).map(|command| command == cortex_exe).unwrap_or(false) && toml_args_match(value, &expected_args) && toml_env_match(value, target) }) @@ -334,19 +273,10 @@ fn is_editor_registered(target: &EditorTarget, cortex_exe: &str) -> Result Result { +fn register_json_editor(target: &EditorTarget, cortex_exe: &str) -> Result { let config_path = editor_config_path(target); if !editor_detected(target) { - return Ok(editor_detection( - target, - false, - false, - Some(cortex_exe), - format!("{} not detected ({})", target.name, config_path.display()), - )); + return Ok(editor_detection(target, false, false, Some(cortex_exe), format!("{} not detected ({})", target.name, config_path.display()))); } let mut config = read_json_config(&config_path)?; @@ -375,34 +305,17 @@ fn register_json_editor( let out = serde_json::to_string_pretty(&config).map_err(|e| e.to_string())?; fs::write(&config_path, out).map_err(|e| e.to_string())?; - Ok(editor_detection( - target, - true, - true, - Some(cortex_exe), - format!("{action} in {}", config_path.display()), - )) + Ok(editor_detection(target, true, true, Some(cortex_exe), format!("{action} in {}", config_path.display()))) } -fn register_toml_editor( - target: &EditorTarget, - cortex_exe: &str, -) -> Result { +fn register_toml_editor(target: &EditorTarget, cortex_exe: &str) -> Result { let config_path = editor_config_path(target); if !editor_detected(target) { - return Ok(editor_detection( - target, - false, - false, - Some(cortex_exe), - format!("{} not detected ({})", target.name, config_path.display()), - )); + return Ok(editor_detection(target, false, false, Some(cortex_exe), format!("{} not detected ({})", target.name, config_path.display()))); } let mut config = read_toml_config(&config_path)?; - let root = config - .as_table_mut() - .ok_or_else(|| format!("Invalid TOML config format in {}", config_path.display()))?; + let root = config.as_table_mut().ok_or_else(|| format!("Invalid TOML config format in {}", config_path.display()))?; let servers = root .entry("mcp_servers") .or_insert_with(|| toml::Value::Table(Default::default())) @@ -419,26 +332,13 @@ fn register_toml_editor( let args = editor_args(target); let mut server = toml::map::Map::new(); - server.insert( - "command".into(), - toml::Value::String(cortex_exe.to_string()), - ); - server.insert( - "args".into(), - toml::Value::Array( - args.into_iter() - .map(|value| toml::Value::String(value.to_string())) - .collect(), - ), - ); + server.insert("command".into(), toml::Value::String(cortex_exe.to_string())); + server.insert("args".into(), toml::Value::Array(args.into_iter().map(|value| toml::Value::String(value.to_string())).collect())); let mut env_table = toml::map::Map::new(); for (key, value) in editor_env_pairs(target) { env_table.insert(key.into(), toml::Value::String(value.to_string())); } - env_table.insert( - "CORTEX_APP_CLIENT".into(), - toml::Value::String(target.agent_name.to_string()), - ); + env_table.insert("CORTEX_APP_CLIENT".into(), toml::Value::String(target.agent_name.to_string())); server.insert("env".into(), toml::Value::Table(env_table)); servers.insert("cortex".into(), toml::Value::Table(server)); @@ -448,13 +348,7 @@ fn register_toml_editor( let out = toml::to_string_pretty(&config).map_err(|e| e.to_string())?; fs::write(&config_path, out).map_err(|e| e.to_string())?; - Ok(editor_detection( - target, - true, - true, - Some(cortex_exe), - format!("{action} in {}", config_path.display()), - )) + Ok(editor_detection(target, true, true, Some(cortex_exe), format!("{action} in {}", config_path.display()))) } pub fn register_editor(target: &EditorTarget, cortex_exe: &str) -> Result { @@ -464,7 +358,6 @@ pub fn register_editor(target: &EditorTarget, cortex_exe: &str) -> Result Result { use crate::daemon::paths::{copy_if_changed, installed_plugin_binary_path, is_disallowed_daemon_binary_path}; let home = cortex_home()?; @@ -472,10 +365,7 @@ pub fn ensure_editor_binary_path() -> Result { "Could not find cortex binary in sidecar directory, ~/.cortex/bin/, or ~/cortex/daemon-rs/{target-control-center-dev,target-control-center-release,target}/{debug,release}/".to_string() })?; if is_disallowed_daemon_binary_path(&source) { - return Err(format!( - "Refusing disallowed daemon binary source path for editor registration: {}", - source.display() - )); + return Err(format!("Refusing disallowed daemon binary source path for editor registration: {}", source.display())); } let installed = installed_plugin_binary_path(&home); @@ -494,10 +384,7 @@ pub fn setup_editors(editor_ids: Option>) -> Result>(); + let requested_ids = editor_ids.unwrap_or_default().into_iter().collect::>(); let use_selection = !requested_ids.is_empty(); let mut results = Vec::new(); @@ -512,13 +399,7 @@ pub fn setup_editors(editor_ids: Option>) -> Result results.push(result), - Err(err) => results.push(editor_detection( - &target, - detected, - false, - Some(&exe_str), - format!("Configuration failed: {err}"), - )), + Err(err) => results.push(editor_detection(&target, detected, false, Some(&exe_str), format!("Configuration failed: {err}"))), } } @@ -528,18 +409,12 @@ pub fn setup_editors(editor_ids: Option>) -> Result Result, String> { let home = cortex_home()?; let cortex_exe = cortex_exe_path(); - let cortex_exe_string = cortex_exe - .as_ref() - .map(|path| path.to_string_lossy().to_string()); + let cortex_exe_string = cortex_exe.as_ref().map(|path| path.to_string_lossy().to_string()); let mut results = Vec::new(); for target in editor_targets(&home) { let detected = editor_detected(&target); - let registered = if let Some(ref exe) = cortex_exe_string { - is_editor_registered(&target, exe).unwrap_or(false) - } else { - false - }; + let registered = if let Some(ref exe) = cortex_exe_string { is_editor_registered(&target, exe).unwrap_or(false) } else { false }; let message = if cortex_exe_string.is_none() { "cortex.exe not found -- build daemon first".into() } else if registered { @@ -550,13 +425,7 @@ pub fn detect_editors() -> Result, String> { format!("Not detected ({})", target.config_path.display()) }; - results.push(editor_detection( - &target, - detected, - registered, - cortex_exe_string.as_deref(), - message, - )); + results.push(editor_detection(&target, detected, registered, cortex_exe_string.as_deref(), message)); } Ok(results) diff --git a/desktop/cortex-control-center/src-tauri/src/editor/tests.rs b/desktop/cortex-control-center/src-tauri/src/editor/tests.rs index a7368dc9..796d1d6e 100644 --- a/desktop/cortex-control-center/src-tauri/src/editor/tests.rs +++ b/desktop/cortex-control-center/src-tauri/src/editor/tests.rs @@ -2,183 +2,116 @@ use super::*; use std::fs; use std::path::Path; - - #[test] - fn editor_registration_uses_explicit_agent_args() { - let home = Path::new("C:/cortex-test/testuser"); - let targets = editor_targets(home); - let cursor = targets.iter().find(|target| target.id == "cursor").unwrap(); - let claude = targets - .iter() - .find(|target| target.id == "claude-code") - .unwrap(); - - assert_eq!(editor_args(cursor), ["mcp", "--agent", "cursor"]); - assert_eq!(editor_args(claude), ["mcp", "--agent", "claude"]); - } - - - #[test] - fn editor_registration_includes_attach_only_env_contract() { - let home = Path::new("C:/cortex-test/testuser"); - let targets = editor_targets(home); - let codex = targets.iter().find(|target| target.id == "codex").unwrap(); - - let registration = - cortex_mcp_registration(codex, "C:/cortex-test/testuser/.cortex/bin/cortex.exe"); - let cortex_entry = registration - .as_object() - .expect("registration should be an object"); - let args = cortex_entry - .get("args") - .and_then(|value| value.as_array()) - .expect("args should exist"); - assert_eq!( - args.iter() - .filter_map(|value| value.as_str()) - .collect::>(), - vec!["mcp", "--agent", "codex"] - ); - - let env = cortex_entry - .get("env") - .and_then(|value| value.as_object()) - .expect("env should exist"); - assert_eq!( - env.get("CORTEX_APP_REQUIRED") - .and_then(|value| value.as_str()), - Some("1") - ); - assert_eq!( - env.get("CORTEX_DAEMON_OWNER_LOCAL_SPAWN") - .and_then(|value| value.as_str()), - Some("0") - ); - assert_eq!( - env.get("CORTEX_APP_CLIENT") - .and_then(|value| value.as_str()), - Some("codex") - ); - } - - - #[test] - fn registration_matchers_require_attach_only_env_contract() { - let home = Path::new("C:/cortex-test/testuser"); - let targets = editor_targets(home); - let cursor = targets.iter().find(|target| target.id == "cursor").unwrap(); - let codex = targets.iter().find(|target| target.id == "codex").unwrap(); - - let json_missing_env = serde_json::json!({ - "env": { - "CORTEX_APP_REQUIRED": "1" - } - }); - assert!(!json_env_match(&json_missing_env, cursor)); - - let json_ok = serde_json::json!({ - "env": { - "CORTEX_APP_REQUIRED": "1", - "CORTEX_DAEMON_OWNER_LOCAL_SPAWN": "0", - "CORTEX_APP_CLIENT": "cursor" - } - }); - assert!(json_env_match(&json_ok, cursor)); - - let toml_missing_env = toml::Value::Table( - [( - "env".to_string(), - toml::Value::Table( - [( - "CORTEX_APP_REQUIRED".to_string(), - toml::Value::String("1".to_string()), - )] - .into_iter() - .collect(), - ), - )] - .into_iter() - .collect(), - ); - assert!(!toml_env_match(&toml_missing_env, codex)); - - let toml_ok = toml::Value::Table( - [( - "env".to_string(), - toml::Value::Table( - [ - ( - "CORTEX_APP_REQUIRED".to_string(), - toml::Value::String("1".to_string()), - ), - ( - "CORTEX_DAEMON_OWNER_LOCAL_SPAWN".to_string(), - toml::Value::String("0".to_string()), - ), - ( - "CORTEX_APP_CLIENT".to_string(), - toml::Value::String("codex".to_string()), - ), - ] - .into_iter() - .collect(), - ), - )] +#[test] +fn editor_registration_uses_explicit_agent_args() { + let home = Path::new("C:/cortex-test/testuser"); + let targets = editor_targets(home); + let cursor = targets.iter().find(|target| target.id == "cursor").unwrap(); + let claude = targets.iter().find(|target| target.id == "claude-code").unwrap(); + + assert_eq!(editor_args(cursor), ["mcp", "--agent", "cursor"]); + assert_eq!(editor_args(claude), ["mcp", "--agent", "claude"]); +} + +#[test] +fn editor_registration_includes_attach_only_env_contract() { + let home = Path::new("C:/cortex-test/testuser"); + let targets = editor_targets(home); + let codex = targets.iter().find(|target| target.id == "codex").unwrap(); + + let registration = cortex_mcp_registration(codex, "C:/cortex-test/testuser/.cortex/bin/cortex.exe"); + let cortex_entry = registration.as_object().expect("registration should be an object"); + let args = cortex_entry.get("args").and_then(|value| value.as_array()).expect("args should exist"); + assert_eq!(args.iter().filter_map(|value| value.as_str()).collect::>(), vec!["mcp", "--agent", "codex"]); + + let env = cortex_entry.get("env").and_then(|value| value.as_object()).expect("env should exist"); + assert_eq!(env.get("CORTEX_APP_REQUIRED").and_then(|value| value.as_str()), Some("1")); + assert_eq!(env.get("CORTEX_DAEMON_OWNER_LOCAL_SPAWN").and_then(|value| value.as_str()), Some("0")); + assert_eq!(env.get("CORTEX_APP_CLIENT").and_then(|value| value.as_str()), Some("codex")); +} + +#[test] +fn registration_matchers_require_attach_only_env_contract() { + let home = Path::new("C:/cortex-test/testuser"); + let targets = editor_targets(home); + let cursor = targets.iter().find(|target| target.id == "cursor").unwrap(); + let codex = targets.iter().find(|target| target.id == "codex").unwrap(); + + let json_missing_env = serde_json::json!({ + "env": { + "CORTEX_APP_REQUIRED": "1" + } + }); + assert!(!json_env_match(&json_missing_env, cursor)); + + let json_ok = serde_json::json!({ + "env": { + "CORTEX_APP_REQUIRED": "1", + "CORTEX_DAEMON_OWNER_LOCAL_SPAWN": "0", + "CORTEX_APP_CLIENT": "cursor" + } + }); + assert!(json_env_match(&json_ok, cursor)); + + let toml_missing_env = toml::Value::Table( + [("env".to_string(), toml::Value::Table([("CORTEX_APP_REQUIRED".to_string(), toml::Value::String("1".to_string()))].into_iter().collect()))] .into_iter() .collect(), - ); - assert!(toml_env_match(&toml_ok, codex)); - } - - - #[test] - fn gemini_prefers_nested_mcp_config_when_present() { - let temp_root = std::env::temp_dir().join(format!( - "cortex_control_center_editor_test_{}", - std::process::id() - )); - let gemini_nested = temp_root.join(".gemini").join("settings").join("mcp.json"); - let gemini_legacy = temp_root.join(".gemini").join("settings.json"); - fs::create_dir_all(gemini_nested.parent().unwrap()).expect("create gemini settings dir"); - fs::write(&gemini_nested, "{}").expect("write nested gemini config"); - fs::write(&gemini_legacy, "{}").expect("write legacy gemini config"); - - let targets = editor_targets(&temp_root); - let gemini = targets.iter().find(|target| target.id == "gemini").unwrap(); - - assert_eq!(editor_config_path(gemini), gemini_nested); - - let _ = fs::remove_file(gemini_nested); - let _ = fs::remove_file(gemini_legacy); - let _ = fs::remove_dir_all(temp_root.join(".gemini")); - let _ = fs::remove_dir(temp_root); - } - - - #[test] - fn claude_desktop_uses_platform_specific_config_path() { - let home = Path::new("/tmp/cortex-home"); - let expected = if cfg!(windows) { - home.join("AppData") - .join("Roaming") - .join("Claude") - .join("claude_desktop_config.json") - } else if cfg!(target_os = "macos") { - home.join("Library") - .join("Application Support") - .join("Claude") - .join("claude_desktop_config.json") - } else { - home.join(".config") - .join("Claude") - .join("claude_desktop_config.json") - }; - - let targets = editor_targets(home); - let claude_desktop = targets - .iter() - .find(|target| target.id == "claude-desktop") - .unwrap(); - - assert_eq!(claude_desktop.config_path, expected); - } + ); + assert!(!toml_env_match(&toml_missing_env, codex)); + + let toml_ok = toml::Value::Table( + [( + "env".to_string(), + toml::Value::Table( + [ + ("CORTEX_APP_REQUIRED".to_string(), toml::Value::String("1".to_string())), + ("CORTEX_DAEMON_OWNER_LOCAL_SPAWN".to_string(), toml::Value::String("0".to_string())), + ("CORTEX_APP_CLIENT".to_string(), toml::Value::String("codex".to_string())), + ] + .into_iter() + .collect(), + ), + )] + .into_iter() + .collect(), + ); + assert!(toml_env_match(&toml_ok, codex)); +} + +#[test] +fn gemini_prefers_nested_mcp_config_when_present() { + let temp_root = std::env::temp_dir().join(format!("cortex_control_center_editor_test_{}", std::process::id())); + let gemini_nested = temp_root.join(".gemini").join("settings").join("mcp.json"); + let gemini_legacy = temp_root.join(".gemini").join("settings.json"); + fs::create_dir_all(gemini_nested.parent().unwrap()).expect("create gemini settings dir"); + fs::write(&gemini_nested, "{}").expect("write nested gemini config"); + fs::write(&gemini_legacy, "{}").expect("write legacy gemini config"); + + let targets = editor_targets(&temp_root); + let gemini = targets.iter().find(|target| target.id == "gemini").unwrap(); + + assert_eq!(editor_config_path(gemini), gemini_nested); + + let _ = fs::remove_file(gemini_nested); + let _ = fs::remove_file(gemini_legacy); + let _ = fs::remove_dir_all(temp_root.join(".gemini")); + let _ = fs::remove_dir(temp_root); +} + +#[test] +fn claude_desktop_uses_platform_specific_config_path() { + let home = Path::new("/tmp/cortex-home"); + let expected = if cfg!(windows) { + home.join("AppData").join("Roaming").join("Claude").join("claude_desktop_config.json") + } else if cfg!(target_os = "macos") { + home.join("Library").join("Application Support").join("Claude").join("claude_desktop_config.json") + } else { + home.join(".config").join("Claude").join("claude_desktop_config.json") + }; + + let targets = editor_targets(home); + let claude_desktop = targets.iter().find(|target| target.id == "claude-desktop").unwrap(); + + assert_eq!(claude_desktop.config_path, expected); +} diff --git a/desktop/cortex-control-center/src-tauri/src/main.rs b/desktop/cortex-control-center/src-tauri/src/main.rs index 185d85dc..99a769e4 100644 --- a/desktop/cortex-control-center/src-tauri/src/main.rs +++ b/desktop/cortex-control-center/src-tauri/src/main.rs @@ -11,14 +11,13 @@ mod editor; mod tray; use commands::{ - daemon_status, detect_editors, fetch_cortex, hide_to_tray, post_cortex, quit_app, - read_auth_token, read_budget_config, save_budget_config, setup_editors, start_daemon, - stop_daemon, write_dev_verification_report, + daemon_status, detect_editors, fetch_cortex, hide_to_tray, post_cortex, quit_app, read_auth_token, read_budget_config, save_budget_config, setup_editors, + start_daemon, stop_daemon, write_dev_verification_report, }; use constants::SUPERVISOR_TICK_MS; use daemon::paths::find_cortex_binary; use daemon::supervisor::{bootstrap_daemon_on_startup, supervisor_tick}; -use daemon::{AppInstanceGuard, DaemonState, LifecycleState, shutdown_daemon}; +use daemon::{shutdown_daemon, AppInstanceGuard, DaemonState, LifecycleState}; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use tauri::Manager; @@ -68,12 +67,7 @@ fn main() { supervisor_tick(&supervisor_handle, &consecutive_failures); } }) - .map_err(|err| { - std::io::Error::new( - err.kind(), - format!("failed to spawn cortex daemon supervisor thread: {err}"), - ) - })?; + .map_err(|err| std::io::Error::new(err.kind(), format!("failed to spawn cortex daemon supervisor thread: {err}")))?; Ok(()) }) diff --git a/desktop/cortex-control-center/src-tauri/src/tray.rs b/desktop/cortex-control-center/src-tauri/src/tray.rs index 9ee21549..25acbf72 100644 --- a/desktop/cortex-control-center/src-tauri/src/tray.rs +++ b/desktop/cortex-control-center/src-tauri/src/tray.rs @@ -29,12 +29,7 @@ pub fn request_app_quit(app: &tauri::AppHandle) { app.exit(0); } pub fn setup_tray(app: &tauri::App) -> tauri::Result<()> { - let tray_menu = MenuBuilder::new(app) - .text(TRAY_SHOW_ID, "Show") - .text(TRAY_HIDE_ID, "Hide / Minimize") - .separator() - .text(TRAY_QUIT_ID, "Quit") - .build()?; + let tray_menu = MenuBuilder::new(app).text(TRAY_SHOW_ID, "Show").text(TRAY_HIDE_ID, "Hide / Minimize").separator().text(TRAY_QUIT_ID, "Quit").build()?; let mut tray_builder = TrayIconBuilder::with_id(TRAY_ID) .menu(&tray_menu) @@ -47,14 +42,7 @@ pub fn setup_tray(app: &tauri::App) -> tauri::Result<()> { _ => {} }) .on_tray_icon_event(|tray, event| { - if matches!( - event, - TrayIconEvent::Click { - button: MouseButton::Left, - button_state: MouseButtonState::Up, - .. - } - ) { + if matches!(event, TrayIconEvent::Click { button: MouseButton::Left, button_state: MouseButtonState::Up, .. }) { show_main_window(tray.app_handle()); } }); diff --git a/desktop/cortex-control-center/src/App.jsx b/desktop/cortex-control-center/src/App.jsx index 870060a5..d5af701e 100644 --- a/desktop/cortex-control-center/src/App.jsx +++ b/desktop/cortex-control-center/src/App.jsx @@ -1 +1,2 @@ -export { App } from "./app/App.jsx"; +import { App } from "./app/App.jsx"; +export { App }; diff --git a/desktop/cortex-control-center/src/BrainVisualizer.jsx b/desktop/cortex-control-center/src/BrainVisualizer.jsx index 8fa777d0..33612235 100644 --- a/desktop/cortex-control-center/src/BrainVisualizer.jsx +++ b/desktop/cortex-control-center/src/BrainVisualizer.jsx @@ -1,88 +1,41 @@ +import React from "react"; import { Component, memo, useState } from "react"; import { AppIcon } from "./ui-icons.jsx"; import { BrainV2 } from "./brain-v2/index.jsx"; - -class GraphErrorBoundary extends Component { - constructor(props) { - super(props); - this.state = { hasError: false, error: null }; +class GraphErrorBoundary extends Component { constructor(props) { (super(props), (this.state = { hasError: !1, error: null })); } - static getDerivedStateFromError(error) { - return { hasError: true, error: error.message }; + static getDerivedStateFromError(error) { return { hasError: !0, error: error.message }; } - render() { - if (this.state.hasError) { - return this.props.fallback || ( -
-
-

3D renderer crashed: {this.state.error}

-

Showing 2D fallback instead.

-
- ); - } - return this.props.children; + render() { return this.state.hasError + ? this.props.fallback || React.createElement( "div", { className: "brain-loading" }, + React.createElement( "div", { className: "coming-icon" }, React.createElement(AppIcon, { name: "brain", size: 48 }), + ), React.createElement("p", null, "3D renderer crashed: ", this.state.error), + React.createElement("p", { className: "brain-fallback-reason" }, "Showing 2D fallback instead."), ) + : this.props.children; } } - -function hasWebGLSupport() { - if (typeof document === "undefined") return false; - try { - const canvas = document.createElement("canvas"); - return Boolean( - canvas.getContext("webgl2") - || canvas.getContext("webgl") - || canvas.getContext("experimental-webgl") - ); - } catch { - return false; +function hasWebGLSupport() { if (typeof document > "u") return !1; + try { const canvas = document.createElement("canvas"); + return !!(canvas.getContext("webgl2") || canvas.getContext("webgl") || canvas.getContext("experimental-webgl")); + } catch { return !1; } } - -function BrainVisualizerComponent({ - api = null, - cortexBase = "http://127.0.0.1:7437", - authToken = "", - active = true, - reducedMotion = false, -}) { - const [webglAvailable] = useState(() => hasWebGLSupport()); - - if (!webglAvailable) { - return ( -
-
- 2D fallback: WebGL unavailable -
-
-
-

WebGL is required for the Brain map.

-
-
- ); - } - - return ( -
-
-
- Neural topology - Cortex Brain Map -

Living constellation. Select satellites to inspect.

-
-
- - - -
- ); +function BrainVisualizerComponent({ api = null, cortexBase = "http://127.0.0.1:7437", authToken = "", + active = !0, reducedMotion = !1, }) { const [webglAvailable] = useState(() => hasWebGLSupport()); + return webglAvailable + ? React.createElement( "div", { className: "brain-container" }, React.createElement( + "div", { className: "brain-hud brain-hud-primary" }, React.createElement( "div", + { className: "brain-hud-copy" }, React.createElement("span", { className: "brain-mode" }, "Neural topology"), + React.createElement("strong", { className: "brain-title" }, "Cortex Brain Map"), + React.createElement("p", null, "Living constellation. Select satellites to inspect."), ), ), React.createElement( + GraphErrorBoundary, null, React.createElement(BrainV2, { api, cortexBase, authToken, active, reducedMotion, }), ), ) + : React.createElement( "div", { className: "brain-container brain-fallback-container" }, React.createElement( + "div", { className: "brain-hud brain-hud-fallback" }, + React.createElement("span", { className: "brain-fallback-reason" }, "2D fallback: WebGL unavailable"), ), React.createElement( "div", + { className: "brain-loading" }, React.createElement( "div", { className: "coming-icon" }, + React.createElement(AppIcon, { name: "brain", size: 48 }), ), React.createElement("p", null, "WebGL is required for the Brain map."), ), ); } - BrainVisualizerComponent.displayName = "BrainVisualizer"; -export const BrainVisualizer = memo(BrainVisualizerComponent); -export default BrainVisualizer; +const BrainVisualizer = memo(BrainVisualizerComponent); +var BrainVisualizer_default = BrainVisualizer; +export { BrainVisualizer, BrainVisualizer_default as default }; diff --git a/desktop/cortex-control-center/src/analytics-metrics.js b/desktop/cortex-control-center/src/analytics-metrics.js index 8dc43476..caa21e7c 100644 --- a/desktop/cortex-control-center/src/analytics-metrics.js +++ b/desktop/cortex-control-center/src/analytics-metrics.js @@ -1,80 +1,43 @@ -const DAY_MS = 24 * 60 * 60 * 1000; -const ISO_DAY_RE = /^\d{4}-\d{2}-\d{2}$/; - -function toIsoUtcDay(date) { - if (!(date instanceof Date) || Number.isNaN(date.getTime())) return ""; - const year = date.getUTCFullYear(); - const month = String(date.getUTCMonth() + 1).padStart(2, "0"); - const day = String(date.getUTCDate()).padStart(2, "0"); +const DAY_MS = 864e5, ISO_DAY_RE = /^\d{4}-\d{2}-\d{2}$/; +function toIsoUtcDay(date) { if (!(date instanceof Date) || Number.isNaN(date.getTime())) return ""; + const year = date.getUTCFullYear(), month = String(date.getUTCMonth() + 1).padStart(2, "0"), day = String(date.getUTCDate()).padStart(2, "0"); return `${year}-${month}-${day}`; } - -function parseIsoUtcDay(isoDay) { - if (!ISO_DAY_RE.test(String(isoDay || ""))) return null; +function parseIsoUtcDay(isoDay) { if (!ISO_DAY_RE.test(String(isoDay || ""))) return null; const parsed = new Date(`${isoDay}T00:00:00Z`); - if (Number.isNaN(parsed.getTime())) return null; - return parsed; + return Number.isNaN(parsed.getTime()) ? null : parsed; } - -function trailingIsoDays(windowDays, nowDate = new Date()) { - const safeWindow = Math.max(1, Math.floor(Number(windowDays) || 1)); - const now = nowDate instanceof Date && !Number.isNaN(nowDate.getTime()) ? nowDate : new Date(); - const endDay = parseIsoUtcDay(toIsoUtcDay(now)); - if (!endDay) return []; - return Array.from({ length: safeWindow }, (_, index) => { - const day = new Date(endDay.getTime() - (safeWindow - 1 - index) * DAY_MS); - return toIsoUtcDay(day); - }); +function trailingIsoDays(windowDays, nowDate = new Date()) { const safeWindow = Math.max(1, Math.floor(Number(windowDays) || 1)), + now = nowDate instanceof Date && !Number.isNaN(nowDate.getTime()) ? nowDate : new Date(), endDay = parseIsoUtcDay(toIsoUtcDay(now)); + return endDay + ? Array.from({ length: safeWindow }, (_, index) => { const day = new Date(endDay.getTime() - (safeWindow - 1 - index) * 864e5); + return toIsoUtcDay(day); + }) + : []; } - -function daysBetweenInclusive(startIsoDay, endIsoDay) { - const start = parseIsoUtcDay(startIsoDay); - const end = parseIsoUtcDay(endIsoDay); - if (!start || !end || start.getTime() > end.getTime()) return 0; - return Math.floor((end.getTime() - start.getTime()) / DAY_MS) + 1; +function daysBetweenInclusive(startIsoDay, endIsoDay) { const start = parseIsoUtcDay(startIsoDay), end = parseIsoUtcDay(endIsoDay); + return !start || !end || start.getTime() > end.getTime() + ? 0 + : Math.floor((end.getTime() - start.getTime()) / 864e5) + 1; } - -function normalizeBootRowsByDay(dailySeries) { - const rows = Array.isArray(dailySeries) ? dailySeries : []; - const byDay = new Map(); - for (const row of rows) { - const day = String(row?.date || ""); +function normalizeBootRowsByDay(dailySeries) { const rows = Array.isArray(dailySeries) ? dailySeries : [], byDay = new Map(); + for (const row of rows) { const day = String(row?.date || ""); if (!ISO_DAY_RE.test(day)) continue; const boots = Number(row?.boots || 0); - if (!Number.isFinite(boots)) continue; - byDay.set(day, (byDay.get(day) || 0) + boots); + Number.isFinite(boots) && byDay.set(day, (byDay.get(day) || 0) + boots); } return byDay; } - -export function summarizeBootThroughput(dailySeries, windowDays = 7, nowDate = new Date()) { - const safeWindow = Math.max(1, Math.floor(Number(windowDays) || 7)); - const byDay = normalizeBootRowsByDay(dailySeries); - const windowDaysIso = trailingIsoDays(safeWindow, nowDate); - const windowStart = windowDaysIso[0] || ""; - const windowEnd = windowDaysIso.at(-1) || ""; - const boots = windowDaysIso.reduce((sum, day) => sum + Number(byDay.get(day) || 0), 0); - const sortedObservedDays = [...byDay.keys()].sort(); - const firstObservedDay = sortedObservedDays[0] || ""; - +function summarizeBootThroughput(dailySeries, windowDays = 7, nowDate = new Date()) { const safeWindow = Math.max(1, Math.floor(Number(windowDays) || 7)), + byDay = normalizeBootRowsByDay(dailySeries), windowDaysIso = trailingIsoDays(safeWindow, nowDate), + windowStart = windowDaysIso[0] || "", windowEnd = windowDaysIso.at(-1) || "", + boots = windowDaysIso.reduce((sum, day) => sum + Number(byDay.get(day) || 0), 0), firstObservedDay = [...byDay.keys()].sort()[0] || ""; let daysRepresented = safeWindow; - if (!firstObservedDay) { - daysRepresented = 0; - } else if (windowStart && firstObservedDay > windowStart) { - daysRepresented = Math.min(safeWindow, daysBetweenInclusive(firstObservedDay, windowEnd)); - } - - const avgPerDay = daysRepresented > 0 - ? Math.round((boots / daysRepresented) * 10) / 10 - : 0; - - return { - windowDays: safeWindow, - daysRepresented, - isPartialHistory: daysRepresented > 0 && daysRepresented < safeWindow, - windowStart, - windowEnd, - boots, - avgPerDay, - }; + firstObservedDay + ? windowStart && firstObservedDay > windowStart && (daysRepresented = Math.min(safeWindow, daysBetweenInclusive(firstObservedDay, windowEnd))) + : (daysRepresented = 0); + const avgPerDay = daysRepresented > 0 ? Math.round((boots / daysRepresented) * 10) / 10 : 0; + return { windowDays: safeWindow, daysRepresented, isPartialHistory: daysRepresented > 0 && daysRepresented < safeWindow, + windowStart, windowEnd, boots, avgPerDay, }; } +export { summarizeBootThroughput }; diff --git a/desktop/cortex-control-center/src/analytics-projection.js b/desktop/cortex-control-center/src/analytics-projection.js index b812b5d2..660a5e6e 100644 --- a/desktop/cortex-control-center/src/analytics-projection.js +++ b/desktop/cortex-control-center/src/analytics-projection.js @@ -1,164 +1,80 @@ -function createSeededRng(seed) { - let state = seed >>> 0; - return () => { - state = (state + 0x6d2b79f5) >>> 0; +function createSeededRng(seed) { let state = seed >>> 0; + return () => { state = (state + 1831565813) >>> 0; let t = Math.imul(state ^ (state >>> 15), 1 | state); - t ^= t + Math.imul(t ^ (t >>> 7), 61 | t); - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + return ((t ^= t + Math.imul(t ^ (t >>> 7), 61 | t)), ((t ^ (t >>> 14)) >>> 0) / 4294967296); }; } - -function gaussianRandom(rng) { - let u = 0; - let v = 0; - while (u === 0) u = rng(); - while (v === 0) v = rng(); - return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); +function gaussianRandom(rng) { let u = 0, v = 0; + for (; u === 0;) u = rng(); + for (; v === 0;) v = rng(); + return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); } - -function percentileFromSorted(sorted, percentile) { - if (!sorted.length) return 0; - const index = (sorted.length - 1) * percentile; - const lower = Math.floor(index); - const upper = Math.ceil(index); +function percentileFromSorted(sorted, percentile) { if (!sorted.length) return 0; + const index = (sorted.length - 1) * percentile, lower = Math.floor(index), upper = Math.ceil(index); if (lower === upper) return sorted[lower]; const weight = index - lower; return sorted[lower] * (1 - weight) + sorted[upper] * weight; } - -function clampNumber(value, min, max) { - return Math.min(Math.max(value, min), max); +function clampNumber(value, min, max) { return Math.min(Math.max(value, min), max); } - -const ABSOLUTE_DAILY_BASIS_CAP = 1_000_000_000; -const ABSOLUTE_PROJECTED_GAIN_CAP = ABSOLUTE_DAILY_BASIS_CAP * 90 * 2; - -function projectionBasisFromSeries(dailySeries, cumulativeSeries) { - const dailyBasis = (Array.isArray(dailySeries) ? dailySeries : []) +const ABSOLUTE_DAILY_BASIS_CAP = 1e9, ABSOLUTE_PROJECTED_GAIN_CAP = ABSOLUTE_DAILY_BASIS_CAP * 90 * 2; +function projectionBasisFromSeries(dailySeries, cumulativeSeries) { const dailyBasis = (Array.isArray(dailySeries) ? dailySeries : []) .map((point) => Number(point?.saved || 0)) .filter((value) => Number.isFinite(value) && value > 0); - if (dailyBasis.length) return dailyBasis; - - return (Array.isArray(cumulativeSeries) ? cumulativeSeries : []) - .map((point) => Number(point?.savedDelta || 0)) - .filter((value) => Number.isFinite(value) && value > 0); + return dailyBasis.length + ? dailyBasis + : (Array.isArray(cumulativeSeries) ? cumulativeSeries : []) + .map((point) => Number(point?.savedDelta || 0)) + .filter((value) => Number.isFinite(value) && value > 0); } - -function sanitizeProjectionBasis(basis) { - if (!Array.isArray(basis) || basis.length < 2) return []; +function sanitizeProjectionBasis(basis) { if (!Array.isArray(basis) || basis.length < 2) return []; const finite = basis.filter((value) => Number.isFinite(value) && value > 0); if (finite.length < 2) return []; - - const sorted = [...finite].sort((left, right) => left - right); - const median = percentileFromSorted(sorted, 0.5); - const upperLimit = Math.min(Math.max(median * 40, 1), ABSOLUTE_DAILY_BASIS_CAP); - const lowerLimit = Math.max(median * 0.02, 1); + const sorted = [...finite].sort((left, right) => left - right), median = percentileFromSorted(sorted, 0.5), + upperLimit = Math.min(Math.max(median * 40, 1), ABSOLUTE_DAILY_BASIS_CAP), lowerLimit = Math.max(median * 0.02, 1); return finite.map((value) => clampNumber(value, lowerLimit, upperLimit)); } - -export function buildMonteCarloProjection(dailySeries, cumulativeSeries, horizonDays = 30, simulationCount = 180) { - const safeHorizonDays = Math.max(1, Math.min(90, Math.floor(Number(horizonDays) || 30))); - const safeSimulationCount = Math.max(20, Math.min(1000, Math.floor(Number(simulationCount) || 180))); - const basis = sanitizeProjectionBasis(projectionBasisFromSeries(dailySeries, cumulativeSeries)); +function buildMonteCarloProjection(dailySeries, cumulativeSeries, horizonDays = 30, simulationCount = 64) { + const safeHorizonDays = Math.max(1, Math.min(90, Math.floor(Number(horizonDays) || 30))), + safeSimulationCount = Math.max(20, Math.min(1e3, Math.floor(Number(simulationCount) || 64))), + basis = sanitizeProjectionBasis(projectionBasisFromSeries(dailySeries, cumulativeSeries)); if (basis.length < 2) return null; - - const recent = basis.slice(-14); - const recentAverage = recent.reduce((sum, value) => sum + value, 0) / recent.length; - const recentMedian = percentileFromSorted([...recent].sort((left, right) => left - right), 0.5); - const recentPeak = Math.max(...recent, 1); - const logReturns = []; - for (let index = 1; index < recent.length; index += 1) { - const previous = Math.max(recent[index - 1], 1); - const current = Math.max(recent[index], 1); + const recent = basis.slice(-14), recentAverage = recent.reduce((sum, value) => sum + value, 0) / recent.length, + recentMedian = percentileFromSorted( [...recent].sort((left, right) => left - right), 0.5, ), recentPeak = Math.max(...recent, 1), logReturns = []; + for (let index = 1; index < recent.length; index += 1) { const previous = Math.max(recent[index - 1], 1), current = Math.max(recent[index], 1); logReturns.push(clampNumber(Math.log(current / previous), -0.6, 0.6)); } - - const rawDrift = logReturns.length - ? logReturns.reduce((sum, value) => sum + value, 0) / logReturns.length - : 0.012; - const shortHistory = recent.length < 4; - const drift = clampNumber(rawDrift, -0.08, shortHistory ? 0.05 : 0.12); - const variance = logReturns.length - ? logReturns.reduce((sum, value) => sum + (value - rawDrift) ** 2, 0) / logReturns.length - : 0.05; - const volatilityFloor = shortHistory ? 0.06 : 0.08; - const volatilityCeiling = shortHistory ? 0.22 : 0.35; - const volatility = clampNumber(Math.max(Math.sqrt(variance), volatilityFloor), volatilityFloor, volatilityCeiling); - const lastDaily = Math.max(recent[recent.length - 1], 1); - const startTotal = Number( - cumulativeSeries?.at?.(-1)?.savedTotal - || cumulativeSeries?.at?.(-1)?.saved - || basis.reduce((sum, value) => sum + value, 0) - ); - // Keep deterministic seeding while avoiding precision collapse from massive totals. - const boundedSeedBase = Number.isFinite(startTotal) - ? Math.abs(startTotal % 1_000_000_000) - : 0; - const rng = createSeededRng(Math.round(boundedSeedBase + lastDaily + recent.length * 13)); - const meanReversionStrength = shortHistory ? 0.03 : 0.04; - const dailyCeiling = Math.min( - Math.max(recentPeak * 4, recentAverage * 6, recentMedian * 10, 1), - ABSOLUTE_DAILY_BASIS_CAP - ); - const maxProjectedGain = Math.min(dailyCeiling * safeHorizonDays * 2, ABSOLUTE_PROJECTED_GAIN_CAP); - - const runs = Array.from({ length: safeSimulationCount }, (_, simIndex) => { - let dailyValue = lastDaily; - // Model gains directly so huge historical totals cannot swallow day-level deltas. - let gainValue = 0; - const series = []; - for (let day = 0; day < safeHorizonDays; day += 1) { - const shock = gaussianRandom(rng) * volatility; - const meanReversion = ((recentAverage - dailyValue) / Math.max(dailyValue, 1)) * meanReversionStrength; - const step = clampNumber(drift + meanReversion + shock, -0.6, 0.6); - const growth = Math.exp(step); - dailyValue = clampNumber(dailyValue * growth, 0, dailyCeiling); - gainValue = clampNumber(gainValue + dailyValue, 0, maxProjectedGain); - series.push({ - day: day + 1, - daily: dailyValue, - cumulative: startTotal + gainValue, - gain: gainValue, - }); - } - return { - key: `sim-${simIndex}`, - series, - final: series.at(-1)?.gain || 0, - }; - }); - - const bandSeries = Array.from({ length: safeHorizonDays }, (_, dayIndex) => { - const values = runs - .map((run) => run.series[dayIndex]?.gain || 0) - .sort((left, right) => left - right); - return { - day: dayIndex + 1, - p10: percentileFromSorted(values, 0.1), - p25: percentileFromSorted(values, 0.25), - p50: percentileFromSorted(values, 0.5), - p75: percentileFromSorted(values, 0.75), - p90: percentileFromSorted(values, 0.9), - }; - }); - - const samples = runs - .filter((_, index) => index % Math.ceil(safeSimulationCount / 14) === 0) - .slice(0, 14) - .map((run) => run.series.map((point) => point.gain)); - - const endingValues = runs.map((run) => run.final).sort((left, right) => left - right); - const summary = { - startTotal, - p10Gain: percentileFromSorted(endingValues, 0.1), - p50Gain: percentileFromSorted(endingValues, 0.5), - p90Gain: percentileFromSorted(endingValues, 0.9), - avgDaily: recentAverage, - }; - - summary.p10Total = startTotal + summary.p10Gain; - summary.p50Total = startTotal + summary.p50Gain; - summary.p90Total = startTotal + summary.p90Gain; - - return { bandSeries, samples, summary, horizonDays: safeHorizonDays, simulationCount: safeSimulationCount }; + const rawDrift = logReturns.length ? logReturns.reduce((sum, value) => sum + value, 0) / logReturns.length : 0.012, shortHistory = recent.length < 4, + drift = clampNumber(rawDrift, -0.08, shortHistory ? 0.05 : 0.12), variance = logReturns.length + ? logReturns.reduce((sum, value) => sum + (value - rawDrift) ** 2, 0) / logReturns.length + : 0.05, volatilityFloor = shortHistory ? 0.06 : 0.08, volatilityCeiling = shortHistory ? 0.22 : 0.35, + volatility = clampNumber(Math.max(Math.sqrt(variance), volatilityFloor), volatilityFloor, volatilityCeiling), + lastDaily = Math.max(recent[recent.length - 1], 1), startTotal = Number( cumulativeSeries?.at?.(-1)?.savedTotal || cumulativeSeries?.at?.(-1)?.saved || + basis.reduce((sum, value) => sum + value, 0), ), boundedSeedBase = Number.isFinite(startTotal) ? Math.abs(startTotal % 1e9) : 0, + rng = createSeededRng(Math.round(boundedSeedBase + lastDaily + recent.length * 13)), meanReversionStrength = shortHistory ? 0.03 : 0.04, + dailyCeiling = Math.min( Math.max(recentPeak * 4, recentAverage * 6, recentMedian * 10, 1), ABSOLUTE_DAILY_BASIS_CAP, ), + maxProjectedGain = Math.min(dailyCeiling * safeHorizonDays * 2, ABSOLUTE_PROJECTED_GAIN_CAP), + runs = Array.from({ length: safeSimulationCount }, (_, simIndex) => { let dailyValue = lastDaily, gainValue = 0; + const series = []; + for (let day = 0; day < safeHorizonDays; day += 1) { const shock = gaussianRandom(rng) * volatility, + meanReversion = ((recentAverage - dailyValue) / Math.max(dailyValue, 1)) * meanReversionStrength, + step = clampNumber(drift + meanReversion + shock, -0.6, 0.6), growth = Math.exp(step); + ((dailyValue = clampNumber(dailyValue * growth, 0, dailyCeiling)), (gainValue = clampNumber(gainValue + dailyValue, 0, maxProjectedGain)), + series.push({ day: day + 1, daily: dailyValue, cumulative: startTotal + gainValue, gain: gainValue, })); + } + return { key: `sim-${simIndex}`, series, final: series.at(-1)?.gain || 0, }; + }), bandSeries = Array.from({ length: safeHorizonDays }, (_, dayIndex) => { + const values = runs.map((run) => run.series[dayIndex]?.gain || 0).sort((left, right) => left - right); + return { day: dayIndex + 1, p10: percentileFromSorted(values, 0.1), p25: percentileFromSorted(values, 0.25), + p50: percentileFromSorted(values, 0.5), p75: percentileFromSorted(values, 0.75), p90: percentileFromSorted(values, 0.9), }; + }), samples = runs + .filter((_, index) => index % Math.ceil(safeSimulationCount / 14) === 0) + .slice(0, 14) + .map((run) => run.series.map((point) => point.gain)), endingValues = runs.map((run) => run.final).sort((left, right) => left - right), + summary = { startTotal, p10Gain: percentileFromSorted(endingValues, 0.1), p50Gain: percentileFromSorted(endingValues, 0.5), + p90Gain: percentileFromSorted(endingValues, 0.9), avgDaily: recentAverage, }; + return ( (summary.p10Total = startTotal + summary.p10Gain), + (summary.p50Total = startTotal + summary.p50Gain), (summary.p90Total = startTotal + summary.p90Gain), { bandSeries, + samples, summary, horizonDays: safeHorizonDays, simulationCount: safeSimulationCount, } ); } +export { buildMonteCarloProjection }; diff --git a/desktop/cortex-control-center/src/api-client.js b/desktop/cortex-control-center/src/api-client.js index f06a29a3..5c245a0b 100644 --- a/desktop/cortex-control-center/src/api-client.js +++ b/desktop/cortex-control-center/src/api-client.js @@ -1,442 +1,71 @@ -/** - * Extracted API client logic for Cortex Control Center. - * Pure functions that accept deps as params -- testable without React. - */ - -const TOKEN_REFRESH_ATTEMPTS = 4; -const TOKEN_REFRESH_DELAY_MS = 250; -const IPC_ABORT_TIMEOUT_MS = 8_000; -const IPC_ABORT_TIMEOUT_HEALTH_MS = 12_000; -const IPC_ABORT_TIMEOUT_MCP_MS = 30_000; -const IPC_ABORT_TIMEOUT_RECALL_MS = 20_000; -const IPC_ABORT_TIMEOUT_CORE_MS = 15_000; -const IPC_ABORT_TIMEOUT_SECONDARY_MS = 20_000; -const IPC_ABORT_TIMEOUT_ANALYTICS_MS = 60_000; -const IPC_TRANSPORT_MARGIN_MS = 500; - -function wait(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function isAuthStatus(status) { - return status === 401 || status === 403; -} - -async function withTimeout(promise, timeoutMs, label) { - let timer = null; - try { - return await Promise.race([ - promise, - new Promise((_, reject) => { - timer = setTimeout(() => { - reject(new Error(`${label}: timed out after ${timeoutMs}ms`)); - }, timeoutMs); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } -} - -function normalizePathForTimeoutRouting(path) { - const raw = String(path || "").trim().toLowerCase(); - if (!raw) return ""; - if (raw.startsWith("http://") || raw.startsWith("https://")) { - try { - const parsed = new URL(raw); - return `${parsed.pathname || "/"}`.toLowerCase() + (parsed.search || ""); - } catch { - return raw; - } - } - if (raw.startsWith("/")) return raw; - return `/${raw}`; -} - -function normalizeCortexBaseUrl(cortexBase) { - let url; - try { - url = new URL(String(cortexBase || "").trim()); - } catch { - throw new Error("Cortex base URL must be a valid URL."); - } - - if (url.protocol !== "http:" && url.protocol !== "https:") { - throw new Error("Cortex base URL must use http or https."); - } - if (!url.hostname) { - throw new Error("Cortex base URL must include a host."); - } - if (url.username || url.password) { - throw new Error("Cortex base URL must not include embedded credentials."); - } - - url.hash = ""; - url.search = ""; - return url.toString().replace(/\/+$/, ""); -} - -function buildHttpFallbackUrl(cortexBase, path) { - const normalizedPath = String(path || "").trim(); - const route = normalizedPath.startsWith("/") ? normalizedPath : `/${normalizedPath}`; - return `${normalizeCortexBaseUrl(cortexBase)}${route}`; -} - -function formatHttpError(path, status, bodyText) { - if (!bodyText) { - return `${path}: HTTP ${status}`; - } - - try { - const parsed = JSON.parse(bodyText); - if (parsed && typeof parsed.error === "string" && parsed.error.trim()) { - return `${path}: HTTP ${status} (${parsed.error.trim()})`; - } - if (parsed && typeof parsed.message === "string" && parsed.message.trim()) { - return `${path}: HTTP ${status} (${parsed.message.trim()})`; - } - } catch { - const trimmed = bodyText.trim().slice(0, 200); - if (trimmed) { - return `${path}: HTTP ${status} (${trimmed})`; - } - } - - return `${path}: HTTP ${status}`; -} - -function resolveIpcTimeoutMs(path) { - const normalized = normalizePathForTimeoutRouting(path); - if (normalized === "/health" || normalized.startsWith("/health?")) return IPC_ABORT_TIMEOUT_HEALTH_MS; - if ( - normalized === "/sessions" - || normalized === "/locks" - || normalized.startsWith("/tasks") - ) { - return IPC_ABORT_TIMEOUT_CORE_MS; - } - if ( - normalized.startsWith("/feed") - || normalized.startsWith("/messages") - || normalized.startsWith("/activity") - || normalized.startsWith("/conflicts") - || normalized.startsWith("/permissions") - ) { - return IPC_ABORT_TIMEOUT_SECONDARY_MS; - } - if (normalized.startsWith("/savings")) return IPC_ABORT_TIMEOUT_ANALYTICS_MS; - if (normalized.startsWith("/mcp-rpc")) return IPC_ABORT_TIMEOUT_MCP_MS; - if (normalized.startsWith("/recall")) return IPC_ABORT_TIMEOUT_RECALL_MS; - return IPC_ABORT_TIMEOUT_MS; -} - -function resolveIpcTransportTimeoutMs(path) { - return Math.max(500, resolveIpcTimeoutMs(path) - IPC_TRANSPORT_MARGIN_MS); -} - -/** - * @param {unknown} value - * @returns {value is {status: number, body: string}} - */ -function isIpcResponseEnvelope(value) { - if (!value || typeof value !== "object") return false; - return typeof value.status === "number" && typeof value.body === "string"; -} - -function shouldFallbackToHttp(error) { - const message = String(error?.message || error || "").toLowerCase(); - return ( - message.includes("ipc request") || - message.includes("task failed") || - message.includes("invalid ipc response") || - message.includes("read failed") || - message.includes("write failed") || - message.includes("os error 10060") || - message.includes("connection attempt failed because the connected party did not properly respond") || - message.includes("established connection failed because connected host has failed to respond") || - message.includes("cannot connect to daemon") || - message.includes("cannot set read timeout") || - message.includes("cannot set write timeout") - ); -} - -/** - * @param {unknown} error - * @returns {string} - */ -function errorMessage(error) { - if (error instanceof Error) return error.message; - return String(error ?? ""); -} - -function buildFallbackFailure(ipcError, httpError) { - const ipcMessage = errorMessage(ipcError); - const httpMessage = errorMessage(httpError); - return new Error(`${ipcMessage}; HTTP fallback failed: ${httpMessage}`); -} - -async function refreshTokenIfChanged(onTokenRefresh, getToken, previousToken) { - if (!onTokenRefresh) return false; - - const requiresRotation = Boolean(previousToken); - for (let attempt = 1; attempt <= TOKEN_REFRESH_ATTEMPTS; attempt += 1) { - await onTokenRefresh(previousToken, attempt); - const nextToken = getToken(); - const ready = Boolean(nextToken) && (!requiresRotation || nextToken !== previousToken); - if (ready) { - return true; - } - if (attempt < TOKEN_REFRESH_ATTEMPTS) { - await wait(TOKEN_REFRESH_DELAY_MS * attempt); - } - } - - return false; -} - -/** - * Creates a GET API caller. - * @template TResponse - * @param {object} deps - * @param {() => Function|null} deps.getInvoke - returns Tauri invoke fn or null - * @param {() => string} deps.getToken - returns current auth token - * @param {string} deps.cortexBase - base URL for browser fallback - * @returns {(path: string, withAuth?: boolean) => Promise} - */ -export function createApi({ getInvoke, getToken, cortexBase, onTokenRefresh }) { - return async function api(path, withAuth = false, _retried = false) { - const invoke = getInvoke(); - let token = getToken(); - - if (withAuth && !token && !_retried) { - // Token not loaded yet -- try refreshing once (daemon may still be writing it) - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return api(path, withAuth, true); - } - token = getToken(); - } - - if (withAuth && !token) { - throw new Error(`${path}: no auth token (Tauri IPC ${invoke ? "available" : "missing"})`); - } - - const requestViaHttp = async () => { - const headers = { "X-Cortex-Request": "true" }; - if (withAuth) headers.Authorization = `Bearer ${token}`; - const response = await fetch(buildHttpFallbackUrl(cortexBase, path), { headers }); - if (isAuthStatus(response.status) && withAuth && !_retried) { - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return api(path, withAuth, true); - } - } - if (!response.ok) { - const bodyText = await response.text().catch(() => ""); - throw new Error(formatHttpError(path, response.status, bodyText)); - } - return await response.json(); - }; - - if (invoke) { - try { - const timeoutMs = resolveIpcTimeoutMs(path); - const transportTimeoutMs = resolveIpcTransportTimeoutMs(path); - const response = await withTimeout(invoke("fetch_cortex", { - path, - authToken: withAuth ? token : "", - timeoutMs: transportTimeoutMs, - }), timeoutMs, `${path}: IPC request`); - if (!isIpcResponseEnvelope(response)) { - throw new Error(`${path}: invalid IPC response`); - } - // On 401, re-read token and retry once (handles daemon token rotation on startup) - if (isAuthStatus(response.status) && withAuth && !_retried) { - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return api(path, withAuth, true); - } - } - if (response.status < 200 || response.status >= 300) { - throw new Error(formatHttpError(path, response.status, response.body)); - } - return JSON.parse(response.body); - } catch (ipcError) { - if (!shouldFallbackToHttp(ipcError)) { - throw ipcError; - } - try { - return await requestViaHttp(); - } catch (httpError) { - throw buildFallbackFailure(ipcError, httpError); - } - } - } - - return requestViaHttp(); - }; -} - -/** - * Creates a POST API caller. - * @template TResponse - * @param {object} deps - * @param {() => Function|null} deps.getInvoke - returns Tauri invoke fn or null - * @param {() => string} deps.getToken - returns current auth token - * @param {string} deps.cortexBase - base URL for browser fallback - * @param {() => Promise|void} [deps.onTokenRefresh] - refreshes auth token once on startup/rotation - * @returns {(path: string, body?: Record) => Promise} - */ -export function createPostApi({ getInvoke, getToken, cortexBase, onTokenRefresh }) { - return async function postApi(path, body = {}, _retried = false) { - const invoke = getInvoke(); - let token = getToken(); - - if (!token && !_retried) { - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return postApi(path, body, true); - } - token = getToken(); - } - - if (!token) { - throw new Error(`POST ${path}: no auth token`); - } - - const requestViaHttp = async () => { - const response = await fetch(buildHttpFallbackUrl(cortexBase, path), { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-Cortex-Request": "true", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify(body), - }); - if (isAuthStatus(response.status) && !_retried) { - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return postApi(path, body, true); - } - } - if (!response.ok) { - const bodyText = await response.text().catch(() => ""); - throw new Error(formatHttpError(`POST ${path}`, response.status, bodyText)); - } - return await response.json(); - }; - - if (invoke) { - try { - const timeoutMs = resolveIpcTimeoutMs(path); - const transportTimeoutMs = resolveIpcTransportTimeoutMs(path); - const response = await withTimeout(invoke("post_cortex", { - path, - authToken: token, - body: JSON.stringify(body), - timeoutMs: transportTimeoutMs, - }), timeoutMs, `POST ${path}: IPC request`); - if (!isIpcResponseEnvelope(response)) { - throw new Error(`POST ${path}: invalid IPC response`); - } - if (isAuthStatus(response.status) && !_retried) { - const refreshed = await refreshTokenIfChanged(onTokenRefresh, getToken, token); - if (refreshed) { - return postApi(path, body, true); - } - } - if (response.status < 200 || response.status >= 300) { - throw new Error(formatHttpError(`POST ${path}`, response.status, response.body)); - } - return JSON.parse(response.body); - } catch (ipcError) { - if (!shouldFallbackToHttp(ipcError)) { - throw ipcError; - } - try { - return await requestViaHttp(); - } catch (httpError) { - throw buildFallbackFailure(ipcError, httpError); - } - } - } - - return requestViaHttp(); - }; -} - -const PANEL_LABELS = { - "/sessions": "Sessions", - "/locks": "Locks", - "/tasks": "Tasks", - "/feed": "Feed", - "/messages": "Messages", - "/activity": "Activity", - "/savings": "Savings", - "/conflicts": "Conflicts", - "/permissions": "Permissions", -}; - -function panelLabelFromError(message) { - const path = String(message || "").split(":")[0]; - const normalized = Object.keys(PANEL_LABELS).find((candidate) => path.startsWith(candidate)); - return normalized ? PANEL_LABELS[normalized] : null; -} - -export function isAuthFailure(message) { - const text = String(message || ""); - return text.includes("HTTP 401") || text.includes("HTTP 403") || text.includes("no auth token"); -} - -export function summarizeDashboardErrors(errors) { - const unique = [...new Set((errors || []).filter(Boolean))]; - if (!unique.length) return ""; - - const authFailures = unique.filter(isAuthFailure); - if (authFailures.length !== unique.length) { - return unique.join("; "); - } - - const panels = authFailures - .map(panelLabelFromError) - .filter(Boolean); - - if (!panels.length) { - return "Protected Cortex panels could not authenticate. Refresh the token or restart the daemon from Control Center."; - } - - return `${panels.join(", ")} could not authenticate. Refresh the token or restart the daemon from Control Center.`; -} - -/** - * Runs multiple async fns via allSettled, applies partial results, - * then re-throws if any failed. - * @template T - * @param {Array<{fn: () => Promise, apply: (value: T | null) => void}>} tasks - */ -export async function settledWithRethrow(tasks) { - const results = await Promise.allSettled(tasks.map(t => t.fn())); - results.forEach((r, i) => { - tasks[i].apply(r.status === "fulfilled" ? r.value : null); - }); - const failed = results.filter(r => r.status === "rejected"); - if (failed.length) { - const reasons = failed.map(f => errorMessage(f.reason)); - throw new Error(reasons.join("; ")); - } -} - -/** - * Runs multiple async fns via allSettled, collects unique error messages. - * Never throws. - * @param {Array<() => Promise>} fns - * @returns {Promise} unique error messages (empty if all succeeded) - */ -export async function settledCollectErrors(fns) { - const results = await Promise.allSettled(fns.map(fn => fn())); - const failures = results.filter(r => r.status === "rejected"); - if (!failures.length) return []; - const reasons = failures.map(f => errorMessage(f.reason)); - return [...new Set(reasons)]; -} +const TOKEN_REFRESH_ATTEMPTS=4,TOKEN_REFRESH_DELAY_MS=250,IPC_ABORT_TIMEOUT_MS=8e3,IPC_ABORT_TIMEOUT_HEALTH_MS=12e3,IPC_ABORT_TIMEOUT_MCP_MS=3e4,IPC_ABORT_TIMEOUT_RECALL_MS=2e4,IPC_ABORT_TIMEOUT_CORE_MS=15e3, +IPC_ABORT_TIMEOUT_SECONDARY_MS=2e4,IPC_ABORT_TIMEOUT_ANALYTICS_MS=6e4,IPC_TRANSPORT_MARGIN_MS=500 +;function wait(ms){return new Promise(resolve=>setTimeout(resolve,ms))}function isAuthStatus(status){return status===401||status===403} +async function withTimeout(promise,timeoutMs,label){let timer=null;try{return await Promise.race([promise,new Promise((_,reject)=>{timer=setTimeout(()=>{ +reject(new Error(`${label}: timed out after ${timeoutMs}ms`))},timeoutMs)})])}finally{timer&&clearTimeout(timer)}}function normalizePathForTimeoutRouting(path){ +const raw=String(path||"").trim().toLowerCase();if(!raw)return"";if(raw.startsWith("http://")||raw.startsWith("https://"))try{const parsed=new URL(raw) +;return`${parsed.pathname||"/"}`.toLowerCase()+(parsed.search||"")}catch{return raw}return raw.startsWith("/")?raw:`/${raw}`} +function normalizeCortexBaseUrl(cortexBase){let url;try{url=new URL(String(cortexBase||"").trim())}catch{throw new Error("Cortex base URL must be a valid URL.") +}if(url.protocol!=="http:"&&url.protocol!=="https:")throw new Error("Cortex base URL must use http or https.") +;if(!url.hostname)throw new Error("Cortex base URL must include a host.") +;if(url.username||url.password)throw new Error("Cortex base URL must not include embedded credentials.");return url.hash="",url.search="", +url.toString().replace(/\/+$/,"")}function buildHttpFallbackUrl(cortexBase,path){ +const normalizedPath=String(path||"").trim(),route=normalizedPath.startsWith("/")?normalizedPath:`/${normalizedPath}` +;return`${normalizeCortexBaseUrl(cortexBase)}${route}`}function formatHttpError(path,status,bodyText){if(!bodyText)return`${path}: HTTP ${status}`;try{ +const parsed=JSON.parse(bodyText);if(parsed&&typeof parsed.error=="string"&&parsed.error.trim())return`${path}: HTTP ${status} (${parsed.error.trim()})` +;if(parsed&&typeof parsed.message=="string"&&parsed.message.trim())return`${path}: HTTP ${status} (${parsed.message.trim()})`}catch{ +const trimmed=bodyText.trim().slice(0,200);if(trimmed)return`${path}: HTTP ${status} (${trimmed})`}return`${path}: HTTP ${status}`} +function resolveIpcTimeoutMs(path){const normalized=normalizePathForTimeoutRouting(path) +;return normalized==="/health"||normalized.startsWith("/health?")?12e3:normalized==="/sessions"||normalized==="/locks"||normalized.startsWith("/tasks")?15e3:normalized.startsWith("/feed")||normalized.startsWith("/messages")||normalized.startsWith("/activity")||normalized.startsWith("/conflicts")||normalized.startsWith("/permissions")?2e4:normalized.startsWith("/savings")?6e4:normalized.startsWith("/mcp-rpc")?3e4:normalized.startsWith("/recall")?2e4:8e3 +}function resolveIpcTransportTimeoutMs(path){return Math.max(500,resolveIpcTimeoutMs(path)-500)}function isIpcResponseEnvelope(value){ +return!value||typeof value!="object"?!1:typeof value.status=="number"&&typeof value.body=="string"}function shouldFallbackToHttp(error){ +const message=String(error?.message||error||"").toLowerCase() +;return message.includes("ipc request")||message.includes("task failed")||message.includes("invalid ipc response")||message.includes("read failed")||message.includes("write failed")||message.includes("os error 10060")||message.includes("connection attempt failed because the connected party did not properly respond")||message.includes("established connection failed because connected host has failed to respond")||message.includes("cannot connect to daemon")||message.includes("cannot set read timeout")||message.includes("cannot set write timeout") +}function errorMessage(error){return error instanceof Error?error.message:String(error??"")}function buildFallbackFailure(ipcError,httpError){ +const ipcMessage=errorMessage(ipcError),httpMessage=errorMessage(httpError);return new Error(`${ipcMessage}; HTTP fallback failed: ${httpMessage}`)} +async function refreshTokenIfChanged(onTokenRefresh,getToken,previousToken){if(!onTokenRefresh)return!1;const requiresRotation=!!previousToken +;for(let attempt=1;attempt<=4;attempt+=1){await onTokenRefresh(previousToken,attempt);const nextToken=getToken() +;if(!!nextToken&&(!requiresRotation||nextToken!==previousToken))return!0;attempt<4&&await wait(250*attempt)}return!1} +function createApi({getInvoke:getInvoke,getToken:getToken,cortexBase:cortexBase,onTokenRefresh:onTokenRefresh}){ +return async function api(path,withAuth=!1,_retried=!1){const invoke=getInvoke();let token=getToken();if(withAuth&&!token&&!_retried){ +if(await refreshTokenIfChanged(onTokenRefresh,getToken,token))return api(path,withAuth,!0);token=getToken()} +if(withAuth&&!token)throw new Error(`${path}: no auth token (Tauri IPC ${invoke?"available":"missing"})`);const requestViaHttp=async()=>{const headers={ +"X-Cortex-Request":"true"};withAuth&&(headers.Authorization=`Bearer ${token}`);const response=await fetch(buildHttpFallbackUrl(cortexBase,path),{headers:headers +});if(isAuthStatus(response.status)&&withAuth&&!_retried&&await refreshTokenIfChanged(onTokenRefresh,getToken,token))return api(path,withAuth,!0) +;if(!response.ok){const bodyText=await response.text().catch(()=>"");throw new Error(formatHttpError(path,response.status,bodyText))} +return await response.json()};if(invoke)try{ +const timeoutMs=resolveIpcTimeoutMs(path),transportTimeoutMs=resolveIpcTransportTimeoutMs(path),response=await withTimeout(invoke("fetch_cortex",{path:path, +authToken:withAuth?token:"",timeoutMs:transportTimeoutMs}),timeoutMs,`${path}: IPC request`) +;if(!isIpcResponseEnvelope(response))throw new Error(`${path}: invalid IPC response`) +;if(isAuthStatus(response.status)&&withAuth&&!_retried&&await refreshTokenIfChanged(onTokenRefresh,getToken,token))return api(path,withAuth,!0) +;if(response.status<200||response.status>=300)throw new Error(formatHttpError(path,response.status,response.body));return JSON.parse(response.body) +}catch(ipcError){if(!shouldFallbackToHttp(ipcError))throw ipcError;try{return await requestViaHttp()}catch(httpError){ +throw buildFallbackFailure(ipcError,httpError)}}return requestViaHttp()}} +function createPostApi({getInvoke:getInvoke,getToken:getToken,cortexBase:cortexBase,onTokenRefresh:onTokenRefresh}){ +return async function postApi(path,body={},_retried=!1){const invoke=getInvoke();let token=getToken();if(!token&&!_retried){ +if(await refreshTokenIfChanged(onTokenRefresh,getToken,token))return postApi(path,body,!0);token=getToken()} +if(!token)throw new Error(`POST ${path}: no auth token`);const requestViaHttp=async()=>{const response=await fetch(buildHttpFallbackUrl(cortexBase,path),{ +method:"POST",headers:{"Content-Type":"application/json","X-Cortex-Request":"true",Authorization:`Bearer ${token}`},body:JSON.stringify(body)}) +;if(isAuthStatus(response.status)&&!_retried&&await refreshTokenIfChanged(onTokenRefresh,getToken,token))return postApi(path,body,!0);if(!response.ok){ +const bodyText=await response.text().catch(()=>"");throw new Error(formatHttpError(`POST ${path}`,response.status,bodyText))}return await response.json()} +;if(invoke)try{const timeoutMs=resolveIpcTimeoutMs(path),transportTimeoutMs=resolveIpcTransportTimeoutMs(path),response=await withTimeout(invoke("post_cortex",{ +path:path,authToken:token,body:JSON.stringify(body),timeoutMs:transportTimeoutMs}),timeoutMs,`POST ${path}: IPC request`) +;if(!isIpcResponseEnvelope(response))throw new Error(`POST ${path}: invalid IPC response`) +;if(isAuthStatus(response.status)&&!_retried&&await refreshTokenIfChanged(onTokenRefresh,getToken,token))return postApi(path,body,!0) +;if(response.status<200||response.status>=300)throw new Error(formatHttpError(`POST ${path}`,response.status,response.body));return JSON.parse(response.body) +}catch(ipcError){if(!shouldFallbackToHttp(ipcError))throw ipcError;try{return await requestViaHttp()}catch(httpError){ +throw buildFallbackFailure(ipcError,httpError)}}return requestViaHttp()}}const PANEL_LABELS={"/sessions":"Sessions","/locks":"Locks","/tasks":"Tasks", +"/feed":"Feed","/messages":"Messages","/activity":"Activity","/savings":"Savings","/conflicts":"Conflicts","/permissions":"Permissions"} +;function panelLabelFromError(message){ +const path=String(message||"").split(":")[0],normalized=Object.keys(PANEL_LABELS).find(candidate=>path.startsWith(candidate)) +;return normalized?PANEL_LABELS[normalized]:null}function isAuthFailure(message){const text=String(message||"") +;return text.includes("HTTP 401")||text.includes("HTTP 403")||text.includes("no auth token")}function summarizeDashboardErrors(errors){ +const unique=[...new Set((errors||[]).filter(Boolean))];if(!unique.length)return"";const authFailures=unique.filter(isAuthFailure) +;if(authFailures.length!==unique.length)return unique.join("; ");const panels=authFailures.map(panelLabelFromError).filter(Boolean) +;return panels.length?`${panels.join(", ")} could not authenticate. Refresh the token or restart the daemon from Control Center.`:"Protected Cortex panels could not authenticate. Refresh the token or restart the daemon from Control Center." +}async function settledWithRethrow(tasks){const results=await Promise.allSettled(tasks.map(t=>t.fn()));results.forEach((r,i)=>{ +tasks[i].apply(r.status==="fulfilled"?r.value:null)});const failed=results.filter(r=>r.status==="rejected");if(failed.length){ +const reasons=failed.map(f=>errorMessage(f.reason));throw new Error(reasons.join("; "))}}async function settledCollectErrors(fns){ +const failures=(await Promise.allSettled(fns.map(fn=>fn()))).filter(r=>r.status==="rejected");if(!failures.length)return[] +;const reasons=failures.map(f=>errorMessage(f.reason));return[...new Set(reasons)]} +export{createApi,createPostApi,isAuthFailure,settledCollectErrors,settledWithRethrow,summarizeDashboardErrors}; diff --git a/desktop/cortex-control-center/src/api-client.test.js b/desktop/cortex-control-center/src/api-client.test.js index e20621c3..72d8fbdb 100644 --- a/desktop/cortex-control-center/src/api-client.test.js +++ b/desktop/cortex-control-center/src/api-client.test.js @@ -13,8 +13,7 @@ import { function makeDeps(overrides = {}) { return { getInvoke: () => overrides.invoke ?? null, - getToken: () => - typeof overrides.getToken === "function" ? overrides.getToken() : (overrides.token ?? ""), + getToken: () => (typeof overrides.getToken === "function" ? overrides.getToken() : (overrides.token ?? "")), cortexBase: overrides.cortexBase ?? "http://127.0.0.1:7437", onTokenRefresh: overrides.onTokenRefresh, }; @@ -27,7 +26,7 @@ function mockFetch(status, body, ok) { status, json: () => Promise.resolve(body), text: () => Promise.resolve(typeof body === "string" ? body : JSON.stringify(body ?? "")), - }) + }), ); } @@ -60,17 +59,13 @@ describe("createApi - api()", () => { it("throws with path when withAuth=true and no token (no IPC)", async () => { const api = createApi(makeDeps({ token: "" })); - await expect(api("/sessions", true)).rejects.toThrow( - "/sessions: no auth token (Tauri IPC missing)" - ); + await expect(api("/sessions", true)).rejects.toThrow("/sessions: no auth token (Tauri IPC missing)"); }); it("throws with path when withAuth=true and no token (IPC available)", async () => { const invoke = vi.fn(); const api = createApi(makeDeps({ token: "", invoke })); - await expect(api("/sessions", true)).rejects.toThrow( - "/sessions: no auth token (Tauri IPC available)" - ); + await expect(api("/sessions", true)).rejects.toThrow("/sessions: no auth token (Tauri IPC available)"); expect(invoke).not.toHaveBeenCalled(); }); @@ -87,9 +82,7 @@ describe("createApi - api()", () => { globalThis.fetch = vi.fn(() => Promise.reject(new Error("network down"))); const invoke = vi.fn(() => new Promise(() => {})); const api = createApi(makeDeps({ invoke, token: "tok" })); - const assertion = expect(api("/health")).rejects.toThrow( - "/health: IPC request: timed out after 12000ms" - ); + const assertion = expect(api("/health")).rejects.toThrow("/health: IPC request: timed out after 12000ms"); await vi.advanceTimersByTimeAsync(12000); await assertion; } finally { @@ -119,7 +112,9 @@ describe("createApi - api()", () => { globalThis.fetch = mockFetch(200, { status: "ok-from-http" }, true); const api = createApi(makeDeps({ invoke, token: "tok" })); - await expect(api("/sessions", true)).resolves.toEqual({ status: "ok-from-http" }); + await expect(api("/sessions", true)).resolves.toEqual({ + status: "ok-from-http", + }); expectFetchCall("http://127.0.0.1:7437/sessions", { headers: { Authorization: "Bearer tok", @@ -134,10 +129,12 @@ describe("createApi - api()", () => { makeDeps({ token: "tok", cortexBase: " http://127.0.0.1:7437///?debug=1#fragment ", - }) + }), ); - await expect(api("/sessions", true)).resolves.toEqual({ status: "ok-from-http" }); + await expect(api("/sessions", true)).resolves.toEqual({ + status: "ok-from-http", + }); expectFetchCall("http://127.0.0.1:7437/sessions", { headers: { Authorization: "Bearer tok", @@ -152,7 +149,7 @@ describe("createApi - api()", () => { makeDeps({ token: "tok", cortexBase: "file:///tmp/cortex.sock", - }) + }), ); await expect(api("/sessions", true)).rejects.toThrow("must use http or https"); @@ -160,13 +157,19 @@ describe("createApi - api()", () => { }); it("falls back to HTTP GET for Windows connection-attempt timeout envelopes", async () => { - const invoke = vi.fn(() => Promise.reject(new Error( - "A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. (os error 10060)" - ))); + const invoke = vi.fn(() => + Promise.reject( + new Error( + "A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. (os error 10060)", + ), + ), + ); globalThis.fetch = mockFetch(200, { status: "ok-from-http" }, true); const api = createApi(makeDeps({ invoke, token: "tok" })); - await expect(api("/sessions", true)).resolves.toEqual({ status: "ok-from-http" }); + await expect(api("/sessions", true)).resolves.toEqual({ + status: "ok-from-http", + }); expectFetchCall("http://127.0.0.1:7437/sessions", { headers: { Authorization: "Bearer tok", @@ -183,34 +186,26 @@ describe("createApi - api()", () => { }); it("throws on IPC HTTP non-2xx", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 403, body: '{"error":"forbidden"}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 403, body: '{"error":"forbidden"}' })); const api = createApi(makeDeps({ invoke, token: "tok" })); await expect(api("/sessions", true)).rejects.toThrow("/sessions: HTTP 403"); }); it("throws on IPC JSON parse failure", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: "not json{{{" }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: "not json{{{" })); const api = createApi(makeDeps({ invoke, token: "tok" })); await expect(api("/health")).rejects.toThrow(); // SyntaxError from JSON.parse }); it("returns parsed JSON on IPC success", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"sessions":[]}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"sessions":[]}' })); const api = createApi(makeDeps({ invoke, token: "tok" })); const result = await api("/sessions", true); expect(result).toEqual({ sessions: [] }); }); it("uses an extended transport timeout for MCP RPC IPC GET requests", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"ok":true}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"ok":true}' })); const api = createApi(makeDeps({ invoke, token: "tok" })); await api("/mcp-rpc", true); expect(invoke).toHaveBeenCalledWith("fetch_cortex", { @@ -221,9 +216,7 @@ describe("createApi - api()", () => { }); it("routes absolute session URLs to core IPC timeout budgets", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"sessions":[]}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"sessions":[]}' })); const api = createApi(makeDeps({ invoke, token: "tok" })); await api("http://127.0.0.1:7437/sessions", true); expect(invoke).toHaveBeenCalledWith("fetch_cortex", { @@ -267,15 +260,13 @@ describe("createApi - api()", () => { const onTokenRefresh = vi.fn(async () => { token = "stale-token"; }); - const invoke = vi.fn(() => - Promise.resolve({ status: 401, body: '{"error":"Unauthorized"}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 401, body: '{"error":"Unauthorized"}' })); const api = createApi( makeDeps({ getToken: () => token, invoke, onTokenRefresh, - }) + }), ); await expect(api("/sessions", true)).rejects.toThrow("/sessions: HTTP 401"); @@ -295,9 +286,7 @@ describe("createPostApi - postApi()", () => { it("throws when no token (always requires auth)", async () => { const postApi = createPostApi(makeDeps({ token: "" })); - await expect(postApi("/resolve")).rejects.toThrow( - "POST /resolve: no auth token" - ); + await expect(postApi("/resolve")).rejects.toThrow("POST /resolve: no auth token"); }); it("refreshes token once before POST when startup token is missing", async () => { @@ -305,16 +294,14 @@ describe("createPostApi - postApi()", () => { const onTokenRefresh = vi.fn(async () => { token = "fresh-token"; }); - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"ok":true}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"ok":true}' })); const postApi = createPostApi( makeDeps({ getToken: () => token, invoke, onTokenRefresh, - }) + }), ); const result = await postApi("/resolve", { keepId: "a" }); @@ -332,9 +319,7 @@ describe("createPostApi - postApi()", () => { globalThis.fetch = vi.fn(() => Promise.reject(new Error("network down"))); const invoke = vi.fn(() => Promise.resolve(undefined)); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); - await expect(postApi("/resolve")).rejects.toThrow( - "POST /resolve: invalid IPC response" - ); + await expect(postApi("/resolve")).rejects.toThrow("POST /resolve: invalid IPC response"); }); it("times out hung IPC POST requests", async () => { @@ -344,7 +329,7 @@ describe("createPostApi - postApi()", () => { const invoke = vi.fn(() => new Promise(() => {})); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); const assertion = expect(postApi("/resolve", { keepId: "a" })).rejects.toThrow( - "POST /resolve: IPC request: timed out after 8000ms" + "POST /resolve: IPC request: timed out after 8000ms", ); await vi.advanceTimersByTimeAsync(8000); await assertion; @@ -379,7 +364,9 @@ describe("createPostApi - postApi()", () => { globalThis.fetch = mockFetch(200, { ok: true }, true); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); - await expect(postApi("/resolve", { keepId: "a" })).resolves.toEqual({ ok: true }); + await expect(postApi("/resolve", { keepId: "a" })).resolves.toEqual({ + ok: true, + }); expectFetchCall("http://127.0.0.1:7437/resolve", { method: "POST", headers: { @@ -395,23 +382,27 @@ describe("createPostApi - postApi()", () => { makeDeps({ token: "tok", cortexBase: "https://user:pass@team.example.com", - }) + }), ); - await expect(postApi("/resolve", { keepId: "a" })).rejects.toThrow( - "must not include embedded credentials" - ); + await expect(postApi("/resolve", { keepId: "a" })).rejects.toThrow("must not include embedded credentials"); expect(globalThis.fetch).not.toHaveBeenCalled(); }); it("falls back to HTTP POST for Windows connection-attempt timeout envelopes", async () => { - const invoke = vi.fn(() => Promise.reject(new Error( - "A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. (os error 10060)" - ))); + const invoke = vi.fn(() => + Promise.reject( + new Error( + "A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond. (os error 10060)", + ), + ), + ); globalThis.fetch = mockFetch(200, { ok: true }, true); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); - await expect(postApi("/resolve", { keepId: "a" })).resolves.toEqual({ ok: true }); + await expect(postApi("/resolve", { keepId: "a" })).resolves.toEqual({ + ok: true, + }); expectFetchCall("http://127.0.0.1:7437/resolve", { method: "POST", headers: { @@ -422,17 +413,13 @@ describe("createPostApi - postApi()", () => { }); it("throws on IPC HTTP non-2xx", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 422, body: '{"error":"bad"}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 422, body: '{"error":"bad"}' })); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); await expect(postApi("/resolve")).rejects.toThrow("POST /resolve: HTTP 422"); }); it("returns parsed JSON on IPC success", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"ok":true}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"ok":true}' })); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); const result = await postApi("/resolve", { keepId: "a" }); expect(result).toEqual({ ok: true }); @@ -459,7 +446,7 @@ describe("createPostApi - postApi()", () => { getToken: () => token, invoke, onTokenRefresh, - }) + }), ); const result = await postApi("/resolve", { keepId: "a" }); @@ -480,9 +467,7 @@ describe("createPostApi - postApi()", () => { }); it("uses an extended transport timeout for MCP RPC IPC POST requests", async () => { - const invoke = vi.fn(() => - Promise.resolve({ status: 200, body: '{"ok":true}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 200, body: '{"ok":true}' })); const postApi = createPostApi(makeDeps({ invoke, token: "tok" })); await postApi("/mcp-rpc", { jsonrpc: "2.0", id: "1" }); expect(invoke).toHaveBeenCalledWith("post_cortex", { @@ -535,7 +520,7 @@ describe("createPostApi - postApi()", () => { makeDeps({ getToken: () => token, onTokenRefresh, - }) + }), ); const result = await postApi("/resolve", { x: 1 }); @@ -556,16 +541,14 @@ describe("createPostApi - postApi()", () => { const onTokenRefresh = vi.fn(async () => { token = "stale-token"; }); - const invoke = vi.fn(() => - Promise.resolve({ status: 401, body: '{"error":"Unauthorized"}' }) - ); + const invoke = vi.fn(() => Promise.resolve({ status: 401, body: '{"error":"Unauthorized"}' })); const postApi = createPostApi( makeDeps({ getToken: () => token, invoke, onTokenRefresh, - }) + }), ); await expect(postApi("/resolve", { keepId: "a" })).rejects.toThrow("POST /resolve: HTTP 401"); @@ -598,7 +581,7 @@ describe("settledWithRethrow", () => { apply: (v) => results.push(v), }, { fn: () => Promise.resolve("also-ok"), apply: (v) => results.push(v) }, - ]) + ]), ).rejects.toThrow("/sessions: HTTP 403"); // partial results were still applied @@ -616,7 +599,7 @@ describe("settledWithRethrow", () => { fn: () => Promise.reject(new Error("err2")), apply: () => {}, }, - ]) + ]), ).rejects.toThrow("err1; err2"); }); }); @@ -627,10 +610,7 @@ describe("settledWithRethrow", () => { describe("settledCollectErrors", () => { it("returns empty array on full success", async () => { - const errors = await settledCollectErrors([ - () => Promise.resolve(), - () => Promise.resolve(), - ]); + const errors = await settledCollectErrors([() => Promise.resolve(), () => Promise.resolve()]); expect(errors).toEqual([]); }); @@ -653,10 +633,7 @@ describe("settledCollectErrors", () => { }); it("handles non-Error rejections gracefully", async () => { - const errors = await settledCollectErrors([ - () => Promise.reject("raw string"), - () => Promise.reject(42), - ]); + const errors = await settledCollectErrors([() => Promise.reject("raw string"), () => Promise.reject(42)]); expect(errors).toEqual(["raw string", "42"]); }); }); @@ -669,19 +646,16 @@ describe("summarizeDashboardErrors", () => { "/locks: HTTP 401", "/tasks?status=all: HTTP 401", "/feed?since=1h: HTTP 401", - ]) + ]), ).toBe( - "Sessions, Locks, Tasks, Feed could not authenticate. Refresh the token or restart the daemon from Control Center." + "Sessions, Locks, Tasks, Feed could not authenticate. Refresh the token or restart the daemon from Control Center.", ); }); it("falls back to the original joined output for mixed failures", () => { - expect( - summarizeDashboardErrors([ - "/sessions: HTTP 401", - "/health: HTTP 500", - ]) - ).toBe("/sessions: HTTP 401; /health: HTTP 500"); + expect(summarizeDashboardErrors(["/sessions: HTTP 401", "/health: HTTP 500"])).toBe( + "/sessions: HTTP 401; /health: HTTP 500", + ); }); }); diff --git a/desktop/cortex-control-center/src/app/App.jsx b/desktop/cortex-control-center/src/app/App.jsx index bc6c6175..993aa1bf 100644 --- a/desktop/cortex-control-center/src/app/App.jsx +++ b/desktop/cortex-control-center/src/app/App.jsx @@ -1,16 +1,13 @@ +import React from "react"; import { useEffect } from "react"; import { DEFAULT_CORTEX_BASE } from "./constants.js"; import { persistBrowserAuthToken } from "./browser-bootstrap.js"; import { useDashboardHooks } from "./hooks/useDashboardHooks.js"; +import { DashboardProvider } from "./DashboardContext.jsx"; import { AppShell } from "./AppShell.jsx"; - -export function App() { - const dashboard = useDashboardHooks(); - const { refreshAllRef, runRefreshAll } = dashboard; - - useEffect(() => { - refreshAllRef.current = runRefreshAll; - }, [refreshAllRef, runRefreshAll]); - - return ; +function App() { const dashboard = useDashboardHooks(), { refreshAllRef, runRefreshAll } = dashboard; + return ( useEffect(() => { refreshAllRef.current = runRefreshAll; + }, [refreshAllRef, runRefreshAll]), React.createElement( DashboardProvider, { value: dashboard }, + React.createElement(AppShell, { DEFAULT_CORTEX_BASE, persistBrowserAuthToken }), ) ); } +export { App }; diff --git a/desktop/cortex-control-center/src/app/AppShell.jsx b/desktop/cortex-control-center/src/app/AppShell.jsx index af1f7417..ae196462 100644 --- a/desktop/cortex-control-center/src/app/AppShell.jsx +++ b/desktop/cortex-control-center/src/app/AppShell.jsx @@ -1,389 +1,152 @@ +import React from "react"; import { useEffect } from "react"; import { installUpdate } from "../updater.js"; import { AppIcon } from "../ui-icons.jsx"; import { PANEL_SEQUENCE } from "./constants.js"; import { PanelStage } from "./panels/panel-stage.jsx"; - -export function AppShell(d) { - const { - effectiveSidebarCollapsed, - panel, - changePanel, - pill, - utilityPill, - sidebarUtilityStats, - activePanelLabel, - daemonState, - daemonRecoveryHint, - handleRestartDaemon, - restartingDaemon, - invokeRef, - handleStartDaemon, - handleStopDaemon, - canStartDaemon, - canStopDaemon, - restartError, - availableUpdate, - updateInstalling, - setUpdateInstalling, - setFeedbackMessage, - feedbackMessage, - setSidebarCollapsed, - topbarRef, - stats, - normalizedSessions, - openConnectionDialog, - hostLabel, - daemonStatusBadge, - showEditorSetupWizard, - isSettingUpEditors, - closeEditorSetupWizard, - editorSetupDialogRef, - editorDetectionSummary, - selectedEditorIds, - toggleEditorSelection, - manualMcpSnippet, - applyEditorSetup, - showConnectionDialog, - dismissConnectionDialog, - connectionDialogRef, - connectionDialogTriggerRef, - isTauriRuntime, - connectionEndpoint, - closeConnectionDialog, - setCortexBase, - tokenRef, - persistBrowserAuthToken, - readAuthToken, - refreshAllRef, - DEFAULT_CORTEX_BASE, - trapFocusInContainer, - restoreFocusToTrigger, - } = d; - - useEffect(() => { - if (!showConnectionDialog || !connectionDialogRef.current) return undefined; - return trapFocusInContainer(connectionDialogRef.current); - }, [showConnectionDialog]); - - useEffect(() => { - if (!showEditorSetupWizard || !editorSetupDialogRef.current) return undefined; - return trapFocusInContainer(editorSetupDialogRef.current); - }, [showEditorSetupWizard]); - - return ( -
- Skip to main content - - -
-

- {feedbackMessage} -

-
-
- CORTEX - / - {activePanelLabel.toUpperCase()} -
-
- MEM {stats.memories} - DEC {stats.decisions} - EVT {stats.events} - AGENTS {normalizedSessions.length} - - - {daemonStatusBadge.label} - -
-
- - {showEditorSetupWizard && ( -
!isSettingUpEditors && closeEditorSetupWizard()}> -
e.stopPropagation()} - > -
-
- Shared MCP Registration -

Setup MCP

-
- - {editorDetectionSummary.detected}/{editorDetectionSummary.results.length} - -
-

- Choose which supported clients should receive the shared Cortex attach-only MCP entry. Every client points at the same - app-owned daemon command. -

-
- {editorDetectionSummary.results.map((entry) => { - const tone = !entry.detected ? "idle" : entry.registered ? "ok" : "warn"; - const stateLabel = !entry.detected ? "Not detected" : entry.registered ? "Configured" : "Detected"; - const selected = selectedEditorIds.includes(entry.id); - return ( - - ); - })} -
-
- Manual Fallback -

If a client is missing from the supported list, register this MCP server manually or paste it into that AI's setup flow:

-
{manualMcpSnippet}
-

Replace codex with that AI's agent ID (for example: claude, cursor, gemini).

-
-
- - -
-
-
- )} - - {showConnectionDialog && ( -
-
e.stopPropagation()} - > -
-

Connection Settings

- -
-

- {isTauriRuntime +import { useDashboard } from "./DashboardContext.jsx"; +function AppShell({ DEFAULT_CORTEX_BASE: defaultCortexBase, persistBrowserAuthToken: persistAuthToken }) { const d = useDashboard(); + const { effectiveSidebarCollapsed, panel, changePanel, pill, utilityPill, sidebarUtilityStats, activePanelLabel, + daemonState, daemonRecoveryHint, handleRestartDaemon, restartingDaemon, invokeRef, handleStartDaemon, handleStopDaemon, canStartDaemon, + canStopDaemon, restartError, availableUpdate, updateInstalling, setUpdateInstalling, setFeedbackMessage, feedbackMessage, setSidebarCollapsed, + topbarRef, stats, normalizedSessions, openConnectionDialog, hostLabel, daemonStatusBadge, showEditorSetupWizard, isSettingUpEditors, + closeEditorSetupWizard, editorSetupDialogRef, editorDetectionSummary, selectedEditorIds, + toggleEditorSelection, manualMcpSnippet, applyEditorSetup, showConnectionDialog, + dismissConnectionDialog, connectionDialogRef, isTauriRuntime, connectionEndpoint, closeConnectionDialog, setCortexBase, tokenRef, + readAuthToken, refreshAllRef, trapFocusInContainer, } = d; + return ( useEffect(() => { if (!(!showConnectionDialog || !connectionDialogRef.current)) + return trapFocusInContainer(connectionDialogRef.current); + }, [showConnectionDialog]), useEffect(() => { if (!(!showEditorSetupWizard || !editorSetupDialogRef.current)) + return trapFocusInContainer(editorSetupDialogRef.current); + }, [showEditorSetupWizard]), React.createElement( "div", { className: `app ${effectiveSidebarCollapsed ? "sidebar-collapsed" : ""}`, }, + React.createElement("a", { className: "skip-link", href: "#main-content" }, "Skip to main content"), React.createElement( "aside", { + className: `sidebar ${effectiveSidebarCollapsed ? "collapsed" : ""}`, "aria-labelledby": "sidebar-title", }, React.createElement( + "div", { className: "sidebar-header" }, React.createElement( "div", + { className: "logo" }, React.createElement("span", { id: "sidebar-title" }, "Cortex"), + ), React.createElement("div", { className: pill.className }, pill.label), ), React.createElement( + "nav", { className: "sidebar-nav", "aria-label": "Primary panels" }, PANEL_SEQUENCE.map((item, idx) => React.createElement( + "button", { key: item.key, type: "button", className: `nav-item ${panel === item.key ? "active" : ""}`, onClick: () => changePanel(item.key), + "data-key": idx + 1, "aria-current": panel === item.key ? "page" : void 0, }, React.createElement( + "span", { style: { opacity: 0.5, fontSize: "12px" } }, React.createElement(AppIcon, { name: item.icon }), ), item.label, ), ), ), + React.createElement( "div", { className: "sidebar-utility" }, React.createElement( "div", { className: "sidebar-utility-header" }, + React.createElement("span", { className: "sidebar-utility-kicker" }, "Mission status"), React.createElement( + "span", { className: `sidebar-utility-pill ${utilityPill.className}` }, utilityPill.label, ), + ), React.createElement( "div", { className: "sidebar-utility-grid" }, sidebarUtilityStats.map((item) => React.createElement( "div", { + key: item.label, className: `sidebar-utility-card tone-${item.tone}`, + }, React.createElement("span", { className: "sidebar-utility-label" }, item.label), + React.createElement("strong", { className: "sidebar-utility-value" }, item.value), ), ), ), + React.createElement( "div", { className: "sidebar-utility-note" }, React.createElement("span", { className: "sidebar-utility-note-label" }, "Focus"), + React.createElement("strong", null, activePanelLabel), React.createElement("p", null, daemonState.message), daemonRecoveryHint + ? React.createElement("p", { className: "sidebar-utility-alert" }, daemonRecoveryHint) + : null, ), ), React.createElement( "div", { className: "sidebar-footer" }, React.createElement( "div", + { className: "daemon-restart-row" }, React.createElement( "button", { + type: "button", className: "btn-ctrl btn-restart", onClick: handleRestartDaemon, disabled: restartingDaemon || !invokeRef.current, + }, restartingDaemon ? "Restarting..." : "Restart", ), ), React.createElement( "div", { className: "daemon-controls-grid" }, React.createElement( + "button", { type: "button", className: "btn-ctrl btn-primary", onClick: handleStartDaemon, disabled: !canStartDaemon, }, "Start", + ), React.createElement( "button", { type: "button", className: "btn-ctrl", onClick: handleStopDaemon, disabled: !canStopDaemon, + }, "Stop", ), React.createElement( "button", { type: "button", className: "btn-ctrl btn-danger", onClick: async () => { if (invokeRef.current) + try { await d.call("quit_app"); + } catch {} }, }, "Exit", ), ), restartError + ? React.createElement( "button", { type: "button", + className: "btn-sm btn-danger btn-restart-retry", onClick: handleRestartDaemon, }, "Retry Restart", ) + : null, availableUpdate && React.createElement( "div", + { className: "update-banner" }, React.createElement("span", null, "v", availableUpdate.version, " available"), React.createElement( "button", + { type: "button", className: "btn-sm btn-primary", disabled: updateInstalling, + onClick: async () => { (setUpdateInstalling(!0), setFeedbackMessage("Downloading update...")); + try { await installUpdate(availableUpdate); + } catch (err) { (setFeedbackMessage(`Update failed: ${String(err)}`), setUpdateInstalling(!1)); + } }, }, updateInstalling ? "Installing..." : "Update", + ), ), React.createElement("p", { className: "sidebar-status", "aria-hidden": "true" }, feedbackMessage), React.createElement( + "button", { type: "button", className: "btn-sidebar-collapse", "aria-label": effectiveSidebarCollapsed ? "Expand sidebar" : "Collapse sidebar", + title: effectiveSidebarCollapsed ? "Expand sidebar" : "Collapse sidebar", onClick: () => setSidebarCollapsed((c) => !c), + }, React.createElement(AppIcon, { name: effectiveSidebarCollapsed ? "chevron-right" : "chevron-left", size: 16, }), ), ), ), + React.createElement( "main", { id: "main-content", className: "content", tabIndex: -1 }, React.createElement( "p", { className: "sr-only", role: "status", + "aria-live": "polite", "aria-atomic": "true", }, feedbackMessage, ), React.createElement( "div", { + ref: topbarRef, className: `topbar ${panel === "overview" ? "topbar-hidden" : ""}`, "aria-hidden": panel === "overview" ? !0 : void 0, }, + React.createElement( "div", { className: "topbar-left" }, React.createElement("span", { className: "topbar-path" }, "CORTEX"), + React.createElement("span", { className: "topbar-sep" }, "/"), + React.createElement("span", { className: "topbar-current" }, activePanelLabel.toUpperCase()), ), React.createElement( "div", + { className: "topbar-right" }, React.createElement( "span", { className: "topbar-stat" }, + React.createElement("span", { className: "topbar-label" }, "MEM"), " ", stats.memories, ), + React.createElement( "span", { className: "topbar-stat" }, React.createElement("span", { className: "topbar-label" }, "DEC"), + " ", stats.decisions, ), React.createElement( + "span", { className: "topbar-stat" }, React.createElement("span", { className: "topbar-label" }, "EVT"), " ", + stats.events, ), React.createElement( "span", + { className: "topbar-stat" }, React.createElement("span", { className: "topbar-label" }, "AGENTS"), " ", normalizedSessions.length, + ), React.createElement( "button", { + type: "button", className: "topbar-stat topbar-connection", onClick: openConnectionDialog, tabIndex: panel === "overview" ? -1 : void 0, + title: "Click to change connection", "aria-label": `Connection host ${hostLabel}. Open connection settings.`, + }, React.createElement("span", { className: "topbar-label" }, "HOST"), hostLabel, ), + React.createElement( "span", { className: `topbar-status ${daemonStatusBadge.className}`, + title: daemonStatusBadge.title, }, daemonStatusBadge.label, ), ), ), showEditorSetupWizard && React.createElement( + "div", { className: "connection-overlay", role: "presentation", + onClick: () => !isSettingUpEditors && closeEditorSetupWizard(), }, React.createElement( "div", + { ref: editorSetupDialogRef, className: "connection-dialog editor-setup-dialog", role: "dialog", + "aria-modal": "true", "aria-labelledby": "editor-setup-title", + "aria-describedby": "editor-setup-description", "aria-busy": isSettingUpEditors ? !0 : void 0, + tabIndex: -1, onClick: (e) => e.stopPropagation(), }, React.createElement( + "div", { className: "editor-setup-dialog-header" }, React.createElement( "div", + null, React.createElement("span", { className: "editor-setup-kicker" }, "Shared MCP Registration"), + React.createElement("h2", { id: "editor-setup-title" }, "Setup MCP"), ), React.createElement( "span", + { className: "badge" }, editorDetectionSummary.detected, "/", editorDetectionSummary.results.length, ), ), React.createElement( "p", + { className: "connection-subtitle", id: "editor-setup-description", }, + "Choose which supported clients should receive the shared Cortex attach-only MCP entry. Every client points at the same app-owned daemon command.", + ), React.createElement( "div", { className: "editor-setup-choice-list" }, + editorDetectionSummary.results.map((entry) => { const tone = entry.detected ? (entry.registered ? "ok" : "warn") : "idle", + stateLabel = entry.detected ? (entry.registered ? "Configured" : "Detected") : "Not detected", + selected = selectedEditorIds.includes(entry.id); + return React.createElement( "label", { key: entry.id, + className: `editor-setup-choice ${tone} ${entry.detected ? "" : "disabled"}`, }, React.createElement("input", { type: "checkbox", + checked: selected, disabled: !entry.detected || isSettingUpEditors, onChange: () => toggleEditorSelection(entry.id), }), + React.createElement( "div", { className: "editor-setup-choice-body" }, React.createElement( + "div", { className: "editor-setup-item-head" }, React.createElement("span", { className: "editor-setup-name" }, entry.name), + React.createElement("span", { className: "editor-setup-state" }, stateLabel), ), + entry.configPath ? React.createElement("code", null, entry.configPath) : null, + React.createElement("p", null, entry.message || "No detail provided."), ), ); + }), ), React.createElement( "div", + { className: "editor-setup-manual" }, React.createElement("span", { className: "editor-setup-kicker" }, "Manual Fallback"), + React.createElement( "p", + null, "If a client is missing from the supported list, register this MCP server manually or paste it into that AI's setup flow:", + ), React.createElement("pre", null, manualMcpSnippet), React.createElement( "p", + null, "Replace ", React.createElement("code", null, "codex"), " with that AI's agent ID (for example: ", + React.createElement("code", null, "claude"), ", ", React.createElement("code", null, "cursor"), ", ", + React.createElement("code", null, "gemini"), ").", ), ), React.createElement( "div", { className: "connection-actions" }, React.createElement( + "button", { type: "button", className: "btn-sm", onClick: closeEditorSetupWizard, disabled: isSettingUpEditors, }, "Cancel", + ), React.createElement( "button", { + type: "button", className: "btn-sm btn-primary", onClick: applyEditorSetup, disabled: isSettingUpEditors || !selectedEditorIds.length, + }, isSettingUpEditors + ? "Applying..." + : `Apply to ${selectedEditorIds.length} Client${selectedEditorIds.length === 1 ? "" : "s"}`, ), ), ), + ), showConnectionDialog && React.createElement( "div", { className: "connection-overlay", role: "presentation", onClick: dismissConnectionDialog, + }, React.createElement( "div", { ref: connectionDialogRef, className: "connection-dialog", role: "dialog", "aria-modal": "true", + "aria-labelledby": "connection-dialog-title", "aria-describedby": "connection-dialog-description", + tabIndex: -1, onClick: (e) => e.stopPropagation(), }, React.createElement( "div", { className: "connection-dialog-header" }, + React.createElement("h2", { id: "connection-dialog-title" }, "Connection Settings"), React.createElement( "button", { + type: "button", className: "connection-dialog-close", "aria-label": "Close connection settings", onClick: dismissConnectionDialog, + }, "\xD7", ), ), React.createElement( "p", { className: "connection-subtitle", id: "connection-dialog-description", }, isTauriRuntime ? "Desktop app mode uses the local app-managed Cortex daemon only." - : "Connect to a local or remote Cortex daemon"} -

-
{ - e.preventDefault(); - if (isTauriRuntime) { - setCortexBase(DEFAULT_CORTEX_BASE); - tokenRef.current = ""; - persistBrowserAuthToken(""); - closeConnectionDialog(); - queueMicrotask(() => refreshAllRef.current()); - return; - } - const fd = new FormData(e.target); - const host = fd.get("host")?.toString().trim() || "127.0.0.1"; - const port = fd.get("port")?.toString().trim() || "7437"; - const token = fd.get("token")?.toString().trim(); - setCortexBase(`http://${host}:${port}`); - tokenRef.current = token || ""; - persistBrowserAuthToken(token || ""); - closeConnectionDialog(); - queueMicrotask(() => refreshAllRef.current()); - }}> - - - -
- - -
- -
-
- )} - - -
-
- ); + : "Connect to a local or remote Cortex daemon", ), React.createElement( "form", + { onSubmit: (e) => { if ((e.preventDefault(), isTauriRuntime)) { (setCortexBase(defaultCortexBase), + (tokenRef.current = ""), persistAuthToken(""), closeConnectionDialog(), queueMicrotask(() => refreshAllRef.current())); + return; + } + const fd = new FormData(e.target), host = fd.get("host")?.toString().trim() || "127.0.0.1", + port = fd.get("port")?.toString().trim() || "7437", token = fd.get("token")?.toString().trim(); + (setCortexBase(`http://${host}:${port}`), (tokenRef.current = token || ""), persistAuthToken(token || ""), closeConnectionDialog(), + queueMicrotask(() => refreshAllRef.current())); }, }, React.createElement( + "label", { className: "connection-field" }, React.createElement("span", null, "Host"), React.createElement("input", { + name: "host", defaultValue: connectionEndpoint.host, placeholder: "127.0.0.1", disabled: isTauriRuntime, + }), ), React.createElement( "label", + { className: "connection-field" }, React.createElement("span", null, "Port"), React.createElement("input", { name: "port", + defaultValue: connectionEndpoint.port, placeholder: "7437", disabled: isTauriRuntime, }), + ), React.createElement( "label", { className: "connection-field" }, + React.createElement("span", null, "Auth Token"), React.createElement("input", { name: "token", type: "password", placeholder: isTauriRuntime + ? "Managed by desktop app token flow" + : "Leave blank for local (auto-read)", disabled: isTauriRuntime, }), ), + React.createElement( "div", { className: "connection-actions" }, React.createElement( "button", { type: "button", className: "btn-sm", + onClick: () => { (setCortexBase(defaultCortexBase), (tokenRef.current = ""), persistAuthToken(""), + closeConnectionDialog(), readAuthToken({ suppressFeedback: !0 }), queueMicrotask(() => refreshAllRef.current())); }, + }, "Reset to Local", ), React.createElement("button", { type: "submit", className: "btn-sm btn-primary" }, "Connect"), ), ), ), ), + React.createElement(PanelStage), ), ) ); } +export { AppShell }; diff --git a/desktop/cortex-control-center/src/app/DashboardContext.jsx b/desktop/cortex-control-center/src/app/DashboardContext.jsx new file mode 100644 index 00000000..da4e0488 --- /dev/null +++ b/desktop/cortex-control-center/src/app/DashboardContext.jsx @@ -0,0 +1,15 @@ +import { createContext, useContext } from "react"; + +const DashboardContext = createContext(null); + +function DashboardProvider({ value, children }) { + return {children}; +} + +function useDashboard() { + const value = useContext(DashboardContext); + if (!value) throw new Error("useDashboard must be used inside DashboardProvider"); + return value; +} + +export { DashboardProvider, useDashboard }; diff --git a/desktop/cortex-control-center/src/app/browser-bootstrap.js b/desktop/cortex-control-center/src/app/browser-bootstrap.js index 6bacef2a..cda341a6 100644 --- a/desktop/cortex-control-center/src/app/browser-bootstrap.js +++ b/desktop/cortex-control-center/src/app/browser-bootstrap.js @@ -1,17 +1,10 @@ import { - CORTEX_AUTH_STORAGE_KEY, - CORTEX_BASE_STORAGE_KEY, - CORTEX_PANEL_STORAGE_KEY, - DEFAULT_CORTEX_BASE, - LEGACY_CORTEX_AUTH_STORAGE_KEYS, - PANEL_SEQUENCE_KEYS, -} from "./constants.js"; + CORTEX_AUTH_STORAGE_KEY, CORTEX_BASE_STORAGE_KEY, CORTEX_PANEL_STORAGE_KEY, DEFAULT_CORTEX_BASE, + LEGACY_CORTEX_AUTH_STORAGE_KEYS, PANEL_SEQUENCE_KEYS, } from "./constants.js"; export function clearLegacyBrowserAuthTokens() { if (typeof window === "undefined") return; - try { - for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { - window.sessionStorage.removeItem(key); + try { for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { window.sessionStorage.removeItem(key); window.localStorage.removeItem(key); } } catch { @@ -21,51 +14,42 @@ export function clearLegacyBrowserAuthTokens() { export function readPersistedBrowserAuthToken() { if (typeof window === "undefined") return ""; - try { - const sessionToken = window.sessionStorage.getItem(CORTEX_AUTH_STORAGE_KEY) || ""; + try { const sessionToken = window.sessionStorage.getItem(CORTEX_AUTH_STORAGE_KEY) || ""; if (sessionToken) return sessionToken; - for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { - const legacySessionToken = window.sessionStorage.getItem(key) || ""; - if (legacySessionToken) { - window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacySessionToken); + for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { const legacySessionToken = window.sessionStorage.getItem(key) || ""; + if (legacySessionToken) { window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacySessionToken); clearLegacyBrowserAuthTokens(); return legacySessionToken; } } const legacyToken = window.localStorage.getItem(CORTEX_AUTH_STORAGE_KEY) || ""; - if (legacyToken) { - window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacyToken); + if (legacyToken) { window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacyToken); window.localStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); clearLegacyBrowserAuthTokens(); return legacyToken; } - for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { - const legacyLocalToken = window.localStorage.getItem(key) || ""; - if (legacyLocalToken) { - window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacyLocalToken); + for (const key of LEGACY_CORTEX_AUTH_STORAGE_KEYS) { const legacyLocalToken = window.localStorage.getItem(key) || ""; + if (legacyLocalToken) { window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, legacyLocalToken); clearLegacyBrowserAuthTokens(); return legacyLocalToken; } } - } catch { - return ""; + } catch { return ""; } return ""; } export function readBrowserBootstrap() { - if (typeof window === "undefined") { - return { cortexBase: "", authToken: "", panel: "overview" }; + if (typeof window === "undefined") { return { cortexBase: "", authToken: "", panel: "overview" }; } const params = new URLSearchParams(window.location.search); let storedPanel = ""; let storedBase = DEFAULT_CORTEX_BASE; - try { - storedPanel = window.localStorage.getItem(CORTEX_PANEL_STORAGE_KEY) || ""; + try { storedPanel = window.localStorage.getItem(CORTEX_PANEL_STORAGE_KEY) || ""; storedBase = window.localStorage.getItem(CORTEX_BASE_STORAGE_KEY) || DEFAULT_CORTEX_BASE; } catch { // Ignore storage failures in restricted browser contexts. @@ -77,19 +61,14 @@ export function readBrowserBootstrap() { const authTokenFromParams = params.get("authToken") || ""; const authToken = authTokenFromParams || readPersistedBrowserAuthToken(); - try { - if (params.get("panel")) { - window.localStorage.setItem(CORTEX_PANEL_STORAGE_KEY, panel); + try { if (params.get("panel")) { window.localStorage.setItem(CORTEX_PANEL_STORAGE_KEY, panel); } - if (params.get("cortexBase")) { - window.localStorage.setItem(CORTEX_BASE_STORAGE_KEY, cortexBase); + if (params.get("cortexBase")) { window.localStorage.setItem(CORTEX_BASE_STORAGE_KEY, cortexBase); } } catch { // Ignore storage failures in restricted browser contexts. } - if (authTokenFromParams) { - try { - window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, authToken); + if (authTokenFromParams) { try { window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, authToken); window.localStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); } catch { // Ignore storage failures in restricted browser contexts. @@ -105,27 +84,24 @@ export function readBrowserBootstrap() { export function readLocalStorageValue(key, fallback = "") { if (typeof window === "undefined") return fallback; - try { - return window.localStorage.getItem(key) || fallback; - } catch { - return fallback; + try { return window.localStorage.getItem(key) || fallback; + } catch { return fallback; } } export function normalizeCurrencyCode(raw) { - const candidate = String(raw || "").trim().toUpperCase(); + const candidate = String(raw || "") + .trim() + .toUpperCase(); return CURRENCY_OPTIONS.includes(candidate) ? candidate : "USD"; } export function persistBrowserAuthToken(token) { if (typeof window === "undefined") return; - try { - if (token) { - window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, token); + try { if (token) { window.sessionStorage.setItem(CORTEX_AUTH_STORAGE_KEY, token); window.localStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); clearLegacyBrowserAuthTokens(); - } else { - window.sessionStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); + } else { window.sessionStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); window.localStorage.removeItem(CORTEX_AUTH_STORAGE_KEY); clearLegacyBrowserAuthTokens(); } @@ -140,13 +116,10 @@ export function priorityRank(priority) { } export async function readTauriInvoke() { - if (typeof window === "undefined" || !window.__TAURI_INTERNALS__) { - return null; + if (typeof window === "undefined" || !window.__TAURI_INTERNALS__) { return null; } - try { - const { invoke } = await import("@tauri-apps/api/core"); + try { const { invoke } = await import("@tauri-apps/api/core"); return invoke; - } catch { - return null; + } catch { return null; } } diff --git a/desktop/cortex-control-center/src/app/components/ActivityItem.jsx b/desktop/cortex-control-center/src/app/components/ActivityItem.jsx index cfc57174..31d29d8d 100644 --- a/desktop/cortex-control-center/src/app/components/ActivityItem.jsx +++ b/desktop/cortex-control-center/src/app/components/ActivityItem.jsx @@ -1,24 +1,19 @@ +import React from "react"; import { timeAgo } from "../../constants.js"; - -export function ActivityItem({ entry }) { - const files = Array.isArray(entry.files) ? entry.files.slice(0, 6) : []; - - return ( -
  • +function ActivityItem({ entry }) { const files = Array.isArray(entry.files) ? entry.files.slice(0, 6) : []; + return (
  • {entry.agent || "unknown"} {timeAgo(entry.timestamp)}
    {entry.description || "(no activity details)"}
    - {files.length ? ( -
    - {files.map((file) => ( - + {files.length ? (
    + {files.map((file) => ( {file} ))}
    ) : null} -
  • - ); + ); } +export { ActivityItem }; diff --git a/desktop/cortex-control-center/src/app/components/AgentItem.jsx b/desktop/cortex-control-center/src/app/components/AgentItem.jsx index 7672631d..56eceaa8 100644 --- a/desktop/cortex-control-center/src/app/components/AgentItem.jsx +++ b/desktop/cortex-control-center/src/app/components/AgentItem.jsx @@ -1,28 +1,29 @@ +import React from "react"; import { timeAgo } from "../../constants.js"; import { agentColor } from "../utils/agent-color.js"; - -export function AgentItem({ session }) { - const color = agentColor(session.agent); - return ( -
  • +function AgentItem({ session }) { const color = agentColor(session.agent); + return (
  • {session.agent} - ACTIVE + + ACTIVE +
    - {session.description || "Working"} - {session.project || "—"} + {session.description || "Working"} + {" - "} + {session.project || "\u2014"}
    - {(session.files || []).slice(0, 4).map((file) => ( - + {(session.files || []).slice(0, 4).map((file) => ( {file} ))} {timeAgo(session.lastHeartbeat)}
    -
  • - ); + ); } +export { AgentItem }; diff --git a/desktop/cortex-control-center/src/app/components/AnimatedNumber.jsx b/desktop/cortex-control-center/src/app/components/AnimatedNumber.jsx index 320305c7..213e7535 100644 --- a/desktop/cortex-control-center/src/app/components/AnimatedNumber.jsx +++ b/desktop/cortex-control-center/src/app/components/AnimatedNumber.jsx @@ -1,42 +1,22 @@ +import React from "react"; import { useEffect, useRef, useState } from "react"; import { MOTION_MS, easeOutCubic } from "../../design/motion.js"; - -export function AnimatedNumber({ value, duration = MOTION_MS.number, reducedMotion = false }) { - const [display, setDisplay] = useState(value); - const prevRef = useRef(value); - - useEffect(() => { - if (reducedMotion) { - setDisplay(value); - prevRef.current = value; - return undefined; - } - - const from = typeof prevRef.current === "number" ? prevRef.current : 0; - const to = typeof value === "number" ? value : 0; - if (from === to || typeof value !== "number") { - setDisplay(value); - prevRef.current = value; - return; - } - - let cancelled = false; - const start = performance.now(); - const diff = to - from; - - function tick(now) { - if (cancelled) return; - const elapsed = now - start; - const progress = Math.min(elapsed / duration, 1); - const eased = easeOutCubic(progress); - setDisplay(Math.round(from + diff * eased)); - if (progress < 1) requestAnimationFrame(tick); - } - - requestAnimationFrame(tick); - prevRef.current = to; - return () => { cancelled = true; }; - }, [value, duration, reducedMotion]); - - return <>{typeof display === "number" ? display.toLocaleString() : display}; +function AnimatedNumber({ value, duration = MOTION_MS.number, reducedMotion = !1 }) { const [display, setDisplay] = useState(value), prevRef = useRef(value); + return ( useEffect(() => { if (reducedMotion) { (setDisplay(value), (prevRef.current = value)); + return; + } + const from = typeof prevRef.current == "number" ? prevRef.current : 0, to = typeof value == "number" ? value : 0; + if (from === to || typeof value != "number") { (setDisplay(value), (prevRef.current = value)); + return; + } + let cancelled = !1; + const start = performance.now(), diff = to - from; + function tick(now) { if (cancelled) return; + const elapsed = now - start, progress = Math.min(elapsed / duration, 1), eased = easeOutCubic(progress); + (setDisplay(Math.round(from + diff * eased)), progress < 1 && requestAnimationFrame(tick)); + } + return ( requestAnimationFrame(tick), (prevRef.current = to), () => { cancelled = !0; + } ); + }, [value, duration, reducedMotion]), React.createElement(React.Fragment, null, typeof display == "number" ? display.toLocaleString() : display) ); } +export { AnimatedNumber }; diff --git a/desktop/cortex-control-center/src/app/components/BrainVisualizerPanel.jsx b/desktop/cortex-control-center/src/app/components/BrainVisualizerPanel.jsx index c6ab2f94..d412c13c 100644 --- a/desktop/cortex-control-center/src/app/components/BrainVisualizerPanel.jsx +++ b/desktop/cortex-control-center/src/app/components/BrainVisualizerPanel.jsx @@ -1,51 +1,30 @@ +import React from "react"; import { Component, lazy, Suspense } from "react"; import { AppIcon } from "../../ui-icons.jsx"; - -const LazyBrainVisualizer = lazy(() => - import("../../BrainVisualizer.jsx").then((module) => ({ default: module.BrainVisualizer })), -); - -class BrainErrorBoundary extends Component { - constructor(props) { super(props); this.state = { crashed: false, error: "" }; } - static getDerivedStateFromError(err) { return { crashed: true, error: err?.message || "Unknown error" }; } - render() { - if (this.state.crashed) return ( -
    -
    -

    Brain visualizer crashed: {this.state.error}

    - -
    - ); - return this.props.children; +import { useDashboard } from "../DashboardContext.jsx"; +const LazyBrainVisualizer = lazy(() => import("../../BrainVisualizer.jsx").then((module) => ({ default: module.BrainVisualizer, })), ); +class BrainErrorBoundary extends Component { constructor(props) { (super(props), (this.state = { crashed: !1, error: "" })); + } + static getDerivedStateFromError(err) { return { crashed: !0, error: err?.message || "Unknown error" }; + } + render() { return this.state.crashed + ? React.createElement( "div", { className: "brain-loading" }, React.createElement( + "div", { className: "coming-icon" }, React.createElement(AppIcon, { name: "brain", size: 48 }), ), + React.createElement("p", null, "Brain visualizer crashed: ", this.state.error), React.createElement( "button", { + className: "btn-sm btn-primary", onClick: () => this.setState({ crashed: !1 }), style: { marginTop: 12 }, }, "Retry", ), ) + : this.props.children; } } - -export function BrainVisualizerPanel({ brainPanelRef, panel, brainPanelMounted, api, cortexBase, authToken, effectiveReducedMotion }) { - if (!brainPanelMounted) return null; - return ( -
    - - -
    -

    Loading brain visualizer…

    - - )} - > - -
    -
    -
    - ); +function BrainVisualizerPanel() { const { brainPanelRef, panel, brainPanelMounted, api, cortexBase, tokenRef, effectiveReducedMotion } = useDashboard(), + authToken = tokenRef.current; + return brainPanelMounted + ? React.createElement( "section", { ref: brainPanelRef, + className: `panel brain-panel ${panel === "brain" ? "active" : "panel-hidden"}`, "aria-hidden": panel === "brain" ? void 0 : !0, + }, React.createElement( BrainErrorBoundary, null, React.createElement( Suspense, { fallback: React.createElement( + "div", { className: "brain-loading" }, React.createElement( "div", + { className: "coming-icon" }, React.createElement(AppIcon, { name: "brain", size: 48 }), + ), React.createElement("p", null, "Loading brain visualizer\u2026"), ), }, + React.createElement(LazyBrainVisualizer, { api, cortexBase, authToken, active: panel === "brain", reducedMotion: effectiveReducedMotion, }), ), ), ) + : null; } +export { BrainVisualizerPanel }; diff --git a/desktop/cortex-control-center/src/app/components/ConflictPairCard.jsx b/desktop/cortex-control-center/src/app/components/ConflictPairCard.jsx index c374bc06..287a33cc 100644 --- a/desktop/cortex-control-center/src/app/components/ConflictPairCard.jsx +++ b/desktop/cortex-control-center/src/app/components/ConflictPairCard.jsx @@ -1,141 +1,138 @@ +import React from "react"; import { timeAgo } from "../../constants.js"; import { - conflictBadgeClass, - formatConfidencePercent, - formatTimestamp, - formatTrustScore, -} from "../normalize/conflicts.js"; + conflictBadgeClass, formatConfidencePercent, formatTimestamp, formatTrustScore, } from "../normalize/conflicts.js"; import { agentColor } from "../utils/agent-color.js"; - -export function ConflictPairCard({ - pair, - conflictLoading = false, - onResolveQuick = null, - onResolveDraft = null, - resolveDraft = null, - onResolveDraftChange = null, -}) { - const draftAction = resolveDraft?.action || "keep"; - const draftWinner = resolveDraft?.winner || "left"; - const leftId = pair?.left?.id; - const rightId = pair?.right?.id; - const canResolve = leftId !== null && leftId !== undefined && rightId !== null && rightId !== undefined; - const winner = draftWinner === "right" ? pair.right : pair.left; - const loser = draftWinner === "right" ? pair.left : pair.right; - - return ( -
    +function ConflictPairCard({ pair, conflictLoading = !1, onResolveQuick = null, onResolveDraft = null, resolveDraft = null, onResolveDraftChange = null, }) { + const draftAction = resolveDraft?.action || "keep", draftWinner = resolveDraft?.winner || "left", leftId = pair?.left?.id, rightId = pair?.right?.id, + canResolve = leftId != null && rightId !== null && rightId !== void 0, winner = draftWinner === "right" ? pair.right : pair.left, + loser = draftWinner === "right" ? pair.left : pair.right; + return (
    Conflict #{pair.conflictId || pair.key} - {pair.classification} + + {pair.classification} + {pair.status}
    - Created {formatTimestamp(pair.createdAt)} - {pair.resolvedAt ? Resolved {formatTimestamp(pair.resolvedAt)} : null} + + {"Created "} + {formatTimestamp(pair.createdAt)} + + {pair.resolvedAt ? ( + {"Resolved "} + {formatTimestamp(pair.resolvedAt)} + + ) : null}
    -
    #{pair.left.id ?? "?"} - + {pair.left.sourceAgent || "unknown"} {timeAgo(pair.left.createdAt)}

    {pair.left.decision}

    {pair.left.context ?

    {pair.left.context}

    : null}
    - Confidence: {formatConfidencePercent(pair.left.confidence)} - Trust: {formatTrustScore(pair.left.trustScore)} + + {"Confidence: "} + {formatConfidencePercent(pair.left.confidence)} + + + {"Trust: "} + {formatTrustScore(pair.left.trustScore)} +
    -
    VS
    -
    #{pair.right.id ?? "?"} - + {pair.right.sourceAgent || "unknown"} {timeAgo(pair.right.createdAt)}

    {pair.right.decision}

    {pair.right.context ?

    {pair.right.context}

    : null}
    - Confidence: {formatConfidencePercent(pair.right.confidence)} - Trust: {formatTrustScore(pair.right.trustScore)} + + {"Confidence: "} + {formatConfidencePercent(pair.right.confidence)} + + + {"Trust: "} + {formatTrustScore(pair.right.trustScore)} +
    - - {pair.resolution ? ( -
    + {pair.resolution ? (
    Winner:{" "} - {pair.resolution.winnerId !== null && pair.resolution.winnerId !== undefined + {pair.resolution.winnerId !== null && pair.resolution.winnerId !== void 0 ? `#${pair.resolution.winnerId}` : "n/a"} {pair.resolution.winnerAgent ? ` (${pair.resolution.winnerAgent})` : ""} Loser:{" "} - {pair.resolution.loserId !== null && pair.resolution.loserId !== undefined + {pair.resolution.loserId !== null && pair.resolution.loserId !== void 0 ? `#${pair.resolution.loserId}` : "n/a"} {pair.resolution.loserAgent ? ` (${pair.resolution.loserAgent})` : ""} - {pair.resolution.action ? Action: {pair.resolution.action} : null} - {pair.resolution.method ? Method: {pair.resolution.method} : null} - {pair.resolution.resolvedBy ? Resolved by: {pair.resolution.resolvedBy} : null} - {pair.resolution.trustDelta !== null ? ( - Trust delta: {pair.resolution.trustDelta.toFixed(3)} + {pair.resolution.action ? ( + Action: {pair.resolution.action} + ) : null} + {pair.resolution.method ? ( + Method: {pair.resolution.method} + ) : null} + {pair.resolution.resolvedBy ? ( + Resolved by: {pair.resolution.resolvedBy} + ) : null} + {pair.resolution.trustDelta !== null ? ( + Trust delta: {pair.resolution.trustDelta.toFixed(3)} ) : null}
    {pair.resolution.notes ?
    {pair.resolution.notes}
    : null}
    ) : null} -
    -
    Manual resolve - {draftAction === "keep" ? ( -
    -
    - ); +
    ); } +export { ConflictPairCard }; diff --git a/desktop/cortex-control-center/src/app/components/FeedItem.jsx b/desktop/cortex-control-center/src/app/components/FeedItem.jsx index 1bf6f1f8..6a2f8a68 100644 --- a/desktop/cortex-control-center/src/app/components/FeedItem.jsx +++ b/desktop/cortex-control-center/src/app/components/FeedItem.jsx @@ -1,30 +1,26 @@ +import React from "react"; import { timeAgo } from "../../constants.js"; import { feedKindLabel } from "../utils/format.js"; - -export function FeedItem({ entry }) { - const files = Array.isArray(entry.files) ? entry.files.slice(0, 6) : []; - const metaBits = [timeAgo(entry.timestamp)]; - if (entry.priority) metaBits.push(entry.priority); - if (typeof entry.tokens === "number") metaBits.push(`${entry.tokens} tok`); - - return ( -
  • -
    - {feedKindLabel(entry.kind)} - {entry.agent || "unknown"} - {metaBits.join(" - ")} -
    -
    {entry.summary || "(no summary)"}
    - {entry.taskId ?
    task: {entry.taskId}
    : null} - {files.length ? ( -
    - {files.map((file) => ( - - {file} - - ))} +function FeedItem({ entry }) { const files = Array.isArray(entry.files) ? entry.files.slice(0, 6) : [], metaBits = [timeAgo(entry.timestamp)]; + return ( entry.priority && metaBits.push(entry.priority), typeof entry.tokens == "number" && metaBits.push(`${entry.tokens} tok`), (
  • +
    + {feedKindLabel(entry.kind)} + {entry.agent || "unknown"} + {metaBits.join(" - ")}
    - ) : null} -
  • - ); +
    {entry.summary || "(no summary)"}
    + {entry.taskId ? (
    + {"task: "} + {entry.taskId} +
    + ) : null} + {files.length ? (
    + {files.map((file) => ( + {file} + + ))} +
    + ) : null} + ) ); } +export { FeedItem }; diff --git a/desktop/cortex-control-center/src/app/components/LockItem.jsx b/desktop/cortex-control-center/src/app/components/LockItem.jsx index e87e0553..ba474ba6 100644 --- a/desktop/cortex-control-center/src/app/components/LockItem.jsx +++ b/desktop/cortex-control-center/src/app/components/LockItem.jsx @@ -1,32 +1,20 @@ +import React from "react"; import { canUnlockLock } from "../../live-surface.js"; - -export function LockItem({ lock, selectedOperator = "", onUnlock = null, busyActionKey = "" }) { - const expiryMinutes = Math.max( - 0, - Math.ceil((new Date(lock.expiresAt).getTime() - Date.now()) / 60000) - ); - const unlockBusy = busyActionKey === `unlock:${lock.path}`; - const unlockable = canUnlockLock(lock, selectedOperator); - - return ( -
  • +function LockItem({ lock, selectedOperator = "", onUnlock = null, busyActionKey = "" }) { + const expiryMinutes = Math.max(0, Math.ceil((new Date(lock.expiresAt).getTime() - Date.now()) / 6e4)), unlockBusy = busyActionKey === `unlock:${lock.path}`, + unlockable = canUnlockLock(lock, selectedOperator); + return (
  • {lock.path}
    {lock.agent} {expiryMinutes}m remaining
    - {unlockable && onUnlock ? ( -
    -
    ) : null} -
  • - ); + ); } +export { LockItem }; diff --git a/desktop/cortex-control-center/src/app/components/MessageItem.jsx b/desktop/cortex-control-center/src/app/components/MessageItem.jsx index df6b5882..14c6bbd2 100644 --- a/desktop/cortex-control-center/src/app/components/MessageItem.jsx +++ b/desktop/cortex-control-center/src/app/components/MessageItem.jsx @@ -1,21 +1,24 @@ +import React from "react"; import { timeAgo } from "../../constants.js"; import { AppIcon } from "../../ui-icons.jsx"; import { agentColor } from "../utils/agent-color.js"; - -export function MessageItem({ entry }) { - const fromColor = agentColor(entry.from); - return ( -
  • +function MessageItem({ entry }) { const fromColor = agentColor(entry.from); + return (
  • - + {entry.from || "unknown"} - + + + {entry.to || "unknown"} {timeAgo(entry.timestamp)}
    {entry.message || "(empty message)"}
    -
  • - ); + ); } +export { MessageItem }; diff --git a/desktop/cortex-control-center/src/app/components/MonteCarloProjectionChart.jsx b/desktop/cortex-control-center/src/app/components/MonteCarloProjectionChart.jsx index 93e8bd05..bf021c0b 100644 --- a/desktop/cortex-control-center/src/app/components/MonteCarloProjectionChart.jsx +++ b/desktop/cortex-control-center/src/app/components/MonteCarloProjectionChart.jsx @@ -1,45 +1,33 @@ +import React from "react"; import { formatSignedCompactNumber } from "../../number-format.js"; - -export function MonteCarloProjectionChart({ projection, width = 820, height = 280 }) { - if (!projection?.bandSeries?.length) return
    Not enough data for a projection yet
    ; - - const bandValues = projection.bandSeries.flatMap((point) => [point.p10, point.p25, point.p50, point.p75, point.p90]); - const minValue = 0; - const maxValue = Math.max(...bandValues, 1); - const maxWithHeadroom = maxValue * 1.14; - const padding = { top: 16, right: 18, bottom: 30, left: 18 }; - const innerWidth = width - padding.left - padding.right; - const innerHeight = height - padding.top - padding.bottom; - const valueRange = maxWithHeadroom - minValue || 1; - const toX = (index) => padding.left + (index / (projection.bandSeries.length - 1)) * innerWidth; - const toY = (value) => padding.top + innerHeight - ((value - minValue) / valueRange) * innerHeight; - const areaPath = (upperKey, lowerKey) => { - const top = projection.bandSeries.map((point, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(point[upperKey])}`).join(" "); - const bottom = [...projection.bandSeries] - .reverse() - .map((point, reverseIndex) => { - const index = projection.bandSeries.length - 1 - reverseIndex; - return `L ${toX(index)} ${toY(point[lowerKey])}`; - }) - .join(" "); - return `${top} ${bottom} Z`; - }; - const linePath = (key) => projection.bandSeries.map((point, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(point[key])}`).join(" "); - const samplePaths = projection.samples.map((sample) => sample.map((value, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(value)}`).join(" ")); - const endPoint = projection.bandSeries.at(-1); - const summaryX = width - padding.right - 138; - const summaryY = padding.top + 10; - - return ( - Not enough data for a projection yet
    ; + const bandValues = projection.bandSeries.flatMap((point) => [point.p10, point.p25, point.p50, point.p75, point.p90]), minValue = 0, + maxWithHeadroom = Math.max(...bandValues, 1) * 1.14, padding = { top: 16, right: 18, bottom: 30, left: 18 }, + innerWidth = width - padding.left - padding.right, innerHeight = height - padding.top - padding.bottom, + valueRange = maxWithHeadroom - minValue || 1, toX = (index) => padding.left + (index / (projection.bandSeries.length - 1)) * innerWidth, + toY = (value) => padding.top + innerHeight - ((value - minValue) / valueRange) * innerHeight, areaPath = (upperKey, lowerKey) => { + const top = projection.bandSeries + .map((point, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(point[upperKey])}`) + .join(" "), bottom = [...projection.bandSeries] + .reverse() + .map((point, reverseIndex) => { const index = projection.bandSeries.length - 1 - reverseIndex; + return `L ${toX(index)} ${toY(point[lowerKey])}`; + }) + .join(" "); + return `${top} ${bottom} Z`; }, linePath = (key) => projection.bandSeries + .map((point, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(point[key])}`) + .join(" "), samplePaths = projection.samples.map((sample) => + sample.map((value, index) => `${index === 0 ? "M" : "L"} ${toX(index)} ${toY(value)}`).join(" "), ), + endPoint = projection.bandSeries.at(-1), summaryX = width - padding.right - 138, summaryY = padding.top + 10; + return ( + aria-label="30-day Monte Carlo projection for cumulative savings gains" > @@ -51,42 +39,51 @@ export function MonteCarloProjectionChart({ projection, width = 820, height = 28 - {Array.from({ length: 4 }, (_, index) => { - const y = padding.top + (index * innerHeight) / 3; - return ; + {Array.from({ length: 4 }, (_, index) => { const y = padding.top + (index * innerHeight) / 3; + return ( ); })} - {Array.from({ length: 6 }, (_, index) => { - const x = padding.left + (index * innerWidth) / 5; - return ; + {Array.from({ length: 6 }, (_, index) => { const x = padding.left + (index * innerWidth) / 5; + return ( ); })} - {samplePaths.map((path, index) => ( - + {samplePaths.map((path, index) => ( ))} - {endPoint ? ( - <> + {endPoint ? ( - p90 {formatSignedCompactNumber(endPoint.p90)} + {"p90 "} + {formatSignedCompactNumber(endPoint.p90)} - p50 {formatSignedCompactNumber(endPoint.p50)} + {"p50 "} + {formatSignedCompactNumber(endPoint.p50)} - p10 {formatSignedCompactNumber(endPoint.p10)} + {"p10 "} + {formatSignedCompactNumber(endPoint.p10)} - + ) : null} - today - +30d gain - - ); + + today + + + +30d gain + + ); } +export { MonteCarloProjectionChart }; diff --git a/desktop/cortex-control-center/src/app/components/OperatorSelector.jsx b/desktop/cortex-control-center/src/app/components/OperatorSelector.jsx index 4dd411fc..bc80cbbc 100644 --- a/desktop/cortex-control-center/src/app/components/OperatorSelector.jsx +++ b/desktop/cortex-control-center/src/app/components/OperatorSelector.jsx @@ -1,22 +1,9 @@ +import React from "react"; import { useId } from "react"; - -export function OperatorSelector({ value, knownAgents, onChange, label = "Operator", placeholder = "codex" }) { - const datalistId = useId(); - return ( - - ); +function OperatorSelector({ value, knownAgents, onChange, label = "Operator", placeholder = "codex" }) { const datalistId = useId(); + return React.createElement( "label", { className: "feed-control" }, React.createElement("span", null, label), + React.createElement("input", { type: "text", list: datalistId, placeholder, + value, onChange: (event) => onChange(event.target.value), }), React.createElement( + "datalist", { id: datalistId }, knownAgents.map((agent) => React.createElement("option", { key: agent, value: agent })), ), ); } +export { OperatorSelector }; diff --git a/desktop/cortex-control-center/src/app/components/Sparkline.jsx b/desktop/cortex-control-center/src/app/components/Sparkline.jsx index 0034a94d..fe113762 100644 --- a/desktop/cortex-control-center/src/app/components/Sparkline.jsx +++ b/desktop/cortex-control-center/src/app/components/Sparkline.jsx @@ -1,50 +1,27 @@ +import React from "react"; import { useState } from "react"; import { buildLineGeometry } from "./sparkline-utils.js"; -export function Sparkline({ - data, - width = 280, - height = 60, - color = "var(--cyan)", - showArea = true, - showEndDot = true, - className = "", -}) { - const [id] = useState(() => `spark-fill-${++sparklineCounter}`); - const geometry = buildLineGeometry(data, width, height, 8); - if (!geometry) return
    No data yet
    ; - const lastPoint = geometry.points.at(-1); - const gridLines = Array.from({ length: 4 }, (_, index) => { - const y = 8 + (index * (height - 16)) / 3; - return ; - }); +let sparklineCounter = 0; - return ( - - ); +function Sparkline({ data, width = 280, height = 60, color = "var(--cyan)", showArea = !0, showEndDot = !0, className = "", +}) { const [id] = useState(() => `spark-fill-${++sparklineCounter}`), geometry = buildLineGeometry(data, width, height, 8); + if (!geometry) return React.createElement("div", { className: "sparkline-empty" }, "No data yet"); + const lastPoint = geometry.points.at(-1), gridLines = Array.from({ length: 4 }, (_, index) => { const y = 8 + (index * (height - 16)) / 3; + return React.createElement("line", { key: `grid-${index}`, x1: "8", x2: width - 8, y1: y, y2: y, className: "sparkline-grid-line", }); + }); + return React.createElement( "svg", { width, + height, viewBox: `0 0 ${width} ${height}`, preserveAspectRatio: "xMidYMid meet", className: `sparkline ${className}`, + "aria-hidden": "true", focusable: "false", }, React.createElement( "defs", null, React.createElement( "linearGradient", + { id, x1: "0", y1: "0", x2: "0", y2: "1" }, React.createElement("stop", { offset: "0%", stopColor: color, + stopOpacity: "0.22", }), React.createElement("stop", { offset: "70%", stopColor: color, stopOpacity: "0.08", }), React.createElement("stop", { + offset: "100%", stopColor: color, stopOpacity: "0", }), ), ), React.createElement("g", { className: "sparkline-grid" }, gridLines), showArea + ? React.createElement("path", { d: geometry.area, fill: `url(#${id})`, className: "sparkline-area", }) + : null, React.createElement("path", { d: geometry.line, fill: "none", stroke: color, strokeWidth: "2.25", strokeLinejoin: "round", strokeLinecap: "round", + className: "sparkline-line", }), showEndDot && lastPoint + ? React.createElement( React.Fragment, null, React.createElement("circle", { cx: lastPoint.x, cy: lastPoint.y, r: "6", fill: color, + fillOpacity: "0.18", }), React.createElement("circle", { cx: lastPoint.x, cy: lastPoint.y, r: "2.75", fill: color, className: "sparkline-end-dot", + }), ) + : null, ); } +export { Sparkline }; diff --git a/desktop/cortex-control-center/src/app/components/TaskItem.jsx b/desktop/cortex-control-center/src/app/components/TaskItem.jsx index b2357e6c..e48df793 100644 --- a/desktop/cortex-control-center/src/app/components/TaskItem.jsx +++ b/desktop/cortex-control-center/src/app/components/TaskItem.jsx @@ -1,32 +1,14 @@ +import React from "react"; import { canClaimTask, canFinalizeTask } from "../../live-surface.js"; import { timeAgo } from "../../constants.js"; - -export function TaskItem({ - task, - selectedOperator = "", - completionDraft = "", - completionExpanded = false, - onClaim = null, - onAbandon = null, - onComplete = null, - onDelete = null, - onCompletionDraftChange = null, - onToggleComplete = null, - busyActionKey = "", -}) { - const operator = String(selectedOperator || "").trim(); - const claimBusy = busyActionKey === `claim:${task.taskId}`; - const abandonBusy = busyActionKey === `abandon:${task.taskId}`; - const completeBusy = busyActionKey === `complete:${task.taskId}`; - const deleteBusy = busyActionKey === `delete:${task.taskId}`; - const operatorOwnsTask = canFinalizeTask(task, operator); - const files = Array.isArray(task.files) ? task.files.slice(0, 4) : []; - const detail = task.claimedBy - ? `${task.claimedBy}${task.summary ? ` — ${task.summary}` : ""} - ${timeAgo(task.claimedAt || task.completedAt)}` - : task.project || "—"; - - return ( -
  • +function TaskItem({ task, selectedOperator = "", completionDraft = "", completionExpanded = !1, onClaim = null, onAbandon = null, onComplete = null, + onDelete = null, onCompletionDraftChange = null, onToggleComplete = null, busyActionKey = "", }) { const operator = String(selectedOperator || "").trim(), + claimBusy = busyActionKey === `claim:${task.taskId}`, abandonBusy = busyActionKey === `abandon:${task.taskId}`, + completeBusy = busyActionKey === `complete:${task.taskId}`, deleteBusy = busyActionKey === `delete:${task.taskId}`, + operatorOwnsTask = canFinalizeTask(task, operator), files = Array.isArray(task.files) ? task.files.slice(0, 4) : [], detail = task.claimedBy + ? `${task.claimedBy}${task.summary ? ` \u2014 ${task.summary}` : ""} - ${timeAgo(task.claimedAt || task.completedAt)}` + : task.project || "\u2014"; + return (
  • {task.priority} @@ -34,80 +16,69 @@ export function TaskItem({
    {detail}
    {task.description ?
    {task.description}
    : null} - {files.length ? ( -
    - {files.map((file) => ( - + {files.length ? (
    + {files.map((file) => ( {file} ))}
    ) : null}
    - {canClaimTask(task, operator) && onClaim ? ( - ) : null} - {task.status === "claimed" && operatorOwnsTask && onToggleComplete ? ( - ) : null} - {task.status === "claimed" && operatorOwnsTask && onAbandon ? ( - ) : null} - {task.status === "claimed" && !operatorOwnsTask && task.claimedBy ? ( - Held by {task.claimedBy} + {task.status === "claimed" && !operatorOwnsTask && task.claimedBy ? ( + {"Held by "} + {task.claimedBy} + ) : null} - {task.status === "completed" && onDelete ? ( - ) : null}
    - {completionExpanded && operatorOwnsTask && onComplete && onCompletionDraftChange ? ( -
    + {completionExpanded && operatorOwnsTask && onComplete && onCompletionDraftChange ? (