From 5b16b33278fb415ba4da310aca9dea6136a338e6 Mon Sep 17 00:00:00 2001 From: Karn Date: Thu, 16 Jul 2026 18:49:33 +0530 Subject: [PATCH] feat: complete offline digestion and privacy controls --- .mise.toml | 4 + Cargo.lock | 48 ++++ Cargo.toml | 3 + README.md | 36 ++- SECURITY.md | 4 + adapters/claude-code/Cargo.toml | 1 + adapters/claude-code/src/importer.rs | 33 ++- adapters/claude-code/tests/import.rs | 18 ++ adapters/codex/Cargo.toml | 1 + adapters/codex/src/importer.rs | 33 ++- crates/autophagy-cli/Cargo.toml | 4 + crates/autophagy-cli/src/main.rs | 282 ++++++++++++++++++- crates/autophagy-cli/tests/cli.rs | 125 ++++++++ crates/autophagy-core/Cargo.toml | 1 + crates/autophagy-core/src/generic_jsonl.rs | 18 ++ crates/autophagy-core/tests/generic_jsonl.rs | 30 ++ crates/autophagy-redaction/Cargo.toml | 22 ++ crates/autophagy-redaction/src/lib.rs | 238 ++++++++++++++++ crates/autophagy-store/src/lib.rs | 4 +- crates/autophagy-store/src/model.rs | 30 ++ crates/autophagy-store/src/store.rs | 103 ++++++- crates/autophagy-store/tests/store.rs | 78 ++++- docs/architecture/repository-structure.md | 5 +- docs/guides/claude-code.md | 5 + docs/guides/codex.md | 5 + docs/guides/generic-jsonl.md | 14 +- docs/guides/privacy-and-lifecycle.md | 72 +++++ docs/security/threat-model.md | 65 +++++ scripts/demo-milestone-1.sh | 24 ++ 29 files changed, 1277 insertions(+), 29 deletions(-) create mode 100644 crates/autophagy-redaction/Cargo.toml create mode 100644 crates/autophagy-redaction/src/lib.rs create mode 100644 docs/guides/privacy-and-lifecycle.md create mode 100644 docs/security/threat-model.md create mode 100755 scripts/demo-milestone-1.sh diff --git a/.mise.toml b/.mise.toml index df64794..d1f0b06 100644 --- a/.mise.toml +++ b/.mise.toml @@ -28,3 +28,7 @@ run = "actionlint" [tasks.check] description = "Run the pull-request quality gate" depends = ["ci", "docs", "fmt", "lint", "test"] + +[tasks.demo] +description = "Run the offline Milestone 1 import-to-digest demonstration" +run = "scripts/demo-milestone-1.sh" diff --git a/Cargo.lock b/Cargo.lock index 5635a40..6f0a81c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,6 +93,7 @@ version = "0.1.0-alpha.1" dependencies = [ "autophagy-adapter-test-support", "autophagy-events", + "autophagy-redaction", "autophagy-store", "directories", "serde", @@ -109,6 +110,7 @@ version = "0.1.0-alpha.1" dependencies = [ "autophagy-adapter-test-support", "autophagy-events", + "autophagy-redaction", "autophagy-store", "directories", "serde", @@ -133,13 +135,17 @@ dependencies = [ "autophagy-adapter-claude-code", "autophagy-adapter-codex", "autophagy-core", + "autophagy-events", + "autophagy-patterns", "autophagy-store", "clap", "directories", "serde", "serde_json", + "sha2", "tempfile", "thiserror", + "time", ] [[package]] @@ -147,6 +153,7 @@ name = "autophagy-core" version = "0.1.0-alpha.1" dependencies = [ "autophagy-events", + "autophagy-redaction", "autophagy-store", "serde", "serde_json", @@ -179,6 +186,18 @@ dependencies = [ "time", ] +[[package]] +name = "autophagy-redaction" +version = "0.1.0-alpha.1" +dependencies = [ + "autophagy-events", + "globset", + "regex", + "serde_json", + "thiserror", + "time", +] + [[package]] name = "autophagy-store" version = "0.1.0-alpha.1" @@ -229,6 +248,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" +[[package]] +name = "bstr" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -498,6 +527,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -737,6 +779,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.3" diff --git a/Cargo.toml b/Cargo.toml index d9a2245..adf27a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/autophagy-core", "crates/autophagy-events", "crates/autophagy-patterns", + "crates/autophagy-redaction", "crates/autophagy-store", ] @@ -21,6 +22,8 @@ repository = "https://github.com/karnstack/autophagy" [workspace.dependencies] clap = { version = "4.6.2", features = ["derive", "env"] } directories = "6.0" +globset = "0.4" +regex = "1.12" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.11" diff --git a/README.md b/README.md index 032c83d..cedfbbf 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,10 @@ change because of what happened?” ## Status -Autophagy is in foundation development. Agent Event Protocol (AEP) v0.1, the -transactional local SQLite event store, generic JSONL CLI vertical slice, and -incremental Claude Code and Codex history adapters are implemented. No daemon -or background capture ships yet. Deterministic repeated-failure and explicit -user-correction detectors now emit versioned, evidence-linked packets. +The local-only Milestone 1 engine is implemented: AEP v0.1, transactional +SQLite storage, generic JSONL plus Claude Code and Codex adapters, deterministic +evidence-linked findings, ingestion redaction, retention, export, and deletion. +No daemon, mutation generation, replay, or background capture ships yet. ## Principles @@ -35,12 +34,14 @@ crates/autophagy-cli/ User-facing import, sessions, and search commands crates/autophagy-core/ Reusable streaming import application services crates/autophagy-events/ AEP Rust types, parsing, and validation crates/autophagy-patterns/ Model-free recurrence detectors and evidence packets +crates/autophagy-redaction/ Secret rules and project/artifact path policy crates/autophagy-store/ SQLite migrations, idempotency, FTS, and deletion docs/architecture/ Planned component and storage boundaries docs/blueprint/ Complete normalized product and implementation brief docs/decisions/ Architecture decision records docs/roadmap/ Small pull-request delivery sequence docs/specs/aep/0.1/ Versioned AEP JSON Schema and examples +docs/specs/evidence/0.1/ Versioned deterministic finding contract ``` The intended repository structure is documented in @@ -94,6 +95,31 @@ The [deterministic findings guide](docs/guides/deterministic-findings.md) documents recurrence thresholds, signature normalization, counterexamples, and the versioned Evidence Packet contract. +## Run the offline milestone demo + +```sh +mise run demo +``` + +The demo imports anonymized evidence, emits two deterministic patterns with +exact evidence IDs, produces a digest that confirms no model or network was +used, and previews retention deletion. Its temporary database is removed on +exit. + +Useful privacy and lifecycle commands: + +```sh +autophagy import history.jsonl --exclude-path '**/private/**' +autophagy export > autophagy-export.jsonl +autophagy prune --older-than-days 30 --dry-run +autophagy prune --older-than-days 30 +autophagy delete session ses_example +autophagy delete all --confirm delete-all +``` + +See the [privacy and lifecycle guide](docs/guides/privacy-and-lifecycle.md) and +[threat model](docs/security/threat-model.md) for guarantees and limitations. + ## Try the contract Install [mise](https://mise.jdx.dev/), then run: diff --git a/SECURITY.md b/SECURITY.md index e5dc80d..31bdd12 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,4 +15,8 @@ repository. Include a minimal, redacted reproduction and the affected version. - Generated mutations are inspectable, permission-scoped, and reversible. - Generated scripts never become executable without explicit approval. +The implemented Milestone 1 trust boundaries, residual risks, deletion +semantics, and disclosure guidance are documented in the +[threat model](docs/security/threat-model.md). + The project is pre-alpha and does not yet make production security guarantees. diff --git a/adapters/claude-code/Cargo.toml b/adapters/claude-code/Cargo.toml index 7604cbe..1e50272 100644 --- a/adapters/claude-code/Cargo.toml +++ b/adapters/claude-code/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] autophagy-events = { path = "../../crates/autophagy-events" } +autophagy-redaction = { path = "../../crates/autophagy-redaction" } autophagy-store = { path = "../../crates/autophagy-store" } directories.workspace = true serde.workspace = true diff --git a/adapters/claude-code/src/importer.rs b/adapters/claude-code/src/importer.rs index 7404028..b8a5248 100644 --- a/adapters/claude-code/src/importer.rs +++ b/adapters/claude-code/src/importer.rs @@ -6,6 +6,7 @@ use std::{ }; use autophagy_events::Event; +use autophagy_redaction::{PrivacyError, PrivacyPolicy}; use autophagy_store::{ EventStore, InsertOutcome, SearchProjection, SourceCursor, SourceIdentity, StoreError, }; @@ -31,6 +32,8 @@ pub struct ClaudeImportOptions { pub display_name: Option, /// Exact working directories to include; empty includes all. pub projects: Vec, + /// Glob patterns that exclude matching project or artifact paths. + pub exclude_paths: Vec, /// Include nested `agent-*.jsonl` transcripts. pub include_subagents: bool, /// Persist prompt, assistant, and result text in namespaced metadata. @@ -54,6 +57,7 @@ impl ClaudeImportOptions { instance_key: instance_key.into(), display_name: None, projects: Vec::new(), + exclude_paths: Vec::new(), include_subagents: false, include_content: false, index_tool_input: false, @@ -98,6 +102,10 @@ pub struct ClaudeImportSummary { pub conflicts: u64, /// Events excluded by project selection. pub skipped: u64, + /// Events excluded by path privacy policy. + pub privacy_skipped: u64, + /// String fields changed by default secret redaction. + pub redacted_fields: u64, /// Structurally valid but unsupported Claude records. pub unsupported: u64, /// Invalid records or normalization failures. @@ -135,6 +143,7 @@ pub fn import_claude_code( options: &ClaudeImportOptions, ) -> Result { validate_options(options)?; + let privacy = PrivacyPolicy::new(&options.exclude_paths)?; if !options.dry_run && store.is_none() { return Err(ClaudeImportError::MissingStore); } @@ -157,6 +166,8 @@ pub fn import_claude_code( duplicates: 0, conflicts: 0, skipped: 0, + privacy_skipped: 0, + redacted_fields: 0, unsupported: 0, rejected: 0, cursor_resets: 0, @@ -166,7 +177,7 @@ pub fn import_claude_code( diagnostics_suppressed: 0, dry_run: options.dry_run, }; - let scope = project_scope(&options.projects); + let scope = import_scope(&options.projects, &options.exclude_paths); for discovered in &discovery.files { summary.files_read += 1; @@ -263,6 +274,12 @@ pub fn import_claude_code( summary.skipped += 1; continue; } + let outcome = privacy.apply(&event); + let Some(event) = outcome.event else { + summary.privacy_skipped += 1; + continue; + }; + summary.redacted_fields += outcome.redacted_fields; if options.dry_run { continue; } @@ -318,10 +335,17 @@ fn validate_options(options: &ClaudeImportOptions) -> Result<(), ClaudeImportErr Ok(()) } -fn project_scope(projects: &[String]) -> String { +fn import_scope(projects: &[String], exclude_paths: &[String]) -> String { let mut selected = projects.to_vec(); selected.sort(); - let digest = Sha256::digest(selected.join("\0").as_bytes()); + let mut excluded = exclude_paths.to_vec(); + excluded.sort(); + let scope = format!( + "privacy/v1\0projects\0{}\0exclusions\0{}", + selected.join("\0"), + excluded.join("\0") + ); + let digest = Sha256::digest(scope.as_bytes()); let mut encoded = String::with_capacity(digest.len() * 2); for byte in digest { write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); @@ -407,6 +431,9 @@ pub enum ClaudeImportError { /// Database operation failed. #[error(transparent)] Store(#[from] StoreError), + /// Privacy policy could not be compiled. + #[error(transparent)] + Privacy(#[from] PrivacyError), /// Cursor JSON could not be encoded. #[error("could not serialize Claude Code cursor state: {0}")] Json(#[from] serde_json::Error), diff --git a/adapters/claude-code/tests/import.rs b/adapters/claude-code/tests/import.rs index fce7349..2b0ab94 100644 --- a/adapters/claude-code/tests/import.rs +++ b/adapters/claude-code/tests/import.rs @@ -119,6 +119,24 @@ fn orphaned_tool_results_are_skipped_without_guessing() { assert_eq!(summary.inserted, 1); } +#[test] +fn changing_path_exclusions_uses_an_independent_cursor_scope() { + let directory = tempfile::tempdir().expect("temp directory"); + copy_tree(&fixture_root(), directory.path()); + let mut store = EventStore::open_in_memory().expect("store"); + let mut options = + ClaudeImportOptions::new(directory.path().to_path_buf(), "fixture:policy-scope"); + options.exclude_paths = vec!["/workspace/**".to_owned()]; + let excluded = import_claude_code(Some(&mut store), &options).expect("excluded import"); + assert_eq!(excluded.inserted, 0); + assert_eq!(excluded.privacy_skipped, 8); + + options.exclude_paths.clear(); + let included = import_claude_code(Some(&mut store), &options).expect("included import"); + assert_eq!(included.records_seen, 7); + assert_eq!(included.inserted, 8); +} + fn append(path: &Path, value: &str) { use std::io::Write; let mut file = fs::OpenOptions::new() diff --git a/adapters/codex/Cargo.toml b/adapters/codex/Cargo.toml index 92e74e5..4049b26 100644 --- a/adapters/codex/Cargo.toml +++ b/adapters/codex/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] autophagy-events = { path = "../../crates/autophagy-events" } +autophagy-redaction = { path = "../../crates/autophagy-redaction" } autophagy-store = { path = "../../crates/autophagy-store" } directories.workspace = true serde.workspace = true diff --git a/adapters/codex/src/importer.rs b/adapters/codex/src/importer.rs index 8ef158b..81f7e91 100644 --- a/adapters/codex/src/importer.rs +++ b/adapters/codex/src/importer.rs @@ -6,6 +6,7 @@ use std::{ }; use autophagy_events::Event; +use autophagy_redaction::{PrivacyError, PrivacyPolicy}; use autophagy_store::{ EventStore, InsertOutcome, SearchProjection, SourceCursor, SourceIdentity, StoreError, }; @@ -31,6 +32,8 @@ pub struct CodexImportOptions { pub display_name: Option, /// Exact working directories to include; empty includes all. pub projects: Vec, + /// Glob patterns that exclude matching project or artifact paths. + pub exclude_paths: Vec, /// Persist prompt, assistant, and tool-result text in metadata. pub include_content: bool, /// Add tool input to the explicit search projection. @@ -52,6 +55,7 @@ impl CodexImportOptions { instance_key: instance_key.into(), display_name: None, projects: Vec::new(), + exclude_paths: Vec::new(), include_content: false, index_tool_input: false, index_metadata: Vec::new(), @@ -95,6 +99,10 @@ pub struct CodexImportSummary { pub conflicts: u64, /// Events excluded by project selection. pub skipped: u64, + /// Events excluded by path privacy policy. + pub privacy_skipped: u64, + /// String fields changed by default secret redaction. + pub redacted_fields: u64, /// Structurally valid unsupported records. pub unsupported: u64, /// Invalid records or supported-shape failures. @@ -132,6 +140,7 @@ pub fn import_codex( options: &CodexImportOptions, ) -> Result { validate_options(options)?; + let privacy = PrivacyPolicy::new(&options.exclude_paths)?; if !options.dry_run && store.is_none() { return Err(CodexImportError::MissingStore); } @@ -151,6 +160,8 @@ pub fn import_codex( duplicates: 0, conflicts: 0, skipped: 0, + privacy_skipped: 0, + redacted_fields: 0, unsupported: 0, rejected: 0, cursor_resets: 0, @@ -160,7 +171,7 @@ pub fn import_codex( diagnostics_suppressed: 0, dry_run: options.dry_run, }; - let scope = project_scope(&options.projects); + let scope = import_scope(&options.projects, &options.exclude_paths); for discovered in &discovery.files { summary.files_read += 1; @@ -256,6 +267,12 @@ pub fn import_codex( summary.skipped += 1; continue; } + let outcome = privacy.apply(&event); + let Some(event) = outcome.event else { + summary.privacy_skipped += 1; + continue; + }; + summary.redacted_fields += outcome.redacted_fields; if options.dry_run { continue; } @@ -320,10 +337,17 @@ fn validate_options(options: &CodexImportOptions) -> Result<(), CodexImportError Ok(()) } -fn project_scope(projects: &[String]) -> String { +fn import_scope(projects: &[String], exclude_paths: &[String]) -> String { let mut selected = projects.to_vec(); selected.sort(); - let digest = Sha256::digest(selected.join("\0").as_bytes()); + let mut excluded = exclude_paths.to_vec(); + excluded.sort(); + let scope = format!( + "privacy/v1\0projects\0{}\0exclusions\0{}", + selected.join("\0"), + excluded.join("\0") + ); + let digest = Sha256::digest(scope.as_bytes()); let mut encoded = String::with_capacity(digest.len() * 2); for byte in digest { write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); @@ -409,6 +433,9 @@ pub enum CodexImportError { /// Database operation failed. #[error(transparent)] Store(#[from] StoreError), + /// Privacy policy could not be compiled. + #[error(transparent)] + Privacy(#[from] PrivacyError), /// Cursor JSON could not be encoded. #[error("could not serialize Codex cursor state: {0}")] Json(#[from] serde_json::Error), diff --git a/crates/autophagy-cli/Cargo.toml b/crates/autophagy-cli/Cargo.toml index 09344c1..23523be 100644 --- a/crates/autophagy-cli/Cargo.toml +++ b/crates/autophagy-cli/Cargo.toml @@ -16,12 +16,16 @@ path = "src/main.rs" autophagy-adapter-claude-code = { path = "../../adapters/claude-code" } autophagy-adapter-codex = { path = "../../adapters/codex" } autophagy-core = { path = "../autophagy-core" } +autophagy-events = { path = "../autophagy-events" } +autophagy-patterns = { path = "../autophagy-patterns" } autophagy-store = { path = "../autophagy-store" } clap.workspace = true directories.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true +time.workspace = true [dev-dependencies] tempfile = "3.27" diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index 4ee54e4..7d2c03c 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -1,6 +1,7 @@ //! Command-line entry point for importing and querying local agent activity. use std::{ + fmt::Write as _, fs::{self, File}, io::{self, BufRead, BufReader, Write}, path::{Path, PathBuf}, @@ -14,10 +15,17 @@ use autophagy_adapter_codex::{ CodexImportOptions, CodexImportSummary, default_sessions_root, import_codex, }; use autophagy_core::{ImportOptions, ImportSummary, import_jsonl}; -use autophagy_store::{EventStore, SearchHit, SessionSummary, StoreError}; +use autophagy_events::Event; +use autophagy_patterns::{DetectorConfig, EvidencePacket, detect}; +use autophagy_store::{ + DeleteAllSummary, DeleteSummary, EventStore, PruneSummary, SearchHit, SessionSummary, + StoreError, +}; use clap::{Parser, Subcommand, ValueEnum}; use directories::ProjectDirs; use serde::Serialize; +use sha2::{Digest, Sha256}; +use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; #[derive(Debug, Parser)] #[command( @@ -76,6 +84,10 @@ enum Commands { #[arg(long = "project", value_name = "PATH")] projects: Vec, + /// Exclude project or artifact paths matching this glob. Repeatable. + #[arg(long = "exclude-path", value_name = "GLOB")] + exclude_paths: Vec, + /// Include Claude Code `agent-*.jsonl` subagent transcripts. #[arg(long)] include_subagents: bool, @@ -117,6 +129,95 @@ enum Commands { #[arg(long, default_value_t = 20, value_name = "COUNT")] limit: u32, }, + + /// Run every deterministic detector and emit a digestion report. + Digest { + /// Limit digestion to one exact project path. + #[arg(long, value_name = "PATH")] + project: Option, + + #[command(flatten)] + thresholds: ThresholdArgs, + }, + + /// List deterministic Evidence Packet v0.1 findings. + Patterns { + /// Limit detection to one exact project path. + #[arg(long, value_name = "PATH")] + project: Option, + + #[command(flatten)] + thresholds: ThresholdArgs, + }, + + /// Export redacted canonical AEP events as JSONL to standard output. + Export { + /// Limit export to one exact project path. + #[arg(long, value_name = "PATH")] + project: Option, + }, + + /// Apply an age-based session retention policy. + Prune { + /// Delete sessions whose last event is older than this many days. + #[arg(long, value_name = "DAYS")] + older_than_days: u32, + + /// Limit pruning to one exact project path. + #[arg(long, value_name = "PATH")] + project: Option, + + /// Report the exact deletion effect and roll the transaction back. + #[arg(long)] + dry_run: bool, + }, + + /// Delete a session or all local Autophagy data. + Delete { + #[command(subcommand)] + target: DeleteTarget, + }, +} + +#[allow(clippy::struct_field_names)] +#[derive(Clone, Copy, Debug, clap::Args)] +struct ThresholdArgs { + /// Minimum supporting events. + #[arg(long, default_value_t = 3, value_name = "COUNT")] + min_occurrences: u32, + + /// Minimum distinct supporting sessions. + #[arg(long, default_value_t = 2, value_name = "COUNT")] + min_sessions: u32, + + /// Minimum support share in basis points (0-10000). + #[arg(long, default_value_t = 5_000, value_parser = clap::value_parser!(u16).range(0..=10_000), value_name = "BPS")] + min_support_ratio_bps: u16, +} + +impl From for DetectorConfig { + fn from(value: ThresholdArgs) -> Self { + Self { + min_occurrences: value.min_occurrences, + min_sessions: value.min_sessions, + min_support_ratio_bps: value.min_support_ratio_bps, + } + } +} + +#[derive(Debug, Subcommand)] +enum DeleteTarget { + /// Delete one session and its evidence. + Session { + /// AEP session identifier. + session_id: String, + }, + /// Delete every local source, cursor, session, event, and artifact. + All { + /// Required destructive confirmation phrase: `delete-all`. + #[arg(long, value_name = "PHRASE")] + confirm: String, + }, } #[derive(Debug, Serialize)] @@ -125,6 +226,22 @@ enum CommandReport { Import(ImportReport), Sessions(Vec), Search(Vec), + Digest(DigestReport), + Patterns(Vec), + Export(Vec), + Prune(PruneSummary), + DeleteSession(DeleteSummary), + DeleteAll(DeleteAllSummary), +} + +#[derive(Debug, Serialize)] +struct DigestReport { + spec_version: &'static str, + generated_at: String, + events_scanned: usize, + model_used: bool, + network_used: bool, + findings: Vec, } #[derive(Debug, Serialize)] @@ -149,7 +266,14 @@ impl CommandReport { const fn has_issues(&self) -> bool { match self { Self::Import(summary) => summary.has_issues(), - Self::Sessions(_) | Self::Search(_) => false, + Self::Sessions(_) + | Self::Search(_) + | Self::Digest(_) + | Self::Patterns(_) + | Self::Export(_) + | Self::Prune(_) + | Self::DeleteSession(_) + | Self::DeleteAll(_) => false, } } } @@ -174,6 +298,10 @@ enum CliError { Json(#[from] serde_json::Error), #[error("could not determine the platform-local application data directory")] DataDirectoryUnavailable, + #[error("could not format report timestamp: {0}")] + TimeFormat(#[from] time::error::Format), + #[error("deleting all data requires --confirm delete-all")] + DeleteAllConfirmation, } fn main() -> ExitCode { @@ -193,6 +321,7 @@ fn main() -> ExitCode { } } +#[allow(clippy::too_many_lines)] fn execute(cli: Cli) -> Result { match cli.command { Commands::Import { @@ -201,6 +330,7 @@ fn execute(cli: Cli) -> Result { instance_key, display_name, projects, + exclude_paths, include_subagents, include_content, index_tool_input, @@ -213,6 +343,7 @@ fn execute(cli: Cli) -> Result { let mut options = ImportOptions::new(instance_key); options.display_name = display_name; options.projects = projects; + options.exclude_paths = exclude_paths; options.index_tool_input = index_tool_input; options.index_metadata = index_metadata; options.dry_run = dry_run; @@ -237,6 +368,7 @@ fn execute(cli: Cli) -> Result { let mut options = ClaudeImportOptions::new(input, instance_key); options.display_name = display_name; options.projects = projects; + options.exclude_paths = exclude_paths; options.include_subagents = include_subagents; options.include_content = include_content; options.index_tool_input = index_tool_input; @@ -262,6 +394,7 @@ fn execute(cli: Cli) -> Result { let mut options = CodexImportOptions::new(input, instance_key); options.display_name = display_name; options.projects = projects; + options.exclude_paths = exclude_paths; options.include_content = include_content; options.index_tool_input = index_tool_input; options.index_metadata = index_metadata; @@ -287,6 +420,68 @@ fn execute(cli: Cli) -> Result { let store = open_store(&database)?; Ok(CommandReport::Search(store.search(&query, limit)?)) } + Commands::Digest { + project, + thresholds, + } => { + let database = resolve_database_path(cli.database)?; + let store = open_store(&database)?; + let events = store.list_events_for_detection(project.as_deref())?; + let findings = detect(&events, thresholds.into()); + Ok(CommandReport::Digest(DigestReport { + spec_version: "digest/0.1", + generated_at: OffsetDateTime::now_utc().format(&Rfc3339)?, + events_scanned: events.len(), + model_used: false, + network_used: false, + findings, + })) + } + Commands::Patterns { + project, + thresholds, + } => { + let database = resolve_database_path(cli.database)?; + let store = open_store(&database)?; + let events = store.list_events_for_detection(project.as_deref())?; + Ok(CommandReport::Patterns(detect(&events, thresholds.into()))) + } + Commands::Export { project } => { + let database = resolve_database_path(cli.database)?; + let store = open_store(&database)?; + Ok(CommandReport::Export( + store.list_events_for_detection(project.as_deref())?, + )) + } + Commands::Prune { + older_than_days, + project, + dry_run, + } => { + let database = resolve_database_path(cli.database)?; + let mut store = open_store(&database)?; + let cutoff = OffsetDateTime::now_utc() - Duration::days(i64::from(older_than_days)); + Ok(CommandReport::Prune(store.prune_before( + cutoff, + project.as_deref(), + dry_run, + )?)) + } + Commands::Delete { target } => { + let database = resolve_database_path(cli.database)?; + let mut store = open_store(&database)?; + match target { + DeleteTarget::Session { session_id } => Ok(CommandReport::DeleteSession( + store.delete_session(&session_id)?, + )), + DeleteTarget::All { confirm } => { + if confirm != "delete-all" { + return Err(CliError::DeleteAllConfirmation); + } + Ok(CommandReport::DeleteAll(store.delete_all()?)) + } + } + } } } @@ -321,7 +516,13 @@ fn derive_instance_key(input: &Path) -> Result { if input == Path::new("-") { Ok("stdin".to_owned()) } else { - Ok(fs::canonicalize(input)?.to_string_lossy().into_owned()) + let canonical = fs::canonicalize(input)?; + let digest = Sha256::digest(canonical.to_string_lossy().as_bytes()); + let mut encoded = String::with_capacity(digest.len() * 2); + for byte in digest { + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + Ok(format!("path:{encoded}")) } } @@ -330,6 +531,13 @@ fn write_report( format: OutputFormat, report: &CommandReport, ) -> Result<(), CliError> { + if let CommandReport::Export(events) = report { + for event in events { + serde_json::to_writer(&mut writer, event)?; + writeln!(writer)?; + } + return Ok(()); + } match format { OutputFormat::Json => { serde_json::to_writer_pretty(&mut writer, report)?; @@ -362,18 +570,74 @@ fn write_report( writeln!(writer, "{}\t{}", hit.event_id, hit.snippet)?; } } + CommandReport::Digest(report) => { + writeln!( + writer, + "{} events · {} findings · local deterministic digest", + report.events_scanned, + report.findings.len() + )?; + write_findings(&mut writer, &report.findings)?; + } + CommandReport::Patterns(findings) => write_findings(&mut writer, findings)?, + CommandReport::Prune(summary) => writeln!( + writer, + "{} sessions · {} events · {} artifacts{}", + summary.sessions_deleted, + summary.events_deleted, + summary.artifacts_deleted, + if summary.dry_run { + " · dry run" + } else { + " deleted" + } + )?, + CommandReport::DeleteSession(summary) => writeln!( + writer, + "session_deleted={} · {} events · {} artifacts", + summary.session_deleted, summary.events_deleted, summary.artifacts_deleted + )?, + CommandReport::DeleteAll(summary) => writeln!( + writer, + "{} sources · {} sessions · {} events · {} artifacts · {} conflicts · {} cursors deleted", + summary.sources_deleted, + summary.sessions_deleted, + summary.events_deleted, + summary.artifacts_deleted, + summary.conflicts_deleted, + summary.cursors_deleted + )?, + CommandReport::Export(_) => unreachable!("export handled before format selection"), }, } Ok(()) } +fn write_findings(writer: &mut impl Write, findings: &[EvidencePacket]) -> io::Result<()> { + if findings.is_empty() { + writeln!(writer, "no findings above threshold")?; + } + for finding in findings { + writeln!( + writer, + "{}\t{}\t{} bps\t{} evidence\t{} counterexamples", + finding.finding_id, + finding.title, + finding.score.score_bps, + finding.evidence.len(), + finding.counterexamples.len() + )?; + } + Ok(()) +} + fn write_codex_import_summary( writer: &mut impl Write, summary: &CodexImportSummary, ) -> io::Result<()> { writeln!( writer, - "{} files · {} records · {} events · {} inserted · {} duplicates · {} conflicts · {} unsupported · {} rejected{}", + "{} files · {} records · {} events · {} inserted · {} duplicates · {} conflicts · {} unsupported · {} privacy excluded · {} redacted fields · {} rejected{}", summary.discovery.files.len(), summary.records_seen, summary.events_emitted, @@ -381,6 +645,8 @@ fn write_codex_import_summary( summary.duplicates, summary.conflicts, summary.unsupported, + summary.privacy_skipped, + summary.redacted_fields, summary.rejected, if summary.dry_run { " · dry run" } else { "" } )?; @@ -410,7 +676,7 @@ fn write_claude_import_summary( ) -> io::Result<()> { writeln!( writer, - "{} files · {} records · {} events · {} inserted · {} duplicates · {} conflicts · {} unsupported · {} rejected{}", + "{} files · {} records · {} events · {} inserted · {} duplicates · {} conflicts · {} unsupported · {} privacy excluded · {} redacted fields · {} rejected{}", summary.discovery.files.len(), summary.records_seen, summary.events_emitted, @@ -418,6 +684,8 @@ fn write_claude_import_summary( summary.duplicates, summary.conflicts, summary.unsupported, + summary.privacy_skipped, + summary.redacted_fields, summary.rejected, if summary.dry_run { " · dry run" } else { "" } )?; @@ -444,13 +712,15 @@ fn write_claude_import_summary( fn write_import_summary(writer: &mut impl Write, summary: &ImportSummary) -> io::Result<()> { writeln!( writer, - "{} lines · {} events · {} inserted · {} duplicates · {} conflicts · {} skipped · {} rejected{}", + "{} lines · {} events · {} inserted · {} duplicates · {} conflicts · {} skipped · {} privacy excluded · {} redacted fields · {} rejected{}", summary.lines_read, summary.events_seen, summary.inserted, summary.duplicates, summary.conflicts, summary.skipped, + summary.privacy_skipped, + summary.redacted_fields, summary.rejected, if summary.dry_run { " · dry run" } else { "" } )?; diff --git a/crates/autophagy-cli/tests/cli.rs b/crates/autophagy-cli/tests/cli.rs index 1c7e27a..3da278e 100644 --- a/crates/autophagy-cli/tests/cli.rs +++ b/crates/autophagy-cli/tests/cli.rs @@ -42,6 +42,7 @@ fn import_sessions_search_and_reimport_work_end_to_end() { assert_eq!(sessions["result"].as_array().expect("sessions").len(), 1); assert_eq!(sessions["result"][0]["session_id"], "ses_cli"); assert_eq!(sessions["result"][0]["event_count"], 2); + assert_eq!(sessions["result"][0]["instance_key"], "fixture:cli"); let search = run_json(&database, ["search", "generated"]); assert_eq!(search["result"].as_array().expect("hits").len(), 1); @@ -175,6 +176,130 @@ fn codex_rollouts_import_and_reimport_incrementally() { ); } +#[test] +fn milestone_demo_digests_exports_deletes_and_prunes_offline() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("autophagy.db"); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../evals/fixtures/findings/deterministic.jsonl"); + let imported = run_json(&database, ["import", fixture.to_str().expect("UTF-8 path")]); + assert_eq!(imported["result"]["inserted"], 11); + + let patterns = run_json(&database, ["patterns"]); + let findings = patterns["result"].as_array().expect("findings"); + assert_eq!(findings.len(), 2); + assert!( + findings + .iter() + .all(|finding| finding["evidence"].as_array().expect("evidence").len() == 3) + ); + + let digest = run_json(&database, ["digest"]); + assert_eq!(digest["result"]["spec_version"], "digest/0.1"); + assert_eq!(digest["result"]["events_scanned"], 11); + assert_eq!(digest["result"]["model_used"], false); + assert_eq!(digest["result"]["network_used"], false); + assert_eq!( + digest["result"]["findings"] + .as_array() + .expect("findings") + .len(), + 2 + ); + + let exported = command(&database).arg("export").output().expect("export"); + assert!(exported.status.success()); + let lines = String::from_utf8(exported.stdout).expect("UTF-8 export"); + assert_eq!(lines.lines().count(), 11); + for line in lines.lines() { + autophagy_events::Event::from_json_str(line).expect("valid exported AEP event"); + } + + let deleted = run_json(&database, ["delete", "session", "ses_failure_1"]); + assert_eq!(deleted["result"]["session_deleted"], true); + assert_eq!(deleted["result"]["events_deleted"], 1); + + let preview = run_json(&database, ["prune", "--older-than-days", "0", "--dry-run"]); + assert_eq!(preview["result"]["events_deleted"], 10); + assert_eq!(preview["result"]["dry_run"], true); + assert_eq!( + run_json(&database, ["sessions"])["result"] + .as_array() + .expect("sessions") + .len(), + 9 + ); + + let pruned = run_json(&database, ["prune", "--older-than-days", "0"]); + assert_eq!(pruned["result"]["events_deleted"], 10); + assert_eq!(pruned["result"]["dry_run"], false); + assert!( + run_json(&database, ["sessions"])["result"] + .as_array() + .expect("sessions") + .is_empty() + ); +} + +#[test] +fn import_redacts_secrets_excludes_paths_and_requires_delete_confirmation() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("autophagy.db"); + let input = directory.path().join("privacy.jsonl"); + fs::write(&input, concat!( + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_cli_secret\",\"session_id\":\"ses_cli_secret\",\"timestamp\":\"2026-07-16T09:00:00Z\",\"source\":\"generic-jsonl\",\"type\":\"tool.called\",\"project\":\"/repo/public\",\"tool\":{\"name\":\"shell\",\"input\":{\"command\":\"API_KEY=abcdefgh12345678\"}}}\n", + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_cli_private\",\"session_id\":\"ses_cli_private\",\"timestamp\":\"2026-07-16T09:01:00Z\",\"source\":\"generic-jsonl\",\"type\":\"session.started\",\"project\":\"/repo/private/client\"}\n" + )).expect("privacy fixture"); + let imported = run_json( + &database, + [ + "import", + input.to_str().expect("UTF-8 path"), + "--exclude-path", + "**/private/**", + ], + ); + assert_eq!(imported["result"]["inserted"], 1); + assert_eq!(imported["result"]["privacy_skipped"], 1); + assert_eq!(imported["result"]["redacted_fields"], 1); + let sessions = run_json(&database, ["sessions"]); + assert!( + sessions["result"][0]["instance_key"] + .as_str() + .expect("instance key") + .starts_with("path:") + ); + assert!( + !sessions["result"][0]["instance_key"] + .as_str() + .expect("instance key") + .contains(directory.path().to_str().expect("path")) + ); + + let exported = command(&database).arg("export").output().expect("export"); + let exported = String::from_utf8(exported.stdout).expect("UTF-8"); + assert!(exported.contains("[REDACTED]")); + assert!(!exported.contains("abcdefgh12345678")); + assert!(!exported.contains("evt_cli_private")); + + let refused = command(&database) + .args(["delete", "all", "--confirm", "nope"]) + .output() + .expect("refused delete"); + assert!(!refused.status.success()); + assert_eq!( + run_json(&database, ["sessions"])["result"] + .as_array() + .expect("sessions") + .len(), + 1 + ); + + let deleted = run_json(&database, ["delete", "all", "--confirm", "delete-all"]); + assert_eq!(deleted["result"]["events_deleted"], 1); + assert_eq!(deleted["result"]["sessions_deleted"], 1); +} + fn run_json(database: &Path, args: [&str; N]) -> Value { let output = command(database) .args(["--output", "json"]) diff --git a/crates/autophagy-core/Cargo.toml b/crates/autophagy-core/Cargo.toml index f0f480f..b77cc73 100644 --- a/crates/autophagy-core/Cargo.toml +++ b/crates/autophagy-core/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] autophagy-events = { path = "../autophagy-events" } +autophagy-redaction = { path = "../autophagy-redaction" } autophagy-store = { path = "../autophagy-store" } serde.workspace = true serde_json.workspace = true diff --git a/crates/autophagy-core/src/generic_jsonl.rs b/crates/autophagy-core/src/generic_jsonl.rs index 8a3480f..39695d8 100644 --- a/crates/autophagy-core/src/generic_jsonl.rs +++ b/crates/autophagy-core/src/generic_jsonl.rs @@ -1,6 +1,7 @@ use std::io::BufRead; use autophagy_events::{Event, EventParseError}; +use autophagy_redaction::{PrivacyError, PrivacyPolicy}; use autophagy_store::{EventStore, InsertOutcome, SearchProjection, SourceIdentity, StoreError}; use serde::Serialize; @@ -13,6 +14,8 @@ pub struct ImportOptions { pub display_name: Option, /// Exact project paths to include. An empty list includes every project. pub projects: Vec, + /// Glob patterns that exclude matching project or artifact paths. + pub exclude_paths: Vec, /// Whether already-redacted tool input may enter FTS5. pub index_tool_input: bool, /// Explicit metadata keys whose already-redacted values may enter FTS5. @@ -31,6 +34,7 @@ impl ImportOptions { instance_key: instance_key.into(), display_name: None, projects: Vec::new(), + exclude_paths: Vec::new(), index_tool_input: false, index_metadata: Vec::new(), dry_run: false, @@ -91,6 +95,10 @@ pub struct ImportSummary { pub conflicts: u64, /// Valid events excluded by project selection. pub skipped: u64, + /// Selected events excluded by path privacy policy. + pub privacy_skipped: u64, + /// String fields changed by default secret redaction. + pub redacted_fields: u64, /// Invalid or store-rejected records. pub rejected: u64, /// Retained line-addressed diagnostics. @@ -118,6 +126,9 @@ pub enum ImportError { /// A database or migration operation failed. #[error("event store failed: {0}")] Store(#[from] StoreError), + /// Privacy policy could not be compiled. + #[error(transparent)] + Privacy(#[from] PrivacyError), /// Non-dry imports require a writable store. #[error("a writable event store is required unless dry_run is enabled")] MissingStore, @@ -142,6 +153,7 @@ pub fn import_jsonl( options: &ImportOptions, ) -> Result { validate_options(options)?; + let privacy = PrivacyPolicy::new(&options.exclude_paths)?; if !options.dry_run && store.is_none() { return Err(ImportError::MissingStore); } @@ -181,6 +193,12 @@ pub fn import_jsonl( summary.skipped += 1; continue; } + let outcome = privacy.apply(&event); + let Some(event) = outcome.event else { + summary.privacy_skipped += 1; + continue; + }; + summary.redacted_fields += outcome.redacted_fields; summary.validated += 1; if options.dry_run { continue; diff --git a/crates/autophagy-core/tests/generic_jsonl.rs b/crates/autophagy-core/tests/generic_jsonl.rs index 7e7f29c..fedf039 100644 --- a/crates/autophagy-core/tests/generic_jsonl.rs +++ b/crates/autophagy-core/tests/generic_jsonl.rs @@ -75,6 +75,36 @@ fn tool_input_indexing_requires_explicit_opt_in() { assert_eq!(store.search("approved", 10).expect("search").len(), 1); } +#[test] +fn privacy_policy_redacts_secrets_and_excludes_paths_before_storage() { + let input = concat!( + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_secret\",", + "\"session_id\":\"ses_secret\",\"timestamp\":\"2026-07-16T09:00:00Z\",", + "\"source\":\"generic-jsonl\",\"type\":\"tool.called\",", + "\"project\":\"/repo/public\",\"tool\":{\"name\":\"shell\",", + "\"input\":{\"command\":\"API_KEY=abcdefgh12345678\"}}}\n", + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_private\",", + "\"session_id\":\"ses_private\",\"timestamp\":\"2026-07-16T09:01:00Z\",", + "\"source\":\"generic-jsonl\",\"type\":\"session.started\",", + "\"project\":\"/repo/private/client\"}\n" + ); + let mut store = EventStore::open_in_memory().expect("store"); + let mut options = ImportOptions::new("fixture:privacy"); + options.exclude_paths = vec!["**/private/**".to_owned()]; + let summary = import_jsonl(Cursor::new(input), Some(&mut store), &options).expect("import"); + assert_eq!(summary.inserted, 1); + assert_eq!(summary.privacy_skipped, 1); + assert_eq!(summary.redacted_fields, 1); + let stored = store + .get_event("evt_secret") + .expect("query") + .expect("event"); + let encoded = serde_json::to_string(&stored).expect("JSON"); + assert!(encoded.contains("[REDACTED]")); + assert!(!encoded.contains("abcdefgh12345678")); + assert!(store.get_event("evt_private").expect("query").is_none()); +} + #[test] fn dry_run_validates_without_a_store_or_writes() { let mut options = ImportOptions::new("fixture:dry-run"); diff --git a/crates/autophagy-redaction/Cargo.toml b/crates/autophagy-redaction/Cargo.toml new file mode 100644 index 0000000..8b417f9 --- /dev/null +++ b/crates/autophagy-redaction/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "autophagy-redaction" +description = "Local secret redaction and path exclusion policy for Autophagy" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +autophagy-events = { path = "../autophagy-events" } +globset.workspace = true +regex.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +time.workspace = true + +[lints] +workspace = true diff --git a/crates/autophagy-redaction/src/lib.rs b/crates/autophagy-redaction/src/lib.rs new file mode 100644 index 0000000..2b5aab6 --- /dev/null +++ b/crates/autophagy-redaction/src/lib.rs @@ -0,0 +1,238 @@ +//! Deterministic privacy enforcement before event persistence. + +use autophagy_events::Event; +use globset::{Glob, GlobSet, GlobSetBuilder}; +use regex::Regex; +use serde_json::Value; + +/// Replacement written in place of recognized credential material. +pub const REDACTED: &str = "[REDACTED]"; + +/// Compiled path and secret policy for one import invocation. +pub struct PrivacyPolicy { + exclusions: GlobSet, + rules: Vec, +} + +struct SecretRule { + expression: Regex, + replacement: &'static str, +} + +/// Outcome of applying privacy policy to one normalized event. +#[derive(Clone, Debug, PartialEq)] +pub struct PrivacyOutcome { + /// Sanitized event, or `None` when path policy excluded it. + pub event: Option, + /// Number of string fields changed by secret rules. + pub redacted_fields: u64, +} + +impl PrivacyPolicy { + /// Compile a conservative default secret policy and caller path exclusions. + /// + /// # Errors + /// Returns an error when an exclusion is not a valid glob expression. + pub fn new(exclude_paths: &[String]) -> Result { + let mut builder = GlobSetBuilder::new(); + for pattern in exclude_paths { + if pattern.trim().is_empty() { + return Err(PrivacyError::BlankExclusion); + } + builder.add( + Glob::new(pattern).map_err(|source| PrivacyError::InvalidGlob { + pattern: pattern.clone(), + source, + })?, + ); + } + Ok(Self { + exclusions: builder.build()?, + rules: default_rules(), + }) + } + + /// Exclude path-matched events and redact secrets from retained payloads. + #[must_use] + pub fn apply(&self, event: &Event) -> PrivacyOutcome { + if event + .project + .as_deref() + .is_some_and(|path| self.exclusions.is_match(path)) + || event.artifacts.iter().any(|artifact| { + artifact + .path + .as_deref() + .is_some_and(|path| self.exclusions.is_match(path)) + }) + { + return PrivacyOutcome { + event: None, + redacted_fields: 0, + }; + } + + let mut event = event.clone(); + let mut redacted_fields = 0; + if let Some(tool) = &mut event.tool { + if let Some(input) = &mut tool.input { + redact_value(input, &self.rules, &mut redacted_fields); + } + for value in tool.metadata.values_mut() { + redact_value(value, &self.rules, &mut redacted_fields); + } + } + for value in event.metadata.values_mut() { + redact_value(value, &self.rules, &mut redacted_fields); + } + for artifact in &mut event.artifacts { + if let Some(path) = &mut artifact.path { + redact_string(path, &self.rules, &mut redacted_fields); + } + if let Some(uri) = &mut artifact.uri { + redact_string(uri, &self.rules, &mut redacted_fields); + } + for value in artifact.metadata.values_mut() { + redact_value(value, &self.rules, &mut redacted_fields); + } + } + PrivacyOutcome { + event: Some(event), + redacted_fields, + } + } +} + +fn default_rules() -> Vec { + [ + (r"\bsk-[A-Za-z0-9_-]{16,}\b", REDACTED), + (r"\bgh[pousr]_[A-Za-z0-9]{20,}\b", REDACTED), + (r"\bAKIA[A-Z0-9]{16}\b", REDACTED), + (r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{16,}", "Bearer [REDACTED]"), + ( + r#"(?i)\b(api[_-]?key|access[_-]?token|password|secret)\s*[:=]\s*["']?[A-Za-z0-9._~+/=-]{8,}["']?"#, + "$1=[REDACTED]", + ), + ] + .into_iter() + .map(|(expression, replacement)| SecretRule { + expression: Regex::new(expression).expect("built-in secret regex must compile"), + replacement, + }) + .collect() +} + +fn redact_value(value: &mut Value, rules: &[SecretRule], count: &mut u64) { + match value { + Value::String(value) => redact_string(value, rules, count), + Value::Array(values) => { + for value in values { + redact_value(value, rules, count); + } + } + Value::Object(values) => { + for value in values.values_mut() { + redact_value(value, rules, count); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + +fn redact_string(value: &mut String, rules: &[SecretRule], count: &mut u64) { + let original = value.clone(); + for rule in rules { + *value = rule + .expression + .replace_all(value, rule.replacement) + .into_owned(); + } + if *value != original { + *count += 1; + } +} + +/// Privacy policy compilation failure. +#[derive(Debug, thiserror::Error)] +pub enum PrivacyError { + /// Empty patterns are rejected instead of matching unpredictably. + #[error("path exclusion must not be blank")] + BlankExclusion, + /// A caller-supplied glob was malformed. + #[error("invalid path exclusion '{pattern}': {source}")] + InvalidGlob { + /// Rejected pattern. + pattern: String, + /// Glob parser failure. + source: globset::Error, + }, + /// Glob set construction failed. + #[error("could not compile path exclusions: {0}")] + GlobSet(#[from] globset::Error), +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use autophagy_events::{Event, EventId, EventKind, SessionId, SpecVersion, ToolCall}; + use serde_json::json; + use time::OffsetDateTime; + + use super::*; + + #[test] + fn redacts_nested_secrets_without_changing_source_event() { + let event = fixture_event("/repo/public"); + let outcome = PrivacyPolicy::new(&[]).expect("policy").apply(&event); + let sanitized = outcome.event.expect("retained"); + assert_eq!(outcome.redacted_fields, 2); + let encoded = serde_json::to_string(&sanitized).expect("JSON"); + assert!(!encoded.contains("sk-abcdefghijklmnop")); + assert!(!encoded.contains("ghp_abcdefghijklmnopqrstuvwxyz")); + assert!(encoded.contains(REDACTED)); + assert!( + serde_json::to_string(&event) + .expect("source JSON") + .contains("sk-abcdefghijklmnop") + ); + } + + #[test] + fn excludes_projects_and_artifacts_by_glob() { + let policy = PrivacyPolicy::new(&["**/private/**".to_owned()]).expect("policy"); + assert!( + policy + .apply(&fixture_event("/repo/private/client")) + .event + .is_none() + ); + assert!(policy.apply(&fixture_event("/repo/public")).event.is_some()); + } + + fn fixture_event(project: &str) -> Event { + Event { + spec_version: SpecVersion::V0_1, + event_id: EventId::new("evt_redaction"), + session_id: SessionId::new("ses_redaction"), + timestamp: OffsetDateTime::UNIX_EPOCH, + sequence: Some(0), + source: "fixture".to_owned(), + kind: EventKind::ToolCalled, + project: Some(project.to_owned()), + parent_event_id: None, + tool: Some(ToolCall { + name: "shell".to_owned(), + input: Some(json!({"command":"API_KEY=abcdefgh12345678 sk-abcdefghijklmnop"})), + exit_code: None, + duration_ms: None, + metadata: BTreeMap::new(), + }), + artifacts: Vec::new(), + metadata: BTreeMap::from([( + "token".to_owned(), + json!("ghp_abcdefghijklmnopqrstuvwxyz"), + )]), + } + } +} diff --git a/crates/autophagy-store/src/lib.rs b/crates/autophagy-store/src/lib.rs index 4b2b99f..3949549 100644 --- a/crates/autophagy-store/src/lib.rs +++ b/crates/autophagy-store/src/lib.rs @@ -12,7 +12,7 @@ mod util; pub use error::StoreError; pub use model::{ - DeleteSummary, InsertOutcome, SearchHit, SearchProjection, SessionSummary, SourceCursor, - SourceIdentity, StoreStats, + DeleteAllSummary, DeleteSummary, InsertOutcome, PruneSummary, SearchHit, SearchProjection, + SessionSummary, SourceCursor, SourceIdentity, StoreStats, }; pub use store::EventStore; diff --git a/crates/autophagy-store/src/model.rs b/crates/autophagy-store/src/model.rs index 311c828..3299501 100644 --- a/crates/autophagy-store/src/model.rs +++ b/crates/autophagy-store/src/model.rs @@ -137,3 +137,33 @@ pub struct DeleteSummary { /// Number of artifacts that became unreferenced and were removed. pub artifacts_deleted: i64, } + +/// Effect of deleting all locally persisted Autophagy data. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct DeleteAllSummary { + /// Removed source identities. + pub sources_deleted: i64, + /// Removed sessions. + pub sessions_deleted: i64, + /// Removed canonical events. + pub events_deleted: i64, + /// Removed artifacts. + pub artifacts_deleted: i64, + /// Removed conflict records. + pub conflicts_deleted: i64, + /// Removed incremental source cursors. + pub cursors_deleted: i64, +} + +/// Effect or dry-run preview of a retention prune. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct PruneSummary { + /// Sessions older than the cutoff. + pub sessions_deleted: i64, + /// Events belonging to selected sessions. + pub events_deleted: i64, + /// Artifacts left unreferenced by selected sessions. + pub artifacts_deleted: i64, + /// Whether the transaction was intentionally rolled back. + pub dry_run: bool, +} diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index d4c66d2..4917292 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -2,10 +2,11 @@ use std::{path::Path, time::Duration}; use autophagy_events::{Event, EventKind}; use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params}; +use time::OffsetDateTime; use crate::{ - DeleteSummary, InsertOutcome, SearchHit, SearchProjection, SessionSummary, SourceCursor, - SourceIdentity, StoreError, StoreStats, migration, util, + DeleteAllSummary, DeleteSummary, InsertOutcome, PruneSummary, SearchHit, SearchProjection, + SessionSummary, SourceCursor, SourceIdentity, StoreError, StoreStats, migration, util, }; /// Transactional owner of one local Autophagy `SQLite` database. @@ -504,6 +505,104 @@ impl EventStore { }) } + /// Delete every local event, source, cursor, conflict, and artifact. + /// + /// # Errors + /// + /// Returns [`StoreError`] when the deletion transaction fails. + pub fn delete_all(&mut self) -> Result { + let before = self.stats()?; + let cursors_deleted = + self.connection + .query_row("SELECT count(*) FROM source_cursors", [], |row| row.get(0))?; + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute("DELETE FROM imports", [])?; + transaction.execute("DELETE FROM source_cursors", [])?; + transaction.execute("DELETE FROM sessions", [])?; + transaction.execute("DELETE FROM artifacts", [])?; + transaction.execute("DELETE FROM sources", [])?; + transaction.commit()?; + Ok(DeleteAllSummary { + sources_deleted: before.sources, + sessions_deleted: before.sessions, + events_deleted: before.events, + artifacts_deleted: before.artifacts, + conflicts_deleted: before.conflicts, + cursors_deleted, + }) + } + + /// Delete sessions whose last event is strictly older than a cutoff. + /// + /// Dry runs execute the same transaction and roll it back, so reported + /// artifact counts match a real prune. An exact project limits selection. + /// + /// # Errors + /// + /// Returns [`StoreError`] when timestamp formatting or the retention + /// transaction fails. + pub fn prune_before( + &mut self, + cutoff: OffsetDateTime, + project: Option<&str>, + dry_run: bool, + ) -> Result { + let cutoff = util::canonical_timestamp(cutoff)?; + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let sessions_deleted = transaction.query_row( + "SELECT count(*) FROM sessions + WHERE last_event_at < ?1 AND (?2 IS NULL OR project_path = ?2)", + params![cutoff, project], + |row| row.get::<_, i64>(0), + )?; + let events_deleted = transaction.query_row( + "SELECT count(*) FROM events + WHERE session_id IN ( + SELECT session_id FROM sessions + WHERE last_event_at < ?1 AND (?2 IS NULL OR project_path = ?2) + )", + params![cutoff, project], + |row| row.get::<_, i64>(0), + )?; + let artifacts_before = + transaction.query_row("SELECT count(*) FROM artifacts", [], |row| { + row.get::<_, i64>(0) + })?; + transaction.execute( + "DELETE FROM sessions + WHERE last_event_at < ?1 AND (?2 IS NULL OR project_path = ?2)", + params![cutoff, project], + )?; + transaction.execute( + "DELETE FROM artifacts + WHERE NOT EXISTS ( + SELECT 1 FROM event_artifacts + WHERE event_artifacts.artifact_id = artifacts.artifact_id + )", + [], + )?; + let artifacts_after = + transaction.query_row("SELECT count(*) FROM artifacts", [], |row| { + row.get::<_, i64>(0) + })?; + let summary = PruneSummary { + sessions_deleted, + events_deleted, + artifacts_deleted: artifacts_before - artifacts_after, + dry_run, + }; + if dry_run { + transaction.rollback()?; + } else { + transaction.commit()?; + } + Ok(summary) + } + fn from_connection(mut connection: Connection) -> Result { configure(&connection)?; migration::apply(&mut connection)?; diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index 9dc6397..0a56dff 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -6,8 +6,8 @@ use autophagy_events::{ Artifact, ArtifactKind, Event, EventId, EventKind, SessionId, SpecVersion, ToolCall, }; use autophagy_store::{ - DeleteSummary, EventStore, InsertOutcome, SearchProjection, SourceCursor, SourceIdentity, - StoreError, StoreStats, + DeleteAllSummary, DeleteSummary, EventStore, InsertOutcome, PruneSummary, SearchProjection, + SourceCursor, SourceIdentity, StoreError, StoreStats, }; use serde_json::{Value, json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -443,6 +443,80 @@ fn invalid_events_are_rejected_before_storage() { )); } +#[test] +fn retention_preview_rolls_back_and_delete_all_removes_local_state() { + let mut store = EventStore::open_in_memory().expect("store"); + let source = source("instance-retention"); + let mut old = tool_failure( + "evt_retention-old", + "ses_retention-old", + "2026-07-01T00:00:00Z", + 0, + ); + old.artifacts.push(file_artifact("old.log")); + store + .insert_event(&source, &old, &SearchProjection::default()) + .expect("old event"); + store + .insert_event( + &source, + &session_event( + "evt_retention-new", + "ses_retention-new", + EventKind::SessionStarted, + "2026-07-15T00:00:00Z", + 0, + ), + &SearchProjection::default(), + ) + .expect("new event"); + store + .save_source_cursor( + &source, + "retention.jsonl", + &SourceCursor { + byte_offset: 10, + line_number: 1, + head_hash: [1; 32], + state: json!({}), + }, + ) + .expect("cursor"); + let cutoff = OffsetDateTime::parse("2026-07-10T00:00:00Z", &Rfc3339).expect("cutoff"); + assert_eq!( + store.prune_before(cutoff, None, true).expect("preview"), + PruneSummary { + sessions_deleted: 1, + events_deleted: 1, + artifacts_deleted: 1, + dry_run: true, + } + ); + assert_eq!(store.stats().expect("stats after preview").events, 2); + assert_eq!( + store.prune_before(cutoff, None, false).expect("prune"), + PruneSummary { + sessions_deleted: 1, + events_deleted: 1, + artifacts_deleted: 1, + dry_run: false, + } + ); + assert_eq!(store.stats().expect("stats after prune").events, 1); + assert_eq!( + store.delete_all().expect("delete all"), + DeleteAllSummary { + sources_deleted: 1, + sessions_deleted: 1, + events_deleted: 1, + artifacts_deleted: 0, + conflicts_deleted: 0, + cursors_deleted: 1, + } + ); + assert_eq!(store.stats().expect("empty stats"), StoreStats::default()); +} + fn source(instance_key: &str) -> SourceIdentity { SourceIdentity::new("codex", instance_key).with_display_name("Codex") } diff --git a/docs/architecture/repository-structure.md b/docs/architecture/repository-structure.md index 974b99e..4f1b19a 100644 --- a/docs/architecture/repository-structure.md +++ b/docs/architecture/repository-structure.md @@ -46,8 +46,9 @@ autophagy/ `autophagy-events`, `autophagy-store`, `autophagy-core`, `autophagy-cli`, the native Claude Code and Codex adapters, and their shared conformance harness exist through PR 5. `autophagy-patterns` and Evidence Packet v0.1 exist through -PR 6. A crate or package is added when its PR contains an executable vertical -slice; empty placeholder crates are avoided. +PR 6. `autophagy-redaction` and the offline digestion/privacy CLI complete +Milestone 1 through PR 7. A crate or package is added when its PR contains an +executable vertical slice; empty placeholder crates are avoided. ## Dependency direction diff --git a/docs/guides/claude-code.md b/docs/guides/claude-code.md index 3c5ac52..4986584 100644 --- a/docs/guides/claude-code.md +++ b/docs/guides/claude-code.md @@ -41,6 +41,11 @@ that local content is approved for search. Changing `--include-content` after a history has already been imported changes canonical event content; use a fresh database when intentionally changing that evidence policy. +Default credential rules redact recognized secrets before persistence. Add +repeatable `--exclude-path GLOB` values for project or artifact exclusions. +Project filters, exclusion globs, and the privacy-policy version are part of the +cursor scope, so changing policy safely rescans instead of losing skipped data. + ## Incremental behavior Only newline-terminated records advance a cursor. An actively written partial diff --git a/docs/guides/codex.md b/docs/guides/codex.md index 6bcb9aa..29df088 100644 --- a/docs/guides/codex.md +++ b/docs/guides/codex.md @@ -42,6 +42,11 @@ Tool inputs remain structural AEP evidence but enter FTS only with is locally approved for search. Change the content policy only with a fresh database because it changes canonical event bodies. +Default credential rules redact recognized secrets before persistence. Add +repeatable `--exclude-path GLOB` values for project or artifact exclusions. +Project filters, exclusion globs, and the privacy-policy version are part of the +cursor scope, so changing policy safely rescans instead of losing skipped data. + ## Incremental and cross-adapter guarantees The cursor advances only past newline-terminated records and retains unmatched diff --git a/docs/guides/generic-jsonl.md b/docs/guides/generic-jsonl.md index b47189a..da68331 100644 --- a/docs/guides/generic-jsonl.md +++ b/docs/guides/generic-jsonl.md @@ -21,8 +21,10 @@ autophagy --output json import sessions.jsonl --dry-run ## Import a file -Give each persisted input or producer a stable instance key. Reimporting the -same events is safe and reports duplicates without changing canonical rows. +Give each persisted input or producer a stable instance key. When omitted, the +CLI stores an opaque hash of the canonical input path rather than the path +itself. Reimporting the same events is safe and reports duplicates without +changing canonical rows. ```sh autophagy import sessions.jsonl \ @@ -39,6 +41,10 @@ autophagy import sessions.jsonl \ --project /work/service-b ``` +Every selected event passes through default secret redaction before persistence. +Use repeatable `--exclude-path GLOB` values to drop events whose project or +artifact path matches. Exclusion happens before SQLite and FTS writes. + Use `-` or omit the file to read standard input: ```sh @@ -77,8 +83,8 @@ autophagy --output json search stale ``` JSON output is a tagged object with `command` and `result` fields. Import -results include line, event, insertion, duplicate, conflict, project-skip, and -rejection counts plus bounded diagnostics. +results include line, event, insertion, duplicate, conflict, project-skip, +privacy-skip, redacted-field, and rejection counts plus bounded diagnostics. ## Exit codes diff --git a/docs/guides/privacy-and-lifecycle.md b/docs/guides/privacy-and-lifecycle.md new file mode 100644 index 0000000..659c82b --- /dev/null +++ b/docs/guides/privacy-and-lifecycle.md @@ -0,0 +1,72 @@ +# Privacy and evidence lifecycle + +Milestone 1 enforces privacy at ingestion, before SQLite storage and FTS +projection. Export is not a second redaction boundary: it emits the canonical +events already retained in the database. + +## Secret redaction + +Default deterministic rules recursively inspect tool input, event/tool/artifact +metadata, and artifact paths and URIs. Recognized values are replaced with +`[REDACTED]`. Built-in rules cover common OpenAI-style keys, GitHub tokens, AWS +access-key IDs, bearer credentials, and assignments to names such as `api_key`, +`access_token`, `password`, and `secret`. + +Regex rules cannot recognize every secret. Do not treat a zero-redaction count +as proof that content is safe. Prefer native adapters' default omission of +prompt/response text and avoid `--include-content` unless necessary. + +## Path exclusions + +Use repeatable glob expressions: + +```sh +autophagy import history.jsonl \ + --exclude-path '**/.env' \ + --exclude-path '**/private/**' +``` + +An event is excluded when its exact project or any artifact path matches. +Import summaries report `privacy_skipped` and `redacted_fields`. Native-adapter +cursors include the sorted exclusion set and privacy-policy version, so later +policy changes use an independent cursor scope. + +## Evidence inspection + +```sh +autophagy --output json patterns --project /workspace/example +autophagy --output json digest --project /workspace/example +``` + +`digest` is deterministic in Milestone 1 and declares `model_used: false` and +`network_used: false` in JSON output. + +## Export + +Export writes canonical AEP JSONL to standard output irrespective of the global +display format: + +```sh +autophagy export --project /workspace/example > evidence.jsonl +``` + +The destination becomes another copy of sensitive data and must be protected or +deleted independently. + +## Retention and deletion + +```sh +autophagy prune --older-than-days 30 --dry-run +autophagy prune --older-than-days 30 +autophagy delete session ses_example +autophagy delete all --confirm delete-all +``` + +Dry-run executes the same retention transaction and rolls it back. Pruning and +session deletion cascade through events, FTS, conflicts, and event-artifact +links, then remove orphaned artifacts. Delete-all also removes sources, import +records, and incremental cursors. + +SQLite uses `secure_delete`, but filesystem snapshots, backups, exported files, +and previous database copies remain outside Autophagy's control. `VACUUM` is not +run automatically because it can be expensive and needs additional free space. diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md new file mode 100644 index 0000000..80b0264 --- /dev/null +++ b/docs/security/threat-model.md @@ -0,0 +1,65 @@ +# Milestone 1 threat model + +Status: implemented baseline, 2026-07-16 + +## Protected assets + +- prompts, agent messages, tool inputs, and tool outputs; +- repository names, local paths, diffs, commands, and test failures; +- credentials accidentally embedded in any of those values; +- evidence lineage and findings derived from private work. + +## Trust boundaries + +```text +local agent transcript + -> adapter normalization + -> path exclusion + secret redaction + -> validated AEP event + -> SQLite canonical event + explicit FTS projection + -> deterministic detector + -> Evidence Packet / local JSONL export +``` + +Milestone 1 contains no network client, telemetry path, cloud inference, or model +invocation. Native source transcripts remain owned by their agent and are never +modified. The SQLite database and user-created exports are local trust +boundaries. + +## Threats and controls + +| Threat | Implemented control | +|---|---| +| Accidental credential persistence | Recursive deterministic redaction before store insertion | +| Sensitive project ingestion | Repeatable project/artifact glob exclusions before persistence | +| Private free text entering search | FTS accepts only explicit redaction-approved projections | +| Adapter rescan leaks previously excluded data | Privacy policy and exclusions are cursor-scoped | +| Fabricated behavioral finding | Thresholded deterministic detectors with exact evidence IDs | +| Counterevidence hidden from review | Evidence Packet carries explicit opposite outcomes | +| Accidental full deletion | Exact `--confirm delete-all` phrase required | +| Unverifiable retention effect | Prune dry-run executes and rolls back the real transaction | +| Dangling deleted evidence | Foreign-key cascades and orphan-artifact cleanup | +| Silent ID overwrite | Conflicting immutable event bodies are quarantined | + +## Residual risks + +- Pattern-based secret detection has false negatives and false positives. +- Tool inputs are structural evidence and may retain private non-secret data. +- `--include-content` materially expands retained private text. +- Exclusion globs protect only paths represented in normalized event fields. +- Agent transcripts, backups, snapshots, WAL copies, shell history, and exported + JSONL are outside delete-all's reach. +- A local process with the user's filesystem permissions can read the database. +- Findings demonstrate recurrence, not causality or intervention correctness. + +## Security invariants + +1. Redaction and exclusions run before canonical persistence. +2. No model or network is needed to import, search, detect, digest, export, or + delete Milestone 1 evidence. +3. Every finding cites exact immutable event IDs. +4. No finding is emitted below configured recurrence and independence thresholds. +5. Destructive operations are explicit, scoped, and report their effect. + +Report failures of these invariants privately using the process in +[`SECURITY.md`](../../SECURITY.md). diff --git a/scripts/demo-milestone-1.sh b/scripts/demo-milestone-1.sh new file mode 100755 index 0000000..f66be76 --- /dev/null +++ b/scripts/demo-milestone-1.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +database="${TMPDIR:-/tmp}/autophagy-milestone-1-$$.db" +trap 'rm -f "$database" "$database-shm" "$database-wal"' EXIT HUP INT TERM + +echo "Importing anonymized AEP evidence into $database" +cargo run --quiet -p autophagy-cli -- \ + --database "$database" \ + import evals/fixtures/findings/deterministic.jsonl \ + --instance-key milestone-1-demo + +echo +echo "Deterministic patterns" +cargo run --quiet -p autophagy-cli -- --database "$database" patterns + +echo +echo "Offline digest" +cargo run --quiet -p autophagy-cli -- --database "$database" digest + +echo +echo "Retention preview" +cargo run --quiet -p autophagy-cli -- \ + --database "$database" prune --older-than-days 0 --dry-run