diff --git a/.mise.toml b/.mise.toml index c449c3f..df64794 100644 --- a/.mise.toml +++ b/.mise.toml @@ -18,8 +18,8 @@ description = "Run the complete Rust test suite" run = "cargo test --workspace --all-features" [tasks.docs] -description = "Parse the normalized blueprint as GitHub-flavored Markdown" -run = "pandoc docs/blueprint/*.md --from=gfm --to=html --fail-if-warnings --output=/dev/null" +description = "Parse documentation as GitHub-flavored Markdown" +run = "find docs -type f -name '*.md' -print0 | xargs -0 -n 1 pandoc --from=gfm --to=html --fail-if-warnings --output=/dev/null" [tasks.ci] description = "Validate GitHub Actions workflows" diff --git a/Cargo.lock b/Cargo.lock index 9470b4a..fc124c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.3.4", "once_cell", "serde", "version_check", @@ -31,12 +31,88 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "autophagy-cli" +version = "0.1.0-alpha.1" +dependencies = [ + "autophagy-core", + "autophagy-store", + "clap", + "directories", + "serde", + "serde_json", + "tempfile", + "thiserror", +] + +[[package]] +name = "autophagy-core" +version = "0.1.0-alpha.1" +dependencies = [ + "autophagy-events", + "autophagy-store", + "serde", + "serde_json", + "tempfile", + "thiserror", +] + [[package]] name = "autophagy-events" version = "0.1.0-alpha.1" @@ -54,6 +130,7 @@ version = "0.1.0-alpha.1" dependencies = [ "autophagy-events", "rusqlite", + "serde", "serde_json", "sha2", "tempfile", @@ -125,6 +202,52 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "clap" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "const-oid" version = "0.10.2" @@ -176,6 +299,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -274,6 +418,17 @@ dependencies = [ "num", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -299,6 +454,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hybrid-array" version = "0.4.13" @@ -410,6 +571,12 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -438,7 +605,7 @@ dependencies = [ "email_address", "fancy-regex", "fraction", - "getrandom", + "getrandom 0.3.4", "idna", "itoa", "jsonschema-regex", @@ -474,6 +641,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.38.1" @@ -608,6 +784,18 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "outref" version = "0.5.2" @@ -697,6 +885,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -725,7 +924,7 @@ checksum = "348e860aeb0b7bd035778fd11dd9cd5290d32e4aed3b8f2274a00287a9fd362b" dependencies = [ "ahash", "fluent-uri", - "getrandom", + "getrandom 0.3.4", "hashbrown", "itoa", "micromap", @@ -873,6 +1072,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.119" @@ -902,7 +1107,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys", @@ -993,6 +1198,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid-simd" version = "0.8.0" @@ -1021,6 +1232,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" diff --git a/Cargo.toml b/Cargo.toml index f98d5be..38b6a6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,11 @@ [workspace] resolver = "3" -members = ["crates/autophagy-events", "crates/autophagy-store"] +members = [ + "crates/autophagy-cli", + "crates/autophagy-core", + "crates/autophagy-events", + "crates/autophagy-store", +] [workspace.package] version = "0.1.0-alpha.1" @@ -10,6 +15,8 @@ license = "Apache-2.0" repository = "https://github.com/karnstack/autophagy" [workspace.dependencies] +clap = { version = "4.6.2", features = ["derive", "env"] } +directories = "6.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.11" diff --git a/README.md b/README.md index 1066769..c42d546 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ change because of what happened?” ## Status -Autophagy is in foundation development. Agent Event Protocol (AEP) v0.1 and the -transactional local SQLite event store are implemented. No daemon, session -importer, or background capture ships yet. +Autophagy is in foundation development. Agent Event Protocol (AEP) v0.1, the +transactional local SQLite event store, and a generic JSONL CLI vertical slice +are implemented. No daemon, native-agent adapter, or background capture ships +yet. ## Principles @@ -26,6 +27,8 @@ importer, or background capture ships yet. ## Repository map ```text +crates/autophagy-cli/ User-facing import, sessions, and search commands +crates/autophagy-core/ Reusable streaming import application services crates/autophagy-events/ AEP Rust types, parsing, and validation crates/autophagy-store/ SQLite migrations, idempotency, FTS, and deletion docs/architecture/ Planned component and storage boundaries @@ -40,6 +43,28 @@ The intended repository structure is documented in The complete product blueprint is available in [`docs/blueprint/`](docs/blueprint/README.md). +## Try the CLI + +Import the anonymized demo corpus into an explicit local database: + +```sh +mise exec -- cargo run -p autophagy-cli -- \ + --database /tmp/autophagy-demo.db \ + import evals/fixtures/generic-jsonl/demo.jsonl \ + --instance-key demo \ + --index-metadata summary + +mise exec -- cargo run -p autophagy-cli -- \ + --database /tmp/autophagy-demo.db sessions + +mise exec -- cargo run -p autophagy-cli -- \ + --database /tmp/autophagy-demo.db search stale +``` + +See the [generic JSONL guide](docs/guides/generic-jsonl.md) for dry-run, +project selection, standard input, JSON output, privacy controls, and exit-code +semantics. + ## Try the contract Install [mise](https://mise.jdx.dev/), then run: diff --git a/crates/autophagy-cli/Cargo.toml b/crates/autophagy-cli/Cargo.toml new file mode 100644 index 0000000..b2f2279 --- /dev/null +++ b/crates/autophagy-cli/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "autophagy-cli" +description = "Command-line interface for the local Autophagy engine" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[[bin]] +name = "autophagy" +path = "src/main.rs" + +[dependencies] +autophagy-core = { path = "../autophagy-core" } +autophagy-store = { path = "../autophagy-store" } +clap.workspace = true +directories.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tempfile = "3.27" + +[lints] +workspace = true diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs new file mode 100644 index 0000000..2bfc65f --- /dev/null +++ b/crates/autophagy-cli/src/main.rs @@ -0,0 +1,290 @@ +//! Command-line entry point for importing and querying local agent activity. + +use std::{ + fs::{self, File}, + io::{self, BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::ExitCode, +}; + +use autophagy_core::{ImportOptions, ImportSummary, import_jsonl}; +use autophagy_store::{EventStore, SearchHit, SessionSummary, StoreError}; +use clap::{Parser, Subcommand, ValueEnum}; +use directories::ProjectDirs; +use serde::Serialize; + +#[derive(Debug, Parser)] +#[command( + name = "autophagy", + version, + about = "The self-improvement layer for local coding agents", + arg_required_else_help = true +)] +struct Cli { + /// Local database path. Defaults to the platform-local application data directory. + #[arg(long, global = true, env = "AUTOPHAGY_DB", value_name = "PATH")] + database: Option, + + /// Output format for command results. + #[arg(long, global = true, value_enum, default_value_t = OutputFormat::Text)] + output: OutputFormat, + + #[command(subcommand)] + command: Commands, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum OutputFormat { + Text, + Json, +} + +#[derive(Debug, Subcommand)] +enum Commands { + /// Import normalized AEP JSONL from a file or standard input. + Import { + /// JSONL file, or `-` for standard input. + #[arg(default_value = "-", value_name = "FILE")] + input: PathBuf, + + /// Stable source identity. Defaults to the canonical input path or `stdin`. + #[arg(long, value_name = "KEY")] + instance_key: Option, + + /// Optional user-facing label for this source. + #[arg(long, value_name = "NAME")] + display_name: Option, + + /// Include only events with this exact project path. Repeatable. + #[arg(long = "project", value_name = "PATH")] + projects: Vec, + + /// Index tool input after confirming the source has already been redacted. + #[arg(long)] + index_tool_input: bool, + + /// Index an already-redacted event metadata key. Repeatable. + #[arg(long = "index-metadata", value_name = "KEY")] + index_metadata: Vec, + + /// Parse and filter without creating or changing the database. + #[arg(long)] + dry_run: bool, + + /// Maximum line diagnostics retained in the result. + #[arg(long, default_value_t = 100, value_name = "COUNT")] + max_diagnostics: usize, + }, + + /// List recently active imported sessions. + Sessions { + /// Maximum number of sessions to return. + #[arg(long, default_value_t = 50, value_name = "COUNT")] + limit: u32, + }, + + /// Search the redaction-approved FTS5 event projection. + Search { + /// FTS5 query expression. + query: String, + + /// Maximum number of matches to return. + #[arg(long, default_value_t = 20, value_name = "COUNT")] + limit: u32, + }, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "command", content = "result", rename_all = "snake_case")] +enum CommandReport { + Import(ImportSummary), + Sessions(Vec), + Search(Vec), +} + +impl CommandReport { + const fn has_issues(&self) -> bool { + match self { + Self::Import(summary) => summary.has_issues(), + Self::Sessions(_) | Self::Search(_) => false, + } + } +} + +#[derive(Debug, thiserror::Error)] +enum CliError { + #[error("I/O operation failed: {0}")] + Io(#[from] io::Error), + #[error(transparent)] + Store(#[from] StoreError), + #[error(transparent)] + Import(#[from] autophagy_core::ImportError), + #[error("could not serialize command output: {0}")] + Json(#[from] serde_json::Error), + #[error("could not determine the platform-local application data directory")] + DataDirectoryUnavailable, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + let output = cli.output; + match execute(cli).and_then(|report| { + let has_issues = report.has_issues(); + write_report(io::stdout().lock(), output, &report)?; + Ok(has_issues) + }) { + Ok(true) => ExitCode::from(2), + Ok(false) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("error: {error}"); + ExitCode::FAILURE + } + } +} + +fn execute(cli: Cli) -> Result { + match cli.command { + Commands::Import { + input, + instance_key, + display_name, + projects, + index_tool_input, + index_metadata, + dry_run, + max_diagnostics, + } => { + let instance_key = instance_key.unwrap_or(derive_instance_key(&input)?); + let mut options = ImportOptions::new(instance_key); + options.display_name = display_name; + options.projects = projects; + options.index_tool_input = index_tool_input; + options.index_metadata = index_metadata; + options.dry_run = dry_run; + options.max_diagnostics = max_diagnostics; + let reader = open_input(&input)?; + + let summary = if dry_run { + import_jsonl(reader, None, &options)? + } else { + let database = resolve_database_path(cli.database)?; + let mut store = open_store(&database)?; + import_jsonl(reader, Some(&mut store), &options)? + }; + Ok(CommandReport::Import(summary)) + } + Commands::Sessions { limit } => { + let database = resolve_database_path(cli.database)?; + let store = open_store(&database)?; + Ok(CommandReport::Sessions(store.list_sessions(limit)?)) + } + Commands::Search { query, limit } => { + let database = resolve_database_path(cli.database)?; + let store = open_store(&database)?; + Ok(CommandReport::Search(store.search(&query, limit)?)) + } + } +} + +fn resolve_database_path(path: Option) -> Result { + if let Some(path) = path { + return Ok(path); + } + let project = ProjectDirs::from("sh", "autophagy", "Autophagy") + .ok_or(CliError::DataDirectoryUnavailable)?; + Ok(project.data_local_dir().join("autophagy.db")) +} + +fn open_store(path: &Path) -> Result { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + Ok(EventStore::open(path)?) +} + +fn open_input(path: &Path) -> Result, CliError> { + if path == Path::new("-") { + Ok(Box::new(BufReader::new(io::stdin()))) + } else { + Ok(Box::new(BufReader::new(File::open(path)?))) + } +} + +fn derive_instance_key(input: &Path) -> Result { + if input == Path::new("-") { + Ok("stdin".to_owned()) + } else { + Ok(fs::canonicalize(input)?.to_string_lossy().into_owned()) + } +} + +fn write_report( + mut writer: impl Write, + format: OutputFormat, + report: &CommandReport, +) -> Result<(), CliError> { + match format { + OutputFormat::Json => { + serde_json::to_writer_pretty(&mut writer, report)?; + writeln!(writer)?; + } + OutputFormat::Text => match report { + CommandReport::Import(summary) => write_import_summary(&mut writer, summary)?, + CommandReport::Sessions(sessions) => { + writeln!(writer, "SESSION\tSOURCE\tEVENTS\tLAST EVENT\tPROJECT")?; + for session in sessions { + writeln!( + writer, + "{}\t{}\t{}\t{}\t{}", + session.session_id, + session.adapter, + session.event_count, + session.last_event_at, + session.project_path.as_deref().unwrap_or("-") + )?; + } + } + CommandReport::Search(hits) => { + for hit in hits { + writeln!(writer, "{}\t{}", hit.event_id, hit.snippet)?; + } + } + }, + } + Ok(()) +} + +fn write_import_summary(writer: &mut impl Write, summary: &ImportSummary) -> io::Result<()> { + writeln!( + writer, + "{} lines · {} events · {} inserted · {} duplicates · {} conflicts · {} skipped · {} rejected{}", + summary.lines_read, + summary.events_seen, + summary.inserted, + summary.duplicates, + summary.conflicts, + summary.skipped, + summary.rejected, + if summary.dry_run { " · dry run" } else { "" } + )?; + for diagnostic in &summary.diagnostics { + writeln!( + writer, + "line {} [{}] {}", + diagnostic.line, + diagnostic.code.as_str(), + diagnostic.message + )?; + } + if summary.diagnostics_suppressed > 0 { + writeln!( + writer, + "{} additional diagnostics suppressed", + summary.diagnostics_suppressed + )?; + } + Ok(()) +} diff --git a/crates/autophagy-cli/tests/cli.rs b/crates/autophagy-cli/tests/cli.rs new file mode 100644 index 0000000..a5d8fac --- /dev/null +++ b/crates/autophagy-cli/tests/cli.rs @@ -0,0 +1,104 @@ +//! End-to-end tests for the user-facing command line. + +use std::{fs, path::Path, process::Command}; + +use serde_json::Value; + +const VALID_JSONL: &str = concat!( + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_cli_start\",", + "\"session_id\":\"ses_cli\",\"timestamp\":\"2026-07-16T09:00:00Z\",", + "\"sequence\":0,\"source\":\"generic-jsonl\",\"type\":\"session.started\",", + "\"project\":\"/repo/cli\"}\n", + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_cli_failure\",", + "\"session_id\":\"ses_cli\",\"timestamp\":\"2026-07-16T09:01:00Z\",", + "\"sequence\":1,\"source\":\"generic-jsonl\",\"type\":\"tool.failed\",", + "\"project\":\"/repo/cli\",\"tool\":{\"name\":\"bash\",\"exit_code\":1},", + "\"metadata\":{\"search\":\"generated client stale\"}}\n" +); + +#[test] +fn import_sessions_search_and_reimport_work_end_to_end() { + let directory = tempfile::tempdir().expect("temporary directory"); + let input = directory.path().join("events.jsonl"); + let database = directory.path().join("autophagy.db"); + fs::write(&input, VALID_JSONL).expect("write fixture"); + + let imported = run_json( + &database, + [ + "import", + input.to_str().expect("UTF-8 path"), + "--instance-key", + "fixture:cli", + "--index-metadata", + "search", + ], + ); + assert_eq!(imported["command"], "import"); + assert_eq!(imported["result"]["inserted"], 2); + assert_eq!(imported["result"]["rejected"], 0); + + let sessions = run_json(&database, ["sessions"]); + assert_eq!(sessions["result"].as_array().expect("sessions").len(), 1); + assert_eq!(sessions["result"][0]["session_id"], "ses_cli"); + assert_eq!(sessions["result"][0]["event_count"], 2); + + let search = run_json(&database, ["search", "generated"]); + assert_eq!(search["result"].as_array().expect("hits").len(), 1); + assert_eq!(search["result"][0]["event_id"], "evt_cli_failure"); + + let duplicate = run_json( + &database, + [ + "import", + input.to_str().expect("UTF-8 path"), + "--instance-key", + "fixture:cli", + ], + ); + assert_eq!(duplicate["result"]["inserted"], 0); + assert_eq!(duplicate["result"]["duplicates"], 2); +} + +#[test] +fn dry_run_with_bad_records_returns_attention_exit_without_creating_database() { + let directory = tempfile::tempdir().expect("temporary directory"); + let input = directory.path().join("invalid.jsonl"); + let database = directory.path().join("must-not-exist.db"); + fs::write(&input, "{not json}\n").expect("write fixture"); + + let output = command(&database) + .args([ + "--output", + "json", + "import", + input.to_str().expect("UTF-8 path"), + "--dry-run", + ]) + .output() + .expect("run command"); + assert_eq!(output.status.code(), Some(2)); + let report: Value = serde_json::from_slice(&output.stdout).expect("JSON output"); + assert_eq!(report["result"]["rejected"], 1); + assert!(!database.exists()); +} + +fn run_json(database: &Path, args: [&str; N]) -> Value { + let output = command(database) + .args(["--output", "json"]) + .args(args) + .output() + .expect("run command"); + assert!( + output.status.success(), + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).expect("JSON output") +} + +fn command(database: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_autophagy")); + command.args(["--database", database.to_str().expect("UTF-8 path")]); + command +} diff --git a/crates/autophagy-core/Cargo.toml b/crates/autophagy-core/Cargo.toml new file mode 100644 index 0000000..f0f480f --- /dev/null +++ b/crates/autophagy-core/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "autophagy-core" +description = "Application services and adapter orchestration for Autophagy" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +autophagy-events = { path = "../autophagy-events" } +autophagy-store = { path = "../autophagy-store" } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tempfile = "3.27" + +[lints] +workspace = true diff --git a/crates/autophagy-core/src/generic_jsonl.rs b/crates/autophagy-core/src/generic_jsonl.rs new file mode 100644 index 0000000..8a3480f --- /dev/null +++ b/crates/autophagy-core/src/generic_jsonl.rs @@ -0,0 +1,323 @@ +use std::io::BufRead; + +use autophagy_events::{Event, EventParseError}; +use autophagy_store::{EventStore, InsertOutcome, SearchProjection, SourceIdentity, StoreError}; +use serde::Serialize; + +/// Configuration for one generic AEP JSONL import stream. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ImportOptions { + /// Stable identity for the file, pipe, or producer being imported. + pub instance_key: String, + /// Optional user-facing source label. + pub display_name: Option, + /// Exact project paths to include. An empty list includes every project. + pub projects: Vec, + /// Whether already-redacted tool input may enter FTS5. + pub index_tool_input: bool, + /// Explicit metadata keys whose already-redacted values may enter FTS5. + pub index_metadata: Vec, + /// Parse and filter without opening or mutating a database. + pub dry_run: bool, + /// Maximum number of diagnostics retained in memory and output. + pub max_diagnostics: usize, +} + +impl ImportOptions { + /// Create conservative import options for a stable source instance. + #[must_use] + pub fn new(instance_key: impl Into) -> Self { + Self { + instance_key: instance_key.into(), + display_name: None, + projects: Vec::new(), + index_tool_input: false, + index_metadata: Vec::new(), + dry_run: false, + max_diagnostics: 100, + } + } +} + +/// Stable category for a rejected JSONL record. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportDiagnosticCode { + /// The line was not structurally valid AEP JSON. + InvalidJson, + /// The decoded line violated an AEP semantic invariant. + InvalidEvent, + /// The event was valid AEP but conflicted with store-level invariants. + StoreRejected, +} + +impl ImportDiagnosticCode { + /// Return the stable machine-readable code. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::InvalidJson => "invalid_json", + Self::InvalidEvent => "invalid_event", + Self::StoreRejected => "store_rejected", + } + } +} + +/// One bounded, line-addressed import diagnostic. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ImportDiagnostic { + /// One-based source line number. + pub line: u64, + /// Stable diagnostic category. + pub code: ImportDiagnosticCode, + /// Human-readable parser, validation, or storage message. + pub message: String, +} + +/// Aggregate result of streaming one JSONL source. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct ImportSummary { + /// Physical lines read, including blank lines. + pub lines_read: u64, + /// Nonblank records presented to the AEP parser. + pub events_seen: u64, + /// Selected events that passed AEP validation. + pub validated: u64, + /// Newly persisted canonical events. + pub inserted: u64, + /// Existing identical events that caused no writes. + pub duplicates: u64, + /// Same-ID/different-content events retained in quarantine. + pub conflicts: u64, + /// Valid events excluded by project selection. + pub skipped: u64, + /// Invalid or store-rejected records. + pub rejected: u64, + /// Retained line-addressed diagnostics. + pub diagnostics: Vec, + /// Diagnostics omitted after reaching `max_diagnostics`. + pub diagnostics_suppressed: u64, + /// Whether this import intentionally performed no writes. + pub dry_run: bool, +} + +impl ImportSummary { + /// Return whether operator attention is required. + #[must_use] + pub const fn has_issues(&self) -> bool { + self.rejected > 0 || self.conflicts > 0 + } +} + +/// Fatal error that prevents the importer from continuing safely. +#[derive(Debug, thiserror::Error)] +pub enum ImportError { + /// The source stream could not be read as UTF-8 text. + #[error("could not read JSONL source: {0}")] + Io(#[from] std::io::Error), + /// A database or migration operation failed. + #[error("event store failed: {0}")] + Store(#[from] StoreError), + /// Non-dry imports require a writable store. + #[error("a writable event store is required unless dry_run is enabled")] + MissingStore, + /// An importer option was empty or internally inconsistent. + #[error("invalid import option: {0}")] + InvalidOptions(String), +} + +/// Stream AEP JSONL records into the local event store. +/// +/// Blank lines are ignored. AEP and record-level storage failures become +/// bounded diagnostics so later lines can still be processed. Infrastructure +/// errors abort immediately. +/// +/// # Errors +/// +/// Returns [`ImportError`] for unreadable input, invalid options, a missing +/// non-dry store, or a fatal database failure. +pub fn import_jsonl( + mut reader: R, + mut store: Option<&mut EventStore>, + options: &ImportOptions, +) -> Result { + validate_options(options)?; + if !options.dry_run && store.is_none() { + return Err(ImportError::MissingStore); + } + + let mut summary = ImportSummary { + dry_run: options.dry_run, + ..ImportSummary::default() + }; + let mut line = String::new(); + + loop { + line.clear(); + let bytes_read = reader.read_line(&mut line)?; + if bytes_read == 0 { + break; + } + summary.lines_read += 1; + let record = line.trim_end_matches(['\r', '\n']); + if record.trim().is_empty() { + continue; + } + summary.events_seen += 1; + + let event = match Event::from_json_str(record) { + Ok(event) => event, + Err(error) => { + let code = match &error { + EventParseError::Json(_) => ImportDiagnosticCode::InvalidJson, + EventParseError::Validation(_) => ImportDiagnosticCode::InvalidEvent, + }; + reject(&mut summary, options, code, error.to_string()); + continue; + } + }; + + if !project_selected(&event, &options.projects) { + summary.skipped += 1; + continue; + } + summary.validated += 1; + if options.dry_run { + continue; + } + + let source = SourceIdentity { + adapter: event.source.clone(), + instance_key: options.instance_key.clone(), + display_name: options.display_name.clone(), + }; + let projection = search_projection(&event, options); + let result = store + .as_deref_mut() + .ok_or(ImportError::MissingStore)? + .insert_event(&source, &event, &projection); + match result { + Ok(InsertOutcome::Inserted { .. }) => summary.inserted += 1, + Ok(InsertOutcome::Duplicate { .. }) => summary.duplicates += 1, + Ok(InsertOutcome::ConflictQuarantined { .. }) => summary.conflicts += 1, + Err(error) if is_record_rejection(&error) => { + reject( + &mut summary, + options, + ImportDiagnosticCode::StoreRejected, + error.to_string(), + ); + } + Err(error) => return Err(error.into()), + } + } + + Ok(summary) +} + +fn validate_options(options: &ImportOptions) -> Result<(), ImportError> { + if options.instance_key.trim().is_empty() { + return Err(ImportError::InvalidOptions( + "instance_key must not be empty".to_owned(), + )); + } + if options + .display_name + .as_ref() + .is_some_and(|name| name.trim().is_empty()) + { + return Err(ImportError::InvalidOptions( + "display_name must not be empty".to_owned(), + )); + } + if options + .projects + .iter() + .any(|project| project.trim().is_empty()) + { + return Err(ImportError::InvalidOptions( + "project selections must not be empty".to_owned(), + )); + } + if options + .index_metadata + .iter() + .any(|key| key.trim().is_empty()) + { + return Err(ImportError::InvalidOptions( + "metadata keys must not be empty".to_owned(), + )); + } + Ok(()) +} + +fn project_selected(event: &Event, projects: &[String]) -> bool { + projects.is_empty() + || event + .project + .as_ref() + .is_some_and(|project| projects.contains(project)) +} + +fn search_projection(event: &Event, options: &ImportOptions) -> SearchProjection { + let tool_input_text = if options.index_tool_input { + event + .tool + .as_ref() + .and_then(|tool| tool.input.as_ref()) + .map(value_as_text) + } else { + None + }; + let searchable_text = options + .index_metadata + .iter() + .filter_map(|key| event.metadata.get(key)) + .map(value_as_text) + .filter(|value| !value.is_empty()) + .collect::>() + .join("\n"); + + SearchProjection { + tool_input_text, + searchable_text: (!searchable_text.is_empty()).then_some(searchable_text), + } +} + +fn value_as_text(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(value) => value.clone(), + value => value.to_string(), + } +} + +fn reject( + summary: &mut ImportSummary, + options: &ImportOptions, + code: ImportDiagnosticCode, + message: String, +) { + summary.rejected += 1; + if summary.diagnostics.len() < options.max_diagnostics { + summary.diagnostics.push(ImportDiagnostic { + line: summary.lines_read, + code, + message, + }); + } else { + summary.diagnostics_suppressed += 1; + } +} + +const fn is_record_rejection(error: &StoreError) -> bool { + matches!( + error, + StoreError::InvalidEvent(_) + | StoreError::InvalidSource { .. } + | StoreError::SourceMismatch { .. } + | StoreError::SessionSourceConflict { .. } + | StoreError::SessionSequenceConflict { .. } + | StoreError::SequenceOutOfRange { .. } + | StoreError::ArtifactOrdinalOutOfRange { .. } + ) +} diff --git a/crates/autophagy-core/src/lib.rs b/crates/autophagy-core/src/lib.rs new file mode 100644 index 0000000..09e69b4 --- /dev/null +++ b/crates/autophagy-core/src/lib.rs @@ -0,0 +1,7 @@ +//! Reusable application services for importing and digesting agent activity. + +mod generic_jsonl; + +pub use generic_jsonl::{ + ImportDiagnostic, ImportDiagnosticCode, ImportError, ImportOptions, ImportSummary, import_jsonl, +}; diff --git a/crates/autophagy-core/tests/fixtures/mixed.jsonl b/crates/autophagy-core/tests/fixtures/mixed.jsonl new file mode 100644 index 0000000..c559772 --- /dev/null +++ b/crates/autophagy-core/tests/fixtures/mixed.jsonl @@ -0,0 +1,5 @@ +{"spec_version":"aep/0.1","event_id":"evt_start","session_id":"ses_fixture","timestamp":"2026-07-16T06:00:00Z","sequence":0,"source":"generic-jsonl","type":"session.started","project":"/repo/a"} +{"spec_version":"aep/0.1","event_id":"evt_failure","session_id":"ses_fixture","timestamp":"2026-07-16T06:01:00Z","sequence":1,"source":"generic-jsonl","type":"tool.failed","project":"/repo/a","tool":{"name":"bash","input":"secret-tool-token","exit_code":1},"metadata":{"search":"generated client was stale"}} +{"spec_version": +{"spec_version":"aep/0.1","event_id":"evt_invalid","session_id":"ses_fixture","timestamp":"2026-07-16T06:02:00Z","source":"generic-jsonl","type":"tool.failed","project":"/repo/a"} +{"spec_version":"aep/0.1","event_id":"evt_other","session_id":"ses_other","timestamp":"2026-07-16T06:03:00Z","sequence":0,"source":"generic-jsonl","type":"session.started","project":"/repo/b"} diff --git a/crates/autophagy-core/tests/generic_jsonl.rs b/crates/autophagy-core/tests/generic_jsonl.rs new file mode 100644 index 0000000..7e7f29c --- /dev/null +++ b/crates/autophagy-core/tests/generic_jsonl.rs @@ -0,0 +1,119 @@ +//! Fixture-based tests for the generic AEP JSONL importer. + +use std::io::Cursor; + +use autophagy_core::{ImportDiagnosticCode, ImportOptions, import_jsonl}; +use autophagy_store::{EventStore, StoreStats}; + +const MIXED: &str = include_str!("fixtures/mixed.jsonl"); + +#[test] +fn streams_selected_records_and_reports_bounded_diagnostics() { + let mut store = EventStore::open_in_memory().expect("store"); + let mut options = ImportOptions::new("fixture:mixed"); + options.projects = vec!["/repo/a".to_owned()]; + options.index_metadata = vec!["search".to_owned()]; + options.max_diagnostics = 1; + + let summary = import_jsonl(Cursor::new(MIXED), Some(&mut store), &options).expect("import"); + assert_eq!(summary.lines_read, 5); + assert_eq!(summary.events_seen, 5); + assert_eq!(summary.validated, 2); + assert_eq!(summary.inserted, 2); + assert_eq!(summary.duplicates, 0); + assert_eq!(summary.conflicts, 0); + assert_eq!(summary.skipped, 1); + assert_eq!(summary.rejected, 2); + assert_eq!(summary.diagnostics.len(), 1); + assert_eq!(summary.diagnostics[0].line, 3); + assert_eq!( + summary.diagnostics[0].code, + ImportDiagnosticCode::InvalidJson + ); + assert_eq!(summary.diagnostics_suppressed, 1); + assert!(summary.has_issues()); + + assert_eq!(store.search("generated", 10).expect("search").len(), 1); + assert!( + store + .search("\"secret-tool-token\"", 10) + .expect("private search") + .is_empty() + ); + assert_eq!( + store.stats().expect("stats"), + StoreStats { + sources: 1, + sessions: 1, + events: 2, + artifacts: 0, + conflicts: 0, + } + ); + + let duplicate = + import_jsonl(Cursor::new(MIXED), Some(&mut store), &options).expect("duplicate import"); + assert_eq!(duplicate.inserted, 0); + assert_eq!(duplicate.duplicates, 2); + assert_eq!(store.stats().expect("stats").events, 2); +} + +#[test] +fn tool_input_indexing_requires_explicit_opt_in() { + let input = concat!( + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_tool\",", + "\"session_id\":\"ses_tool\",\"timestamp\":\"2026-07-16T07:00:00Z\",", + "\"source\":\"generic-jsonl\",\"type\":\"tool.failed\",", + "\"tool\":{\"name\":\"bash\",\"input\":\"approved search phrase\",\"exit_code\":1}}\n" + ); + let mut store = EventStore::open_in_memory().expect("store"); + let mut options = ImportOptions::new("fixture:tool"); + options.index_tool_input = true; + + let summary = import_jsonl(Cursor::new(input), Some(&mut store), &options).expect("import"); + assert_eq!(summary.inserted, 1); + assert_eq!(store.search("approved", 10).expect("search").len(), 1); +} + +#[test] +fn dry_run_validates_without_a_store_or_writes() { + let mut options = ImportOptions::new("fixture:dry-run"); + options.dry_run = true; + + let summary = import_jsonl(Cursor::new(MIXED), None, &options).expect("dry run"); + assert!(summary.dry_run); + assert_eq!(summary.validated, 3); + assert_eq!(summary.inserted, 0); + assert_eq!(summary.skipped, 0); + assert_eq!(summary.rejected, 2); +} + +#[test] +fn same_session_sequence_is_a_record_diagnostic_not_a_fatal_error() { + let input = concat!( + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_one\",", + "\"session_id\":\"ses_sequence\",\"timestamp\":\"2026-07-16T08:00:00Z\",", + "\"sequence\":0,\"source\":\"generic-jsonl\",\"type\":\"session.started\"}\n", + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_two\",", + "\"session_id\":\"ses_sequence\",\"timestamp\":\"2026-07-16T08:01:00Z\",", + "\"sequence\":0,\"source\":\"generic-jsonl\",\"type\":\"session.ended\"}\n", + "{\"spec_version\":\"aep/0.1\",\"event_id\":\"evt_three\",", + "\"session_id\":\"ses_sequence\",\"timestamp\":\"2026-07-16T08:02:00Z\",", + "\"sequence\":1,\"source\":\"generic-jsonl\",\"type\":\"session.ended\"}\n" + ); + let mut store = EventStore::open_in_memory().expect("store"); + let summary = import_jsonl( + Cursor::new(input), + Some(&mut store), + &ImportOptions::new("fixture:sequence"), + ) + .expect("import should continue"); + + assert_eq!(summary.inserted, 2); + assert_eq!(summary.rejected, 1); + assert_eq!(summary.diagnostics[0].line, 2); + assert_eq!( + summary.diagnostics[0].code, + ImportDiagnosticCode::StoreRejected + ); +} diff --git a/crates/autophagy-store/Cargo.toml b/crates/autophagy-store/Cargo.toml index 4fdf4f6..5f2bc6e 100644 --- a/crates/autophagy-store/Cargo.toml +++ b/crates/autophagy-store/Cargo.toml @@ -11,6 +11,7 @@ publish = false [dependencies] autophagy-events = { path = "../autophagy-events" } rusqlite = { version = "0.40.1", default-features = false, features = ["bundled"] } +serde.workspace = true serde_json.workspace = true sha2.workspace = true thiserror.workspace = true diff --git a/crates/autophagy-store/src/error.rs b/crates/autophagy-store/src/error.rs index 587dc27..1009f53 100644 --- a/crates/autophagy-store/src/error.rs +++ b/crates/autophagy-store/src/error.rs @@ -38,6 +38,18 @@ pub enum StoreError { /// Conflicting session identifier. session_id: String, }, + /// A sequence position was already occupied by a different event. + #[error( + "session '{session_id}' sequence {sequence} already belongs to event '{existing_event_id}'" + )] + SessionSequenceConflict { + /// Conflicting session identifier. + session_id: String, + /// Conflicting sequence position. + sequence: i64, + /// Canonical event already stored at this position. + existing_event_id: String, + }, /// An unsigned sequence cannot fit `SQLite`'s signed integer representation. #[error("event sequence {sequence} exceeds SQLite's integer range")] SequenceOutOfRange { diff --git a/crates/autophagy-store/src/model.rs b/crates/autophagy-store/src/model.rs index 46d4fdc..293bde4 100644 --- a/crates/autophagy-store/src/model.rs +++ b/crates/autophagy-store/src/model.rs @@ -1,5 +1,7 @@ +use serde::Serialize; + /// Stable provenance for one adapter installation or history directory. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct SourceIdentity { /// Adapter identifier; must match the AEP event's `source` value. pub adapter: String, @@ -32,7 +34,7 @@ impl SourceIdentity { /// /// Project paths and tool names come from the already policy-processed AEP /// envelope. Tool input and event payload text require this explicit projection. -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] pub struct SearchProjection { /// Sanitized tool input. Raw event input is not indexed automatically. pub tool_input_text: Option, @@ -41,7 +43,8 @@ pub struct SearchProjection { } /// Result of attempting to persist one event. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "status", rename_all = "snake_case")] pub enum InsertOutcome { /// A new immutable event row was created. Inserted { @@ -63,7 +66,7 @@ pub enum InsertOutcome { } /// Compact session record returned by storage queries. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct SessionSummary { /// AEP session identifier. pub session_id: String, @@ -86,7 +89,7 @@ pub struct SessionSummary { } /// One full-text search result. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct SearchHit { /// Exact AEP evidence identifier. pub event_id: String, @@ -97,7 +100,7 @@ pub struct SearchHit { } /// Row counts useful for diagnostics and idempotency assertions. -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] pub struct StoreStats { /// Number of source instances. pub sources: i64, @@ -112,7 +115,7 @@ pub struct StoreStats { } /// Effect of deleting one session. -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] pub struct DeleteSummary { /// Whether a session row existed and was removed. pub session_deleted: bool, diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index 0a95b7f..ce66f43 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -112,6 +112,7 @@ impl EventStore { let source_id = upsert_source(&transaction, source, &occurred_at)?; ensure_session(&transaction, source_id, event, &occurred_at)?; + ensure_sequence_available(&transaction, event, sequence)?; let tool_name = event.tool.as_ref().map(|tool| tool.name.as_str()); let exit_code = event.tool.as_ref().and_then(|tool| tool.exit_code); @@ -224,6 +225,47 @@ impl EventStore { .optional()?) } + /// List the most recently active sessions with their source provenance. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn list_sessions(&self, limit: u32) -> Result, StoreError> { + if limit == 0 { + return Ok(Vec::new()); + } + let mut statement = self.connection.prepare( + "SELECT + sessions.session_id, + sources.adapter, + sources.instance_key, + sessions.project_path, + sessions.started_at, + sessions.ended_at, + sessions.first_event_at, + sessions.last_event_at, + sessions.event_count + FROM sessions + JOIN sources USING (source_id) + ORDER BY sessions.last_event_at DESC, sessions.session_id + LIMIT ?1", + )?; + let rows = statement.query_map([i64::from(limit)], |row| { + Ok(SessionSummary { + session_id: row.get(0)?, + adapter: row.get(1)?, + instance_key: row.get(2)?, + project_path: row.get(3)?, + started_at: row.get(4)?, + ended_at: row.get(5)?, + first_event_at: row.get(6)?, + last_event_at: row.get(7)?, + event_count: row.get(8)?, + }) + })?; + Ok(rows.collect::>()?) + } + /// Search the explicit redaction-approved FTS5 projection. /// /// # Errors @@ -494,6 +536,31 @@ fn ensure_session( Ok(()) } +fn ensure_sequence_available( + transaction: &Transaction<'_>, + event: &Event, + sequence: Option, +) -> Result<(), StoreError> { + let Some(sequence) = sequence else { + return Ok(()); + }; + if let Some(existing_event_id) = transaction + .query_row( + "SELECT event_id FROM events WHERE session_id = ?1 AND sequence = ?2", + params![event.session_id.as_str(), sequence], + |row| row.get::<_, String>(0), + ) + .optional()? + { + return Err(StoreError::SessionSequenceConflict { + session_id: event.session_id.to_string(), + sequence, + existing_event_id, + }); + } + Ok(()) +} + fn insert_artifacts( transaction: &Transaction<'_>, event_row_id: i64, diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index 8d6709f..caa9c73 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -100,6 +100,13 @@ fn insertion_rolls_up_sessions_and_indexes_only_approved_text() { assert_eq!(session.started_at.as_deref(), Some("2026-07-16T01:00:00Z")); assert_eq!(session.ended_at.as_deref(), Some("2026-07-16T01:20:00Z")); assert_eq!(session.project_path.as_deref(), Some("/workspace/project")); + assert_eq!(store.list_sessions(1).expect("session list"), vec![session]); + assert!( + store + .list_sessions(0) + .expect("empty session list") + .is_empty() + ); let hits = store.search("generated", 10).expect("approved search"); assert_eq!(hits.len(), 1); @@ -256,7 +263,7 @@ fn provenance_and_sequence_conflicts_roll_back_atomically() { &sequence_conflict, &SearchProjection::default() ), - Err(StoreError::Database(_)) + Err(StoreError::SessionSequenceConflict { .. }) )); assert_eq!( store.stats().expect("stats"), diff --git a/docs/architecture/repository-structure.md b/docs/architecture/repository-structure.md index e2bfb90..43ffac6 100644 --- a/docs/architecture/repository-structure.md +++ b/docs/architecture/repository-structure.md @@ -43,9 +43,9 @@ autophagy/ └── website/ ``` -`autophagy-events` and `autophagy-store` exist through PR 2. A crate or package -is added when its PR contains an executable vertical slice; empty placeholder -crates are avoided. +`autophagy-events`, `autophagy-store`, `autophagy-core`, and `autophagy-cli` +exist through PR 3. A crate or package is added when its PR contains an +executable vertical slice; empty placeholder crates are avoided. ## Dependency direction diff --git a/docs/guides/generic-jsonl.md b/docs/guides/generic-jsonl.md new file mode 100644 index 0000000..b47189a --- /dev/null +++ b/docs/guides/generic-jsonl.md @@ -0,0 +1,92 @@ +# Generic AEP JSONL import + +The generic importer accepts UTF-8 JSON Lines containing one complete Agent +Event Protocol v0.1 event per nonblank line. It streams input, validates each +record independently, and retains bounded line-addressed diagnostics so one bad +record does not hide later valid evidence. + +## Database location + +Pass `--database PATH` or set `AUTOPHAGY_DB`. Without either, Autophagy uses the +platform-local application data directory. + +## Preview safely + +Dry-run parses, validates, and applies project selection without opening or +creating a database: + +```sh +autophagy --output json import sessions.jsonl --dry-run +``` + +## Import a file + +Give each persisted input or producer a stable instance key. Reimporting the +same events is safe and reports duplicates without changing canonical rows. + +```sh +autophagy import sessions.jsonl \ + --instance-key laptop-history \ + --display-name "Laptop export" +``` + +Use repeated exact project selections to import only approved repositories: + +```sh +autophagy import sessions.jsonl \ + --instance-key laptop-history \ + --project /work/service-a \ + --project /work/service-b +``` + +Use `-` or omit the file to read standard input: + +```sh +producer | autophagy import - --instance-key live-pipe +``` + +## Search privacy + +Validated, path-policy-processed project paths and tool names are searchable. +Raw event JSON and raw tool input are not copied into FTS5 by default. + +Only enable these flags after confirming the selected source fields are already +redacted: + +```sh +autophagy import sessions.jsonl \ + --instance-key redacted-export \ + --index-tool-input \ + --index-metadata summary \ + --index-metadata correction +``` + +Search accepts an SQLite FTS5 query expression: + +```sh +autophagy search '"generated client"' +autophagy search 'tool AND failed' --limit 10 +``` + +## Sessions and machine-readable output + +```sh +autophagy sessions --limit 25 +autophagy --output json sessions +autophagy --output json search stale +``` + +JSON output is a tagged object with `command` and `result` fields. Import +results include line, event, insertion, duplicate, conflict, project-skip, and +rejection counts plus bounded diagnostics. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Command completed without rejected or conflicting events. | +| `1` | Fatal I/O, configuration, migration, or database failure. | +| `2` | Import completed, but one or more records were rejected or quarantined. | + +Exit code `2` may accompany successful inserts. Read the import summary before +deciding whether to retry, repair the source, or inspect conflict evidence. diff --git a/evals/fixtures/generic-jsonl/demo.jsonl b/evals/fixtures/generic-jsonl/demo.jsonl new file mode 100644 index 0000000..0749dfa --- /dev/null +++ b/evals/fixtures/generic-jsonl/demo.jsonl @@ -0,0 +1,3 @@ +{"spec_version":"aep/0.1","event_id":"evt_demo_start","session_id":"ses_demo","timestamp":"2026-07-16T10:00:00Z","sequence":0,"source":"generic-jsonl","type":"session.started","project":"/workspace/demo"} +{"spec_version":"aep/0.1","event_id":"evt_demo_failure","session_id":"ses_demo","timestamp":"2026-07-16T10:01:00Z","sequence":1,"source":"generic-jsonl","type":"tool.failed","project":"/workspace/demo","tool":{"name":"bash","input":"cargo test","exit_code":1},"metadata":{"summary":"schema changed; generated client was stale"},"artifacts":[{"type":"file","path":"schema.graphql"}]} +{"spec_version":"aep/0.1","event_id":"evt_demo_end","session_id":"ses_demo","timestamp":"2026-07-16T10:02:00Z","sequence":2,"source":"generic-jsonl","type":"session.ended","project":"/workspace/demo"}