diff --git a/Cargo.lock b/Cargo.lock index 0592d0a..5635a40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,6 +165,20 @@ dependencies = [ "time", ] +[[package]] +name = "autophagy-patterns" +version = "0.1.0-alpha.1" +dependencies = [ + "autophagy-core", + "autophagy-events", + "autophagy-store", + "jsonschema", + "serde", + "serde_json", + "sha2", + "time", +] + [[package]] name = "autophagy-store" version = "0.1.0-alpha.1" diff --git a/Cargo.toml b/Cargo.toml index 1b77d36..d9a2245 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/autophagy-cli", "crates/autophagy-core", "crates/autophagy-events", + "crates/autophagy-patterns", "crates/autophagy-store", ] diff --git a/README.md b/README.md index 44fc3c0..032c83d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ change because of what happened?” Autophagy is in foundation development. Agent Event Protocol (AEP) v0.1, the transactional local SQLite event store, generic JSONL CLI vertical slice, and incremental Claude Code and Codex history adapters are implemented. No daemon -or background capture ships yet. +or background capture ships yet. Deterministic repeated-failure and explicit +user-correction detectors now emit versioned, evidence-linked packets. ## Principles @@ -33,6 +34,7 @@ crates/autophagy-adapter-test-support/ Shared native-adapter conformance checks crates/autophagy-cli/ User-facing import, sessions, and search commands crates/autophagy-core/ Reusable streaming import application services crates/autophagy-events/ AEP Rust types, parsing, and validation +crates/autophagy-patterns/ Model-free recurrence detectors and evidence packets crates/autophagy-store/ SQLite migrations, idempotency, FTS, and deletion docs/architecture/ Planned component and storage boundaries docs/blueprint/ Complete normalized product and implementation brief @@ -88,6 +90,10 @@ mise exec -- cargo run -p autophagy-cli -- --output json \ The [Codex adapter guide](docs/guides/codex.md) documents its intentionally narrow compatibility matrix and the upstream transcript-stability boundary. +The [deterministic findings guide](docs/guides/deterministic-findings.md) +documents recurrence thresholds, signature normalization, counterexamples, and +the versioned Evidence Packet contract. + ## Try the contract Install [mise](https://mise.jdx.dev/), then run: diff --git a/crates/autophagy-patterns/Cargo.toml b/crates/autophagy-patterns/Cargo.toml new file mode 100644 index 0000000..15fefe7 --- /dev/null +++ b/crates/autophagy-patterns/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "autophagy-patterns" +description = "Deterministic, evidence-linked pattern detectors for Autophagy" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +autophagy-events = { path = "../autophagy-events" } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +time.workspace = true + +[dev-dependencies] +autophagy-core = { path = "../autophagy-core" } +autophagy-store = { path = "../autophagy-store" } +jsonschema = { version = "0.47", default-features = false } + +[lints] +workspace = true diff --git a/crates/autophagy-patterns/src/correction.rs b/crates/autophagy-patterns/src/correction.rs new file mode 100644 index 0000000..367c2a2 --- /dev/null +++ b/crates/autophagy-patterns/src/correction.rs @@ -0,0 +1,62 @@ +use std::collections::BTreeMap; + +use autophagy_events::{Event, EventKind}; + +use crate::{ + DetectorConfig, DetectorKind, EvidencePacket, EvidenceReference, EvidenceSpecVersion, + score::{qualifies, score}, + signature::{correction_counterexample, correction_signature, finding_id}, +}; + +pub(crate) fn detect(events: &[Event], config: DetectorConfig) -> Vec { + let mut corrections: BTreeMap> = BTreeMap::new(); + let mut counterexamples: BTreeMap> = BTreeMap::new(); + for event in events { + if event.kind == EventKind::UserCorrectedAgent { + if let Some(signature) = correction_signature(event) { + corrections.entry(signature).or_default().push(event); + } + } else if event.kind == EventKind::DecisionRecorded { + if let Some(signature) = correction_counterexample(event) { + counterexamples.entry(signature).or_default().push(event); + } + } + } + + corrections + .into_iter() + .filter_map(|(signature, evidence)| { + let opposite = counterexamples + .get(&signature) + .map_or(&[][..], Vec::as_slice); + let recurrence = score(&evidence, opposite)?; + let packet_signature = format!("correction/v1|{signature}"); + qualifies(&recurrence, config).then(|| EvidencePacket { + spec_version: EvidenceSpecVersion::V0_1, + finding_id: finding_id( + DetectorKind::RepeatedUserCorrection.as_str(), + &packet_signature, + ), + detector: DetectorKind::RepeatedUserCorrection, + signature: packet_signature, + title: format!("Repeated user correction: {signature}"), + score: recurrence, + evidence: references(&evidence), + counterexamples: references(opposite), + }) + }) + .collect() +} + +fn references(events: &[&Event]) -> Vec { + let mut references = events + .iter() + .map(|event| EvidenceReference::from_event(event)) + .collect::>(); + references.sort_by(|left, right| { + left.timestamp + .cmp(&right.timestamp) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + references +} diff --git a/crates/autophagy-patterns/src/evidence.rs b/crates/autophagy-patterns/src/evidence.rs new file mode 100644 index 0000000..3e9eaf6 --- /dev/null +++ b/crates/autophagy-patterns/src/evidence.rs @@ -0,0 +1,97 @@ +use autophagy_events::Event; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +/// Evidence packet wire-format version. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum EvidenceSpecVersion { + /// Initial evidence packet contract. + #[serde(rename = "evidence/0.1")] + V0_1, +} + +/// Deterministic detector that produced a finding. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DetectorKind { + /// The same normalized command failed recurrently. + RepeatedCommandFailure, + /// The same explicitly classified user correction recurred. + RepeatedUserCorrection, +} + +impl DetectorKind { + /// Stable serialized detector name. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::RepeatedCommandFailure => "repeated_command_failure", + Self::RepeatedUserCorrection => "repeated_user_correction", + } + } +} + +/// Exact AEP event cited as support or a counterexample. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct EvidenceReference { + /// Stable event evidence identifier. + pub event_id: String, + /// Source session containing the event. + pub session_id: String, + /// Canonical occurrence timestamp. + #[serde(with = "time::serde::rfc3339")] + pub timestamp: OffsetDateTime, + /// AEP event kind. + pub event_type: String, + /// Policy-processed project path, when available. + #[serde(skip_serializing_if = "Option::is_none")] + pub project: Option, +} + +impl EvidenceReference { + pub(crate) fn from_event(event: &Event) -> Self { + Self { + event_id: event.event_id.as_str().to_owned(), + session_id: event.session_id.as_str().to_owned(), + timestamp: event.timestamp, + event_type: event.kind.as_str().to_owned(), + project: event.project.clone(), + } + } +} + +/// Inspectable integer-only recurrence score. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RecurrenceScore { + /// Supporting events. + pub occurrences: u32, + /// Distinct sessions containing support. + pub distinct_sessions: u32, + /// Events demonstrating the opposite outcome. + pub counterexamples: u32, + /// Support share among support and counterexamples, in basis points. + pub support_ratio_bps: u16, + /// Overall deterministic score in basis points. + pub score_bps: u16, +} + +/// Versioned, model-free finding with exact evidence lineage. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct EvidencePacket { + /// Evidence packet contract version. + pub spec_version: EvidenceSpecVersion, + /// Stable content-derived finding identity. + pub finding_id: String, + /// Detector implementation that emitted the packet. + pub detector: DetectorKind, + /// Versioned normalized recurrence signature. + pub signature: String, + /// Deterministic human-readable label. + pub title: String, + /// Inspectable recurrence statistics and score. + pub score: RecurrenceScore, + /// Supporting events in canonical order. + pub evidence: Vec, + /// Opposite-outcome events in canonical order. + pub counterexamples: Vec, +} diff --git a/crates/autophagy-patterns/src/failure.rs b/crates/autophagy-patterns/src/failure.rs new file mode 100644 index 0000000..94ef30d --- /dev/null +++ b/crates/autophagy-patterns/src/failure.rs @@ -0,0 +1,98 @@ +use std::collections::BTreeMap; + +use autophagy_events::{Event, EventKind}; + +use crate::{ + DetectorConfig, DetectorKind, EvidencePacket, EvidenceReference, EvidenceSpecVersion, + score::{qualifies, score}, + signature::{FailureOperation, failure_operation, finding_id}, +}; + +struct FailureGroup<'a> { + label: String, + success_key: String, + events: Vec<&'a Event>, +} + +pub(crate) fn detect(events: &[Event], config: DetectorConfig) -> Vec { + let mut failures: BTreeMap> = BTreeMap::new(); + let mut successes: BTreeMap> = BTreeMap::new(); + + for event in events { + match event.kind { + EventKind::ToolFailed => { + if let Some(operation) = failure_operation(event, true) { + add_failure(&mut failures, operation, event); + } + } + EventKind::ToolCompleted => { + if let Some(operation) = failure_operation(event, false) { + successes + .entry(operation.success_key) + .or_default() + .push(event); + } + } + _ => {} + } + } + + failures + .into_iter() + .filter_map(|(signature, group)| { + let counterexamples = successes + .get(&group.success_key) + .map_or(&[][..], Vec::as_slice); + let recurrence = score(&group.events, counterexamples)?; + qualifies(&recurrence, config).then(|| EvidencePacket { + spec_version: EvidenceSpecVersion::V0_1, + finding_id: finding_id(DetectorKind::RepeatedCommandFailure.as_str(), &signature), + detector: DetectorKind::RepeatedCommandFailure, + signature, + title: format!("Repeated command failure: {}", truncate(&group.label, 160)), + score: recurrence, + evidence: references(&group.events), + counterexamples: references(counterexamples), + }) + }) + .collect() +} + +fn add_failure<'a>( + failures: &mut BTreeMap>, + operation: FailureOperation, + event: &'a Event, +) { + failures + .entry(operation.signature) + .or_insert_with(|| FailureGroup { + label: operation.label, + success_key: operation.success_key, + events: Vec::new(), + }) + .events + .push(event); +} + +fn references(events: &[&Event]) -> Vec { + let mut references = events + .iter() + .map(|event| EvidenceReference::from_event(event)) + .collect::>(); + references.sort_by(|left, right| { + left.timestamp + .cmp(&right.timestamp) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + references +} + +fn truncate(value: &str, limit: usize) -> String { + let mut chars = value.chars(); + let prefix = chars.by_ref().take(limit).collect::(); + if chars.next().is_some() { + format!("{prefix}…") + } else { + prefix + } +} diff --git a/crates/autophagy-patterns/src/lib.rs b/crates/autophagy-patterns/src/lib.rs new file mode 100644 index 0000000..26845d1 --- /dev/null +++ b/crates/autophagy-patterns/src/lib.rs @@ -0,0 +1,46 @@ +//! Deterministic pattern discovery over validated AEP events. +//! +//! Detectors never call a model. Every finding carries exact supporting and +//! counterexample event identifiers and uses integer-only recurrence scoring. + +mod correction; +mod evidence; +mod failure; +mod score; +mod signature; + +pub use evidence::{ + DetectorKind, EvidencePacket, EvidenceReference, EvidenceSpecVersion, RecurrenceScore, +}; + +use autophagy_events::Event; + +/// Thresholds shared by deterministic recurrence detectors. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DetectorConfig { + /// Minimum supporting events required. + pub min_occurrences: u32, + /// Minimum distinct sessions containing support. + pub min_sessions: u32, + /// Minimum evidence share among support and counterexamples, in basis points. + pub min_support_ratio_bps: u16, +} + +impl Default for DetectorConfig { + fn default() -> Self { + Self { + min_occurrences: 3, + min_sessions: 2, + min_support_ratio_bps: 5_000, + } + } +} + +/// Run every Milestone 1 deterministic detector and return stable ordering. +#[must_use] +pub fn detect(events: &[Event], config: DetectorConfig) -> Vec { + let mut findings = failure::detect(events, config); + findings.extend(correction::detect(events, config)); + findings.sort_by(|left, right| left.finding_id.cmp(&right.finding_id)); + findings +} diff --git a/crates/autophagy-patterns/src/score.rs b/crates/autophagy-patterns/src/score.rs new file mode 100644 index 0000000..696208e --- /dev/null +++ b/crates/autophagy-patterns/src/score.rs @@ -0,0 +1,43 @@ +use std::collections::BTreeSet; + +use autophagy_events::Event; + +use crate::{DetectorConfig, RecurrenceScore}; + +pub(crate) fn score(evidence: &[&Event], counterexamples: &[&Event]) -> Option { + let occurrences = u32::try_from(evidence.len()).ok()?; + let counterexample_count = u32::try_from(counterexamples.len()).ok()?; + let distinct_sessions = u32::try_from( + evidence + .iter() + .map(|event| event.session_id.as_str()) + .collect::>() + .len(), + ) + .ok()?; + let total = occurrences.saturating_add(counterexample_count); + let support_ratio_bps = if total == 0 { + 0 + } else { + u16::try_from((u64::from(occurrences) * 10_000) / u64::from(total)).unwrap_or(10_000) + }; + let occurrence_component = occurrences.saturating_sub(1).min(6) * 500; + let session_component = distinct_sessions.saturating_sub(1).min(4) * 750; + let score = (u32::from(support_ratio_bps) * 6 / 10) + .saturating_add(occurrence_component) + .saturating_add(session_component) + .min(10_000); + Some(RecurrenceScore { + occurrences, + distinct_sessions, + counterexamples: counterexample_count, + support_ratio_bps, + score_bps: u16::try_from(score).unwrap_or(10_000), + }) +} + +pub(crate) const fn qualifies(score: &RecurrenceScore, config: DetectorConfig) -> bool { + score.occurrences >= config.min_occurrences + && score.distinct_sessions >= config.min_sessions + && score.support_ratio_bps >= config.min_support_ratio_bps +} diff --git a/crates/autophagy-patterns/src/signature.rs b/crates/autophagy-patterns/src/signature.rs new file mode 100644 index 0000000..758e557 --- /dev/null +++ b/crates/autophagy-patterns/src/signature.rs @@ -0,0 +1,99 @@ +use std::fmt::Write as _; + +use autophagy_events::Event; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +pub(crate) struct FailureOperation { + pub signature: String, + pub success_key: String, + pub label: String, +} + +pub(crate) fn failure_operation(event: &Event, include_exit: bool) -> Option { + let tool = event.tool.as_ref()?; + let tool_name = normalize_tool(&tool.name); + let command = command(tool.input.as_ref()?)?; + let command = normalize_command(&command, event.project.as_deref()); + if command.is_empty() { + return None; + } + let success_key = format!("operation/v1|{tool_name}|{command}"); + let signature = if include_exit { + format!("failure/v1|{tool_name}|{command}|exit:{}", tool.exit_code?) + } else { + success_key.clone() + }; + Some(FailureOperation { + signature, + success_key, + label: format!("{tool_name}: {command}"), + }) +} + +pub(crate) fn correction_signature(event: &Event) -> Option { + [ + "autophagy.signature", + "correction_signature", + "correction_key", + ] + .iter() + .find_map(|key| event.metadata.get(*key).and_then(Value::as_str)) + .map(normalize_label) + .filter(|value| !value.is_empty()) +} + +pub(crate) fn correction_counterexample(event: &Event) -> Option { + let outcome = event + .metadata + .get("autophagy.outcome") + .or_else(|| event.metadata.get("correction_outcome")) + .and_then(Value::as_str)?; + matches!(outcome, "followed" | "accepted" | "complied") + .then(|| correction_signature(event)) + .flatten() +} + +pub(crate) fn finding_id(detector: &str, signature: &str) -> String { + let digest = Sha256::digest(format!("{detector}\0{signature}").as_bytes()); + let mut encoded = String::with_capacity(digest.len() * 2); + for byte in digest { + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + format!("fnd_{encoded}") +} + +fn normalize_tool(tool: &str) -> String { + match tool.trim().to_ascii_lowercase().as_str() { + "bash" | "exec" | "exec_command" | "shell" | "terminal" => "shell".to_owned(), + other => other.to_owned(), + } +} + +fn command(input: &Value) -> Option { + match input { + Value::String(value) => Some(value.clone()), + Value::Object(object) => object + .get("command") + .or_else(|| object.get("cmd")) + .and_then(Value::as_str) + .map(str::to_owned), + _ => None, + } +} + +fn normalize_command(command: &str, project: Option<&str>) -> String { + let command = project.map_or_else( + || command.to_owned(), + |project| command.replace(project, "$PROJECT"), + ); + command.split_whitespace().collect::>().join(" ") +} + +fn normalize_label(label: &str) -> String { + label + .split_whitespace() + .map(str::to_ascii_lowercase) + .collect::>() + .join(" ") +} diff --git a/crates/autophagy-patterns/tests/detectors.rs b/crates/autophagy-patterns/tests/detectors.rs new file mode 100644 index 0000000..01ea75f --- /dev/null +++ b/crates/autophagy-patterns/tests/detectors.rs @@ -0,0 +1,92 @@ +//! End-to-end deterministic detector and evidence-contract tests. + +use std::io::Cursor; + +use autophagy_core::{ImportOptions, import_jsonl}; +use autophagy_patterns::{DetectorConfig, DetectorKind, detect}; +use autophagy_store::EventStore; + +const CORPUS: &str = include_str!("../../../evals/fixtures/findings/deterministic.jsonl"); + +#[test] +fn demo_corpus_produces_two_stable_evidence_linked_findings() { + let mut store = EventStore::open_in_memory().expect("store"); + let options = ImportOptions::new("fixture:deterministic-findings"); + let imported = import_jsonl(Cursor::new(CORPUS), Some(&mut store), &options).expect("import"); + assert_eq!(imported.inserted, 11); + assert_eq!(imported.rejected, 0); + + let events = store.list_events_for_detection(None).expect("events"); + let findings = detect(&events, DetectorConfig::default()); + assert_eq!(findings.len(), 2); + assert_eq!(findings, detect(&events, DetectorConfig::default())); + let mut reversed = events.clone(); + reversed.reverse(); + assert_eq!(findings, detect(&reversed, DetectorConfig::default())); + + let failure = findings + .iter() + .find(|finding| finding.detector == DetectorKind::RepeatedCommandFailure) + .expect("failure finding"); + assert_eq!(failure.score.occurrences, 3); + assert_eq!(failure.score.distinct_sessions, 3); + assert_eq!(failure.score.counterexamples, 1); + assert_eq!(failure.evidence[0].event_id, "evt_failure_1"); + assert_eq!(failure.counterexamples[0].event_id, "evt_success_1"); + + let correction = findings + .iter() + .find(|finding| finding.detector == DetectorKind::RepeatedUserCorrection) + .expect("correction finding"); + assert_eq!(correction.score.occurrences, 3); + assert_eq!(correction.score.distinct_sessions, 2); + assert_eq!(correction.score.counterexamples, 1); + assert!( + correction + .evidence + .iter() + .all(|item| item.event_id.starts_with("evt_correction_")) + ); + + let schema: serde_json::Value = + serde_json::from_str(include_str!("../../../docs/specs/evidence/0.1/schema.json")) + .expect("schema JSON"); + let validator = jsonschema::validator_for(&schema).expect("compile schema"); + for finding in &findings { + let instance = serde_json::to_value(finding).expect("finding JSON"); + assert!(validator.is_valid(&instance), "schema rejected {instance}"); + } +} + +#[test] +fn below_threshold_corpus_produces_no_findings() { + let mut store = EventStore::open_in_memory().expect("store"); + let options = ImportOptions::new("fixture:below-threshold"); + import_jsonl(Cursor::new(CORPUS), Some(&mut store), &options).expect("import"); + let events = store.list_events_for_detection(None).expect("events"); + let config = DetectorConfig { + min_occurrences: 4, + ..DetectorConfig::default() + }; + assert!(detect(&events, config).is_empty()); +} + +#[test] +fn exact_project_selection_limits_detector_input() { + let mut store = EventStore::open_in_memory().expect("store"); + let options = ImportOptions::new("fixture:project-query"); + import_jsonl(Cursor::new(CORPUS), Some(&mut store), &options).expect("import"); + assert_eq!( + store + .list_events_for_detection(Some("/workspace/demo")) + .expect("selected") + .len(), + 11 + ); + assert!( + store + .list_events_for_detection(Some("/workspace/other")) + .expect("excluded") + .is_empty() + ); +} diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index 5ed6b96..d4c66d2 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -365,6 +365,34 @@ impl EventStore { Ok(rows.collect::>()?) } + /// Return canonical events in deterministic evidence order. + /// + /// An exact project path limits the result when supplied. This deliberately + /// returns validated AEP envelopes rather than exposing `SQLite` rows to + /// detector crates. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` fails or persisted event JSON no + /// longer satisfies the AEP contract. + pub fn list_events_for_detection( + &self, + project: Option<&str>, + ) -> Result, StoreError> { + let mut statement = self.connection.prepare( + "SELECT event_json + FROM events + WHERE (?1 IS NULL OR project_path = ?1) + ORDER BY occurred_at, session_id, coalesce(sequence, 9223372036854775807), row_id", + )?; + let rows = statement.query_map([project], |row| row.get::<_, String>(0))?; + rows.map(|row| { + let json = row?; + Event::from_json_str(&json).map_err(StoreError::from) + }) + .collect() + } + /// Search the explicit redaction-approved FTS5 projection. /// /// # Errors diff --git a/docs/architecture/repository-structure.md b/docs/architecture/repository-structure.md index 2331889..974b99e 100644 --- a/docs/architecture/repository-structure.md +++ b/docs/architecture/repository-structure.md @@ -45,8 +45,9 @@ autophagy/ `autophagy-events`, `autophagy-store`, `autophagy-core`, `autophagy-cli`, the native Claude Code and Codex adapters, and their shared conformance harness -exist through PR 5. A crate or package is added when its PR contains an -executable vertical slice; empty placeholder crates are avoided. +exist through PR 5. `autophagy-patterns` and Evidence Packet v0.1 exist through +PR 6. A crate or package is added when its PR contains an executable vertical +slice; empty placeholder crates are avoided. ## Dependency direction diff --git a/docs/guides/deterministic-findings.md b/docs/guides/deterministic-findings.md new file mode 100644 index 0000000..b15451b --- /dev/null +++ b/docs/guides/deterministic-findings.md @@ -0,0 +1,55 @@ +# Deterministic findings + +Autophagy's first detectors are local, model-free functions over validated AEP +events. They return Evidence Packet v0.1 values; they do not persist findings, +generate prose with a model, or execute a proposed intervention. + +The normative output schema is +[`docs/specs/evidence/0.1/schema.json`](../specs/evidence/0.1/schema.json). +Each finding contains its stable detector signature, inspectable integer score, +exact supporting event IDs, and exact counterexample IDs. + +## Default recurrence policy + +A group becomes a finding only when it has: + +- at least three supporting events; +- support in at least two distinct sessions; and +- at least 50% support among support plus explicit counterexamples. + +The detector API accepts different thresholds for evaluation, but production +callers should keep the defaults until fixture precision justifies a change. + +## Repeated command failures + +The failure detector considers only `tool.failed` events with a non-zero exit +code and an inspectable string command. It normalizes common shell tool aliases, +collapses whitespace, replaces the event's exact project prefix with +`$PROJECT`, and groups by normalized operation plus exit code. + +A matching `tool.completed` operation is an explicit counterexample. It lowers +both the support ratio and overall score. Different commands, exit codes, and +non-shell structured tools do not get merged speculatively. + +## Repeated user corrections + +The correction detector considers only explicit `user.corrected_agent` events. +Because native adapters intentionally do not infer corrections from private +prompt text, grouping requires one of these string metadata keys: + +- `autophagy.signature` +- `correction_signature` +- `correction_key` + +Whitespace and case are normalized. A `decision.recorded` event with the same +signature and an explicit `followed`, `accepted`, or `complied` outcome is a +counterexample. Unclassified corrections remain available as evidence but do +not produce a potentially misleading finding. + +## Evaluation corpus + +The anonymized corpus at +[`evals/fixtures/findings/deterministic.jsonl`](../../evals/fixtures/findings/deterministic.jsonl) +contains both supported patterns and counterexamples. Contract tests prove that +it emits exactly two stable packets regardless of input order, while a threshold +above its recurrence count emits none. diff --git a/docs/specs/evidence/0.1/README.md b/docs/specs/evidence/0.1/README.md new file mode 100644 index 0000000..91f686e --- /dev/null +++ b/docs/specs/evidence/0.1/README.md @@ -0,0 +1,39 @@ +# Evidence Packet v0.1 + +Evidence Packet v0.1 is the versioned output of Autophagy's deterministic +detectors. The normative contract is [`schema.json`](schema.json). + +Every packet contains a stable content-derived finding ID, detector and +signature versions, an inspectable integer score, exact supporting AEP event +IDs, and exact counterexample IDs. Packets contain no model-generated claims. + +## Default recurrence threshold + +A finding requires at least three supporting events across at least two +sessions. Support must represent at least 50% of support plus counterexamples. +The basis-point score is deterministic: + +```text +score = min( + 10000, + support_ratio_bps * 0.6 + + min(occurrences - 1, 6) * 500 + + min(distinct_sessions - 1, 4) * 750 +) +``` + +The implementation uses integer arithmetic; the decimal expression above is +only explanatory. + +## Signatures and counterexamples + +Repeated command failures normalize common shell tool aliases to `shell`, +collapse command whitespace, replace the event's exact project prefix with +`$PROJECT`, and retain the non-zero exit code. A successful matching operation +is a counterexample. + +Repeated user corrections require an explicit string metadata key named +`autophagy.signature`, `correction_signature`, or `correction_key`. This avoids +guessing correction semantics from private prompt text. A `decision.recorded` +event with the same signature and an explicit `followed`, `accepted`, or +`complied` outcome is a counterexample. diff --git a/docs/specs/evidence/0.1/schema.json b/docs/specs/evidence/0.1/schema.json new file mode 100644 index 0000000..08a6395 --- /dev/null +++ b/docs/specs/evidence/0.1/schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://autophagy.sh/specs/evidence/0.1/schema.json", + "title": "Autophagy Evidence Packet v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "spec_version", + "finding_id", + "detector", + "signature", + "title", + "score", + "evidence", + "counterexamples" + ], + "properties": { + "spec_version": { "const": "evidence/0.1" }, + "finding_id": { + "type": "string", + "pattern": "^fnd_[A-Za-z0-9._:-]{1,127}$" + }, + "detector": { + "enum": ["repeated_command_failure", "repeated_user_correction"] + }, + "signature": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "score": { "$ref": "#/$defs/score" }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/reference" } + }, + "counterexamples": { + "type": "array", + "items": { "$ref": "#/$defs/reference" } + } + }, + "$defs": { + "score": { + "type": "object", + "additionalProperties": false, + "required": [ + "occurrences", + "distinct_sessions", + "counterexamples", + "support_ratio_bps", + "score_bps" + ], + "properties": { + "occurrences": { "type": "integer", "minimum": 1 }, + "distinct_sessions": { "type": "integer", "minimum": 1 }, + "counterexamples": { "type": "integer", "minimum": 0 }, + "support_ratio_bps": { "type": "integer", "minimum": 0, "maximum": 10000 }, + "score_bps": { "type": "integer", "minimum": 0, "maximum": 10000 } + } + }, + "reference": { + "type": "object", + "additionalProperties": false, + "required": ["event_id", "session_id", "timestamp", "event_type"], + "properties": { + "event_id": { "type": "string", "pattern": "^evt_[A-Za-z0-9._:-]{1,127}$" }, + "session_id": { "type": "string", "pattern": "^ses_[A-Za-z0-9._:-]{1,127}$" }, + "timestamp": { "type": "string", "format": "date-time" }, + "event_type": { "type": "string", "minLength": 1 }, + "project": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/evals/fixtures/findings/deterministic.jsonl b/evals/fixtures/findings/deterministic.jsonl new file mode 100644 index 0000000..91bc82b --- /dev/null +++ b/evals/fixtures/findings/deterministic.jsonl @@ -0,0 +1,11 @@ +{"spec_version":"aep/0.1","event_id":"evt_failure_1","session_id":"ses_failure_1","timestamp":"2026-07-16T10:00:00Z","sequence":0,"source":"generic-jsonl","type":"tool.failed","project":"/workspace/demo","tool":{"name":"bash","input":{"command":"mise run check"},"exit_code":1}} +{"spec_version":"aep/0.1","event_id":"evt_failure_2","session_id":"ses_failure_2","timestamp":"2026-07-16T10:01:00Z","sequence":0,"source":"generic-jsonl","type":"tool.failed","project":"/workspace/demo","tool":{"name":"exec_command","input":{"cmd":"mise run check"},"exit_code":1}} +{"spec_version":"aep/0.1","event_id":"evt_failure_3","session_id":"ses_failure_3","timestamp":"2026-07-16T10:02:00Z","sequence":0,"source":"generic-jsonl","type":"tool.failed","project":"/workspace/demo","tool":{"name":"shell","input":"mise run check","exit_code":1}} +{"spec_version":"aep/0.1","event_id":"evt_success_1","session_id":"ses_success_1","timestamp":"2026-07-16T10:03:00Z","sequence":0,"source":"generic-jsonl","type":"tool.completed","project":"/workspace/demo","tool":{"name":"exec","input":{"command":"mise run check"},"exit_code":0}} +{"spec_version":"aep/0.1","event_id":"evt_correction_1","session_id":"ses_correction_1","timestamp":"2026-07-16T10:04:00Z","sequence":0,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo","metadata":{"autophagy.signature":"run codegen before tests"}} +{"spec_version":"aep/0.1","event_id":"evt_correction_2","session_id":"ses_correction_1","timestamp":"2026-07-16T10:05:00Z","sequence":1,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo","metadata":{"correction_key":"Run codegen before tests"}} +{"spec_version":"aep/0.1","event_id":"evt_correction_3","session_id":"ses_correction_2","timestamp":"2026-07-16T10:06:00Z","sequence":0,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo","metadata":{"correction_signature":"run codegen before tests"}} +{"spec_version":"aep/0.1","event_id":"evt_correction_counterexample","session_id":"ses_correction_3","timestamp":"2026-07-16T10:07:00Z","sequence":0,"source":"generic-jsonl","type":"decision.recorded","project":"/workspace/demo","metadata":{"autophagy.signature":"run codegen before tests","autophagy.outcome":"followed"}} +{"spec_version":"aep/0.1","event_id":"evt_unclassified_correction_1","session_id":"ses_unclassified_1","timestamp":"2026-07-16T10:08:00Z","sequence":0,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo"} +{"spec_version":"aep/0.1","event_id":"evt_unclassified_correction_2","session_id":"ses_unclassified_2","timestamp":"2026-07-16T10:09:00Z","sequence":0,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo"} +{"spec_version":"aep/0.1","event_id":"evt_unclassified_correction_3","session_id":"ses_unclassified_3","timestamp":"2026-07-16T10:10:00Z","sequence":0,"source":"generic-jsonl","type":"user.corrected_agent","project":"/workspace/demo"}