From 0d06fcaaed12c1bb9e0eb8ca55aec52a88f99c43 Mon Sep 17 00:00:00 2001 From: dhgoal <153369624+dhgoal@users.noreply.github.com> Date: Sat, 4 Jul 2026 06:08:21 +0900 Subject: [PATCH] =?UTF-8?q?feat(anomaly):=20vouch=20flag-anomalies=20?= =?UTF-8?q?=E2=80=94=20advisory=20flags=20on=20pending=20proposals=20(#323?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reviewers scan the pending queue linearly and every item looks the same weight; the ones that most deserve a hard look blend in. the near- duplicate direction already surfaces (find_similar_on_propose); this surfaces the outlier direction. scores every pending claim proposal worst-first with reason codes: - thin_evidence: evidence count at or below a floor (barely cited) - contradicts_many: declares contradictions against >= n approved live claims (retired claims don't count) - far_from_corpus: nearest-neighbour cosine to the approved claim corpus below a floor (an outlier). embedding-derived, so it degrades gracefully to no code when the embeddings extra is absent — the same swallow find_similar_on_propose does — leaving the two non-embedding codes still computed. thresholds resolve from review.anomaly.* (mirroring similarity_threshold). read-only by construction: a hint for the reviewer that emits no proposal, writes no artifact, and never rejects or quarantines — the human gate is untouched. scoring lives in a new anomaly.py, not storage.py. shipped cli-only (`vouch flag-anomalies`); the kb.flag_anomalies method and a list-pending --flag-anomalies flag are a separate concern. --- CHANGELOG.md | 10 ++ src/vouch/anomaly.py | 225 ++++++++++++++++++++++++++++++++++++++++++ src/vouch/cli.py | 57 +++++++++++ tests/test_anomaly.py | 188 +++++++++++++++++++++++++++++++++++ 4 files changed, 480 insertions(+) create mode 100644 src/vouch/anomaly.py create mode 100644 tests/test_anomaly.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f22048..37448fa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,16 @@ All notable changes to vouch are documented here. Format follows successor (pages still require an explicit `new_id` — they have no successor pointer). Read-only throughout; still no writes, proposals, or audit events (#327). +- `vouch flag-anomalies` — advisory anomaly flags on pending proposals (#323). + scores every pending claim proposal worst-first with reason codes: + `thin_evidence` (barely cited), `contradicts_many` (declares contradictions + against approved live claims), and `far_from_corpus` (an outlier vs the + approved corpus by nearest-neighbour cosine). the embedding-derived + `far_from_corpus` degrades gracefully to no code when the embeddings extra is + absent — the two non-embedding codes still compute. thresholds resolve from + `review.anomaly.*` in `.vouch/config.yaml`. read-only by construction — a hint + for the reviewer that flags nothing durable and never rejects or quarantines, + so the review gate is untouched. `--json` for the machine shape. - auto-capture: claude code sessions are harvested via hooks and filed as a single pending session-summary proposal for human approval. a `PostToolUse` hook (`vouch capture observe`) appends compact tool-use observations to an diff --git a/src/vouch/anomaly.py b/src/vouch/anomaly.py new file mode 100644 index 00000000..ddad0562 --- /dev/null +++ b/src/vouch/anomaly.py @@ -0,0 +1,225 @@ +"""Advisory anomaly flags on pending proposals (vouchdev/vouch#323). + +Reviewers scan the pending queue linearly, and at a glance every item carries +the same apparent weight. The ones that most deserve a hard look — a claim far +from anything already in the kb, a claim that contradicts a pile of approved +ones, a claim scraping by with the barest evidence — blend in. The +near-*duplicate* direction already surfaces (``find_similar_on_propose`` +attaches non-blocking warnings on propose); this surfaces the *outlier* +direction. + +This computes a small set of heuristics per pending claim proposal and returns +them as read-side annotations. It is a **hint** for the human reviewer: it never +rejects, approves, blocks, or rewrites anything, emits no proposal, and touches +no artifact — so it goes nowhere near ``proposals.approve()``. Scoring logic +lives here, not in ``storage.py``, which stays pure I/O. + +Reason codes: + +* ``thin_evidence`` — evidence list at or below a floor (``propose_claim`` + already requires ≥1; this is the softer "technically cited but suspiciously + thin" case). Non-embedding. +* ``contradicts_many`` — the proposal declares it contradicts a threshold number + of *approved, live* claims (over ``kb.contradict``'s existing notion). + Non-embedding. +* ``far_from_corpus`` — nearest-neighbour cosine to the approved claim corpus is + below a floor (no neighbour is close → an outlier). Embedding-derived, so it + degrades gracefully to *no code* when the embeddings extra is absent — the + same swallow ``find_similar_on_propose`` does — leaving the two non-embedding + codes still computed. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +import yaml + +from .models import ClaimStatus, ProposalStatus +from .storage import KBStore + +_log = logging.getLogger("vouch.anomaly") + +# Defaults. All overridable per-KB under ``review.anomaly.*`` in config.yaml, +# resolved like ``embeddings.similarity.similarity_threshold``. +DEFAULT_MIN_EVIDENCE = 1 # evidence count <= this => thin +DEFAULT_CONTRADICTION_COUNT = 1 # >= this many approved contradictions => flag +DEFAULT_FAR_FROM_CORPUS_FLOOR = 0.35 # best cosine < this => outlier + +# Claims in these statuses are not live corpus (mirrors metrics / health), so a +# declared contradiction against one of them does not count. +_RETIRED = frozenset( + {ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED} +) + +_MAX_IDS = 10 + + +@dataclass(frozen=True) +class Anomaly: + """One flagged pending proposal and its reason codes (worst-first list).""" + + proposal_id: str + kind: str + proposed_by: str + reasons: list[dict[str, Any]] + + def to_dict(self) -> dict[str, Any]: + return { + "proposal_id": self.proposal_id, + "kind": self.kind, + "proposed_by": self.proposed_by, + "reasons": self.reasons, + } + + +def _resolve_cfg( + store: KBStore, + min_evidence: int | None, + contradiction_count: int | None, + far_floor: float | None, +) -> tuple[int, int, float]: + """Fill unset thresholds from ``review.anomaly.*``, else the module defaults.""" + review: dict[str, Any] = {} + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + if isinstance(loaded, dict) and isinstance(loaded.get("review"), dict): + anomaly = loaded["review"].get("anomaly") + if isinstance(anomaly, dict): + review = anomaly + except Exception: + pass + + def pick(val, key, default): # type: ignore[no-untyped-def] + if val is not None: + return val + return review[key] if review.get(key) is not None else default + + return ( + int(pick(min_evidence, "min_evidence", DEFAULT_MIN_EVIDENCE)), + int(pick(contradiction_count, "contradiction_count", DEFAULT_CONTRADICTION_COUNT)), + float(pick(far_floor, "far_from_corpus_floor", DEFAULT_FAR_FROM_CORPUS_FLOOR)), + ) + + +def _try_embedder() -> Any: + """Return an embedder, or ``None`` when the embeddings extra is absent.""" + try: + from .embeddings import get_embedder + + return get_embedder() + except Exception as e: # ImportError, missing model, etc. + _log.debug("anomaly: embeddings unavailable, far_from_corpus skipped: %s", e) + return None + + +def _far_from_corpus( + store: KBStore, embedder: Any, text: str, floor: float +) -> dict[str, Any] | None: + """Nearest-neighbour cosine to the approved claim corpus, or ``None``. + + ``None`` when embeddings error out or the corpus is empty/unindexed (nothing + to be an outlier *from*). A code only when a neighbour exists and the best + cosine is below ``floor``. + """ + try: + from .index_db import search_embedding + + vec = embedder.encode(text.strip()) + hits = search_embedding( + store.kb_dir, query_vec=vec, kinds=("claim",), limit=1, min_score=0.0 + ) + except Exception as e: + _log.debug("anomaly far_from_corpus skipped: %s", e) + return None + if not hits: + return None + best = round(float(hits[0][3]), 4) + if best < floor: + return {"code": "far_from_corpus", "best_cosine": best, "floor": floor} + return None + + +def flag_anomalies( + store: KBStore, + *, + min_evidence: int | None = None, + contradiction_count: int | None = None, + far_floor: float | None = None, +) -> list[Anomaly]: + """Score every pending claim proposal, returning the flagged ones worst-first. + + Read-only: it reads proposals + approved claims (+ the embedding index when + present) and computes; it writes nothing. + """ + ev_floor, contra_floor, far = _resolve_cfg( + store, min_evidence, contradiction_count, far_floor + ) + approved_live = { + c.id for c in store.list_claims() if c.status not in _RETIRED + } + embedder = _try_embedder() + + out: list[Anomaly] = [] + for pr in store.list_proposals(ProposalStatus.PENDING): + if pr.kind.value != "claim": + continue # the heuristics are claim-shaped + payload = pr.payload + reasons: list[dict[str, Any]] = [] + + evidence = payload.get("evidence") or [] + if len(evidence) <= ev_floor: + reasons.append( + {"code": "thin_evidence", "evidence_count": len(evidence), "floor": ev_floor} + ) + + contra = [ + cid for cid in (payload.get("contradicts") or []) if cid in approved_live + ] + if len(contra) >= contra_floor: + reasons.append( + {"code": "contradicts_many", "count": len(contra), "ids": contra[:_MAX_IDS]} + ) + + text = payload.get("text") + if embedder is not None and text and str(text).strip(): + code = _far_from_corpus(store, embedder, str(text), far) + if code is not None: + reasons.append(code) + + if reasons: + out.append(Anomaly(pr.id, pr.kind.value, pr.proposed_by, reasons)) + + # worst-first: most reason codes first, id tie-break for deterministic output. + out.sort(key=lambda a: (-len(a.reasons), a.proposal_id)) + return out + + +# --- rendering ------------------------------------------------------------ + + +def _reason_str(r: dict[str, Any]) -> str: + code = r["code"] + if code == "thin_evidence": + return f"thin_evidence (cites {r['evidence_count']}, floor {r['floor']})" + if code == "contradicts_many": + return f"contradicts_many ({r['count']} approved)" + if code == "far_from_corpus": + return f"far_from_corpus (best cosine {r['best_cosine']} < {r['floor']})" + return code + + +def render_text(anomalies: list[Anomaly]) -> str: + out: list[str] = [] + out.append(f"vouch flag-anomalies ({len(anomalies)} flagged)") + out.append("") + if not anomalies: + out.append(" (no pending proposals look anomalous)") + return "\n".join(out) + "\n" + for a in anomalies: + out.append(f" {a.proposal_id} [{a.kind}] by {a.proposed_by}") + for r in a.reasons: + out.append(f" • {_reason_str(r)}") + return "\n".join(out) + "\n" diff --git a/src/vouch/cli.py b/src/vouch/cli.py index ce9e6a05..d96b3b59 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -23,6 +23,7 @@ import yaml from . import __version__, bundle, health, volunteer_context +from . import anomaly as anomaly_mod from . import audit as audit_mod from . import capture as capture_mod from . import codex_rollout as codex_rollout_mod @@ -837,6 +838,62 @@ def pending(as_json: bool) -> None: click.echo(f" {str(preview).strip()[:120]}") +@cli.command(name="flag-anomalies") +@click.option( + "--min-evidence", + default=None, + type=int, + help="Flag a pending claim whose evidence count is at or below this " + "(default from review.anomaly.min_evidence, else 1).", +) +@click.option( + "--contradictions", + "contradiction_count", + default=None, + type=int, + help="Flag a claim declaring at least this many approved contradictions " + "(default from review.anomaly.contradiction_count, else 1).", +) +@click.option( + "--far-floor", + default=None, + type=float, + help="Flag a claim whose best cosine to the approved corpus is below this " + "(default from review.anomaly.far_from_corpus_floor, else 0.35; needs the " + "embeddings extra — skipped otherwise).", +) +@click.option("--json", "as_json", is_flag=True, help="Emit the machine shape instead of text.") +def flag_anomalies( + min_evidence: int | None, + contradiction_count: int | None, + far_floor: float | None, + as_json: bool, +) -> None: + """Advisory flags on pending proposals that deserve a closer look (#323). + + \b + Scores every pending claim proposal, worst-first, with reason codes: + • thin_evidence — barely cited + • contradicts_many — declares contradictions against approved claims + • far_from_corpus — an outlier vs the approved corpus (needs embeddings) + + Read-only — a hint for the reviewer. It flags nothing durable: no proposal, + no approval, no write. Never rejects or quarantines; the human gate is + untouched. + """ + store = _load_store() + anomalies = anomaly_mod.flag_anomalies( + store, + min_evidence=min_evidence, + contradiction_count=contradiction_count, + far_floor=far_floor, + ) + if as_json: + _emit_json([a.to_dict() for a in anomalies]) + return + click.echo(anomaly_mod.render_text(anomalies), nl=False) + + def _proposal_preview(pr: Proposal) -> str: preview = ( pr.payload.get("text") diff --git a/tests/test_anomaly.py b/tests/test_anomaly.py new file mode 100644 index 00000000..9860f209 --- /dev/null +++ b/tests/test_anomaly.py @@ -0,0 +1,188 @@ +"""`vouch flag-anomalies` — advisory anomaly flags on pending proposals (#323). + +Covers the non-embedding codes (thin_evidence, contradicts_many), graceful +degradation when no embedder is present (far_from_corpus produces no code), the +read-only / no-mutation invariant, worst-first ordering, config resolution, and +the cli surface. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch.anomaly import ( + DEFAULT_FAR_FROM_CORPUS_FLOOR, + Anomaly, + flag_anomalies, + render_text, +) +from vouch.audit import count_events +from vouch.cli import cli +from vouch.models import Claim, ClaimStatus, Proposal, ProposalKind +from vouch.storage import KBStore + + +@pytest.fixture +def store(tmp_path: Path) -> KBStore: + return KBStore.init(tmp_path) + + +def _pending_claim( + store: KBStore, pid: str, *, evidence, contradicts=None, text="a claim", by="agent" +): + payload = { + "id": pid, "text": text, "type": "observation", + "confidence": 0.7, "evidence": list(evidence), "entities": [], "tags": [], + } + if contradicts is not None: + payload["contradicts"] = list(contradicts) + store.put_proposal(Proposal(id=pid, kind=ProposalKind.CLAIM, proposed_by=by, payload=payload)) + + +def _seed(store: KBStore) -> None: + src = store.put_source(b"evidence") + # an approved live claim (a valid contradiction target) + a retired one + store.put_claim(Claim(id="approved-1", text="live claim", evidence=[src.id])) + store.put_claim(Claim(id="retired-1", text="old claim", evidence=[src.id], + status=ClaimStatus.ARCHIVED)) + # p_thin: only one citation -> thin_evidence + _pending_claim(store, "p_thin", evidence=[src.id]) + # p_contra: well-cited but declares a contradiction against an approved claim + _pending_claim(store, "p_contra", evidence=[src.id, src.id], contradicts=["approved-1"]) + # p_both: thin AND contradicts -> two reasons (worst) + _pending_claim(store, "p_both", evidence=[src.id], contradicts=["approved-1"]) + # p_clean: well cited, no contradictions -> not flagged + _pending_claim(store, "p_clean", evidence=[src.id, src.id]) + + +# --- non-embedding codes -------------------------------------------------- + + +def test_thin_evidence_flagged(store: KBStore) -> None: + _seed(store) + flagged = {a.proposal_id: a for a in flag_anomalies(store)} + assert "p_thin" in flagged + codes = {r["code"] for r in flagged["p_thin"].reasons} + assert codes == {"thin_evidence"} + assert flagged["p_thin"].reasons[0]["evidence_count"] == 1 + + +def test_clean_proposal_not_flagged(store: KBStore) -> None: + _seed(store) + ids = {a.proposal_id for a in flag_anomalies(store)} + assert "p_clean" not in ids + + +def test_contradicts_counts_only_approved_live(store: KBStore) -> None: + _seed(store) + # declaring a contradiction against a RETIRED claim must not count + _pending_claim(store, "p_retired_contra", evidence=[store.list_sources()[0].id, "x"], + contradicts=["retired-1"]) + flagged = {a.proposal_id: a for a in flag_anomalies(store)} + assert "p_retired_contra" not in flagged # 2 evidence, contradiction not live -> clean + assert "contradicts_many" in {r["code"] for r in flagged["p_contra"].reasons} + + +def test_worst_first_ordering(store: KBStore) -> None: + _seed(store) + order = [a.proposal_id for a in flag_anomalies(store)] + # p_both has two reasons -> sorts ahead of the single-reason ones + assert order[0] == "p_both" + assert set(order) == {"p_thin", "p_contra", "p_both"} + + +def test_thresholds_configurable(store: KBStore) -> None: + _seed(store) + # raise the evidence floor so 2-citation claims also flag as thin + hi = flag_anomalies(store, min_evidence=2) + assert "p_contra" in {a.proposal_id for a in hi} # now thin too (2 <= 2) + # a huge contradiction threshold suppresses contradicts_many + lo = {a.proposal_id: a for a in flag_anomalies(store, contradiction_count=99)} + assert "p_contra" not in lo # its only reason was the contradiction + + +# --- embeddings graceful degradation -------------------------------------- + + +def test_far_from_corpus_absent_without_embedder(store: KBStore) -> None: + _seed(store) + # the dev install has no embedder -> far_from_corpus never appears, but the + # non-embedding codes still compute. + for a in flag_anomalies(store): + assert all(r["code"] != "far_from_corpus" for r in a.reasons) + assert flag_anomalies(store) # still produced non-embedding flags + + +def test_far_from_corpus_uses_floor_when_embedder_present(store: KBStore, monkeypatch) -> None: + _seed(store) + + class FakeEmbedder: + def encode(self, text): + return [1.0, 0.0, 0.0] + + monkeypatch.setattr("vouch.anomaly._try_embedder", lambda: FakeEmbedder()) + # force the corpus search to return a far neighbour (low cosine) + monkeypatch.setattr( + "vouch.index_db.search_embedding", + lambda *a, **k: [("claim", "approved-1", "live claim", 0.10)], + ) + flagged = { + a.proposal_id: a + for a in flag_anomalies(store, far_floor=DEFAULT_FAR_FROM_CORPUS_FLOOR) + } + far = [r for r in flagged["p_clean"].reasons if r["code"] == "far_from_corpus"] + assert far and far[0]["best_cosine"] == pytest.approx(0.10) + # a close neighbour (high cosine) must NOT flag + monkeypatch.setattr( + "vouch.index_db.search_embedding", + lambda *a, **k: [("claim", "approved-1", "live claim", 0.95)], + ) + flagged2 = {a.proposal_id: a for a in flag_anomalies(store)} + assert "p_clean" not in flagged2 + + +# --- read-only invariant -------------------------------------------------- + + +def test_flag_writes_nothing(store: KBStore) -> None: + _seed(store) + before = count_events(store.kb_dir) + flag_anomalies(store) + flag_anomalies(store, min_evidence=3) + assert count_events(store.kb_dir) == before + + +def test_empty_kb(store: KBStore) -> None: + assert flag_anomalies(store) == [] + assert "no pending proposals look anomalous" in render_text([]) + + +# --- cli ------------------------------------------------------------------ + + +def test_cli_text(store: KBStore, monkeypatch) -> None: + _seed(store) + monkeypatch.chdir(store.root) + res = CliRunner().invoke(cli, ["flag-anomalies"]) + assert res.exit_code == 0, res.output + assert "flag-anomalies" in res.output + assert "p_both" in res.output + assert "thin_evidence" in res.output + + +def test_cli_json(store: KBStore, monkeypatch) -> None: + _seed(store) + monkeypatch.chdir(store.root) + res = CliRunner().invoke(cli, ["flag-anomalies", "--json"]) + assert res.exit_code == 0, res.output + doc = json.loads(res.output) + assert doc[0]["proposal_id"] == "p_both" # worst-first + assert {r["code"] for r in doc[0]["reasons"]} == {"thin_evidence", "contradicts_many"} + + +def test_cli_defaults_and_dataclass() -> None: + assert isinstance(Anomaly("p", "claim", "a", []).to_dict(), dict)