diff --git a/README.md b/README.md index e1342ab..3ee75b0 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,17 @@ Details: [discord/smol-doc-analyzer/README.md](discord/smol-doc-analyzer/README. ## Evaluation -Reports land in `evaluation/reports/` (`classification_report.*`, `vit_classification_report.*`, `extraction_report.json`, `failure_modes.md`). +Reports land in `evaluation/reports/` (`classification_report.*`, `vit_classification_report.*`, `extraction_report.json`, `outcome_prediction_report.*`, `failure_modes.md`). + +Claim **outcome prediction** accuracy is a complementary metric to classification / extraction: + +```bash +python -m src.pipeline.eval_outcome \ + --in data/synthetic/documents/documents_from_skeletons_n240_seed42.jsonl \ + --no-wandb --limit 50 +``` + +Gold `expected_outcome` is written on synthetic skeletons and scored against the `predict_outcome` pipeline stage. ## Experiment tracking (Weights & Biases) diff --git a/data/profiles/insurance_distributions.json b/data/profiles/insurance_distributions.json index 331d617..236d437 100644 --- a/data/profiles/insurance_distributions.json +++ b/data/profiles/insurance_distributions.json @@ -57,6 +57,7 @@ "ambiguous": 0.22, "fraud_flagged": 0.08 }, + "expected_outcome_notes": "Gold expected_outcome is derived deterministically from narrative_complexity + financials + injuries (see src/pipeline/outcome.py). Tracked in evaluation/reports/outcome_prediction_report.*", "police_report_rate": 0.42, "injuries_reported_rate": 0.18, "acord_form_by_document_type": { diff --git a/data/schemas/claim_skeleton.schema.json b/data/schemas/claim_skeleton.schema.json index b80bc29..c191a5d 100644 --- a/data/schemas/claim_skeleton.schema.json +++ b/data/schemas/claim_skeleton.schema.json @@ -3,7 +3,7 @@ "title": "ClaimSkeleton", "description": "Structured intermediate representation used to generate synthetic insurance documents and memos. This is entirely fictional data -- see docs/data_provenance.md.", "type": "object", - "required": ["claim_id", "document_type", "policy", "loss_event", "parties", "financials", "narrative_complexity"], + "required": ["claim_id", "document_type", "policy", "loss_event", "parties", "financials", "narrative_complexity", "expected_outcome"], "properties": { "claim_id": { "type": "string", @@ -82,6 +82,17 @@ "enum": ["clean", "standard", "ambiguous", "fraud_flagged"], "description": "Controls how straightforward vs. messy/edge-case the generated document and memo should be." }, + "expected_outcome": { + "type": "string", + "enum": [ + "pay_full", + "pay_partial", + "deny", + "investigate", + "close_without_payment" + ], + "description": "Synthetic supervisory label for predicted claim disposition. Deterministic from skeleton features so pipeline outcome accuracy tracks end-to-end feature recovery." + }, "multi_doc_group_id": { "type": ["string", "null"], "description": "If set, links this skeleton to other skeletons that belong to the same claim file (e.g. loss notice + repair estimate + adjuster memo all sharing a claim_id)." diff --git a/docs/architecture.md b/docs/architecture.md index 80b6a2f..d90eacc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,8 @@ execute chronologically; each stage reacts to prior stage outputs: 2. **Classification** — DeBERTa-v3 encoder (heuristic fallback) maps document text → taxonomy label; optionally a **ViT** image classifier maps rendered page images → the same taxonomy 3. **Extraction** — LayoutLMv3 / token classifier (heuristic fallback) pulls structured fields; conditioned on the predicted document type 4. **Vision LLM refine** — markdown-first local multimodal/text model (default target: Qwen2-VL class) corrects fields using classify+extract context; optional page image via `VISION_LLM_USE_IMAGE=1` -5. **Summarization** — generative LLM or template memo grounded in upstream markdown + payloads (not ground-truth skeletons) +5. **Predict outcome** — deterministic claim-disposition prediction (`pay_full` / `pay_partial` / `deny` / `investigate` / `close_without_payment`) from extracted features; gold `expected_outcome` on synthetic skeletons enables accuracy tracking alongside classification / extraction metrics +6. **Summarization** — generative LLM or template memo grounded in upstream markdown + payloads + predicted outcome (not ground-truth skeletons) Entry points: @@ -70,7 +71,7 @@ which runs the same chronological pipeline as the CLI. See Public corpora → profiles → skeletons → documents (+ noisy) → classifier (text and/or ViT on renders) / extractor └→ memos (Phase 4 training targets) -Inbound PNG/PDF/text → to_markdown → classify → extract → vision_llm → summarize +Inbound PNG/PDF/text → to_markdown → classify → extract → vision_llm → predict_outcome → summarize ↓ structured markdown (LLM context) ``` @@ -86,6 +87,7 @@ reorders by name. Each stage receives an accumulating `AnalysisContext`: | classify | markdown plain_text (preferred) | `classification.document_type`, confidence | | extract | markdown + classification | `extraction.fields*`, optional page render | | vision_llm | markdown (+ optional image) + classify + extract | `vision.refined_fields` (merged into extraction) | +| predict_outcome | extraction (+ vision refine) + text cues | `outcome.expected_outcome`, confidence, optional gold compare | | summarize | markdown + all prior payloads | `summary.memo` | Low-confidence stages append flags (`low_confidence_classification`, etc.) diff --git a/evaluation/reports/outcome_prediction_report.json b/evaluation/reports/outcome_prediction_report.json new file mode 100644 index 0000000..dc46d4a --- /dev/null +++ b/evaluation/reports/outcome_prediction_report.json @@ -0,0 +1,117 @@ +{ + "n": 40, + "n_skipped_no_gold": 0, + "accuracy": 0.4, + "macro_f1": 0.3111111111111111, + "per_class": { + "pay_full": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 19.0 + }, + "pay_partial": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 2.0 + }, + "deny": { + "precision": 1.0, + "recall": 1.0, + "f1-score": 1.0, + "support": 1.0 + }, + "investigate": { + "precision": 0.38461538461538464, + "recall": 1.0, + "f1-score": 0.5555555555555556, + "support": 15.0 + }, + "close_without_payment": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 3.0 + }, + "accuracy": 0.4, + "macro avg": { + "precision": 0.27692307692307694, + "recall": 0.4, + "f1-score": 0.3111111111111111, + "support": 40.0 + }, + "weighted avg": { + "precision": 0.16923076923076924, + "recall": 0.4, + "f1-score": 0.23333333333333334, + "support": 40.0 + } + }, + "confusion_matrix": [ + [ + 0, + 0, + 0, + 19, + 0 + ], + [ + 0, + 0, + 0, + 2, + 0 + ], + [ + 0, + 0, + 1, + 0, + 0 + ], + [ + 0, + 0, + 0, + 15, + 0 + ], + [ + 0, + 0, + 0, + 3, + 0 + ] + ], + "label_order": [ + "pay_full", + "pay_partial", + "deny", + "investigate", + "close_without_payment" + ], + "gold_distribution": { + "pay_full": 19, + "investigate": 15, + "pay_partial": 2, + "close_without_payment": 3, + "deny": 1 + }, + "pred_distribution": { + "investigate": 39, + "deny": 1 + }, + "n_correct": 16, + "chain": [ + "to_markdown", + "classify", + "extract", + "vision_llm", + "predict_outcome", + "summarize" + ], + "metric_family": "claim_outcome_prediction", + "n_detail_rows": 40 +} diff --git a/evaluation/reports/outcome_prediction_report.md b/evaluation/reports/outcome_prediction_report.md new file mode 100644 index 0000000..0110067 --- /dev/null +++ b/evaluation/reports/outcome_prediction_report.md @@ -0,0 +1,27 @@ +# Claim outcome prediction report + +- N scored: **40** +- Skipped (no gold): 0 +- Accuracy: **0.4** +- Macro F1: **0.3111111111111111** +- Chain: `to_markdown → classify → extract → vision_llm → predict_outcome → summarize` + +## Label distribution (gold) + +- `close_without_payment`: 3 +- `deny`: 1 +- `investigate`: 15 +- `pay_full`: 19 +- `pay_partial`: 2 + +## Per-class F1 + +- `pay_full`: precision=0.000 recall=0.000 f1=0.000 support=19.0 +- `pay_partial`: precision=0.000 recall=0.000 f1=0.000 support=2.0 +- `deny`: precision=1.000 recall=1.000 f1=1.000 support=1.0 +- `investigate`: precision=0.385 recall=1.000 f1=0.556 support=15.0 +- `close_without_payment`: precision=0.000 recall=0.000 f1=0.000 support=3.0 + +## Notes + +Gold `expected_outcome` is a deterministic function of skeleton features (complexity, injuries, damage vs deductible/reserve). Accuracy therefore tracks how well upstream extraction recovers those features for the predictive disposition rule — complementary to classification accuracy and extraction field F1. diff --git a/src/discord_bot/formatters.py b/src/discord_bot/formatters.py index b08e560..b92cb15 100644 --- a/src/discord_bot/formatters.py +++ b/src/discord_bot/formatters.py @@ -10,6 +10,7 @@ def compact_analysis(result: dict[str, Any], *, max_memo_chars: int = 1200) -> d classification = result.get("classification") or {} extraction = result.get("extraction") or {} vision = result.get("vision") or {} + outcome = result.get("outcome") or {} summary = result.get("summary") or {} markdown = result.get("markdown") or {} @@ -44,6 +45,13 @@ def compact_analysis(result: dict[str, Any], *, max_memo_chars: int = 1200) -> d or classification.get("label") or classification.get("predicted_label"), "classification_confidence": classification.get("confidence"), + "expected_outcome": result.get("expected_outcome") + or outcome.get("expected_outcome") + or outcome.get("outcome_label"), + "outcome_confidence": outcome.get("confidence"), + "outcome_description": outcome.get("description"), + "gold_outcome": outcome.get("gold_outcome"), + "outcome_correct": outcome.get("correct"), "fields": fields, "memo": memo, "flags": result.get("flags") or [], @@ -66,6 +74,17 @@ def format_discord_summary(compact: dict[str, Any]) -> str: rid = compact.get("claim_id") or compact.get("record_id") lines.append(f"**Record:** `{rid}`") + outcome = compact.get("expected_outcome") + if outcome: + oconf = compact.get("outcome_confidence") + oconf_s = f" ({float(oconf):.0%})" if isinstance(oconf, (int, float)) else "" + lines.append(f"**Predicted outcome:** `{outcome}`{oconf_s}") + if compact.get("outcome_description"): + lines.append(f"-# {compact['outcome_description']}") + if compact.get("gold_outcome") is not None: + mark = "✓" if compact.get("outcome_correct") else "✗" + lines.append(f"**Gold outcome:** `{compact['gold_outcome']}` {mark}") + fields = compact.get("fields") or {} if fields: lines.append("") diff --git a/src/discord_bot/runner.py b/src/discord_bot/runner.py index 4d73168..1e435f3 100644 --- a/src/discord_bot/runner.py +++ b/src/discord_bot/runner.py @@ -61,9 +61,8 @@ def _overlay_secrets(raw: dict) -> dict: ("google-gla:", "google-vertex:", "openai:", "anthropic:", "xai:", "groq:") ) and "/" not in model.split(":", 1)[-1] if chloride_native or not model: - data["AI_MODEL_NAME"] = os.getenv( - "DISCORD_AI_MODEL", "anthropic/claude-sonnet-4.5" - ) + override = os.getenv("DISCORD_AI_MODEL", "").strip() + data["AI_MODEL_NAME"] = override or "anthropic/claude-sonnet-4.5" return data diff --git a/src/generation/skeleton_sampler.py b/src/generation/skeleton_sampler.py index ee0d83c..22a9de1 100644 --- a/src/generation/skeleton_sampler.py +++ b/src/generation/skeleton_sampler.py @@ -17,6 +17,7 @@ from src.utils.config import Config from src.utils.io import read_json, write_json, write_jsonl from src.utils.provenance import ProvenanceRecord, log_provenance +from src.pipeline.outcome import derive_expected_outcome, features_from_skeleton logger = logging.getLogger(__name__) @@ -126,6 +127,7 @@ def sample_skeleton( "multi_doc_group_id": multi_doc_group_id, "target_outputs": {"document_text": None, "memo_text": None}, } + skeleton["expected_outcome"] = derive_expected_outcome(features_from_skeleton(skeleton)) return skeleton diff --git a/src/pipeline/batch_runner.py b/src/pipeline/batch_runner.py index 8edf995..f37d927 100644 --- a/src/pipeline/batch_runner.py +++ b/src/pipeline/batch_runner.py @@ -79,6 +79,7 @@ def run_batch( "review_queue_path": str(review_path), "by_document_type": _count_by_type(results), "flag_counts": _count_flags(results), + "outcome_metrics": _outcome_metrics(results, rows), } write_json(summary_path, summary) log_provenance( @@ -123,10 +124,45 @@ def _count_flags(results: list[dict[str, Any]]) -> dict[str, int]: return counts +def _outcome_metrics( + results: list[dict[str, Any]], rows: list[dict[str, Any]] +) -> dict[str, Any]: + """Accuracy of predicted claim outcomes vs gold when labels are available.""" + from src.pipeline.outcome import derive_expected_outcome, features_from_skeleton + + y_true: list[str] = [] + y_pred: list[str] = [] + for row, result in zip(rows, results): + gold = row.get("expected_outcome") + skeleton = row.get("skeleton") + if not gold and isinstance(skeleton, dict): + gold = skeleton.get("expected_outcome") or derive_expected_outcome( + features_from_skeleton(skeleton) + ) + pred = (result.get("outcome") or {}).get("expected_outcome") + if not gold or not pred: + continue + y_true.append(str(gold)) + y_pred.append(str(pred)) + + if not y_true: + return { + "n_scored": 0, + "accuracy": None, + "note": "no gold expected_outcome on inputs", + } + correct = sum(1 for a, b in zip(y_true, y_pred) if a == b) + return { + "n_scored": len(y_true), + "n_correct": correct, + "accuracy": correct / len(y_true), + } + + def main() -> None: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") parser = argparse.ArgumentParser( - description="Batch-run the chained to_markdown→classify→extract→vision→summarize pipeline" + description="Batch-run the chained to_markdown→classify→extract→vision→predict_outcome→summarize pipeline" ) parser.add_argument("--in", dest="inp", type=Path, required=True) parser.add_argument("--out-dir", type=Path, default=None) diff --git a/src/pipeline/eval_outcome.py b/src/pipeline/eval_outcome.py new file mode 100644 index 0000000..5f61f8b --- /dev/null +++ b/src/pipeline/eval_outcome.py @@ -0,0 +1,285 @@ +"""Evaluate predicted claim outcomes against gold expected_outcome labels. + +Adds outcome accuracy / macro-F1 / confusion to the evaluation battery alongside +classification and extraction reports. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from collections import Counter +from pathlib import Path +from typing import Any + +from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score + +from src.pipeline.orchestrator import DocumentAnalysisOrchestrator +from src.pipeline.outcome import OUTCOME_LABELS, derive_expected_outcome, features_from_skeleton +from src.pipeline.types import AnalysisDocument +from src.utils.config import Config +from src.utils.io import load_jsonl, write_json +from src.utils.wandb_utils import load_wandb_settings, start_run + +logger = logging.getLogger(__name__) + + +def _gold_outcome(row: dict[str, Any]) -> str | None: + if isinstance(row.get("expected_outcome"), str) and row["expected_outcome"]: + return row["expected_outcome"] + skeleton = row.get("skeleton") + if isinstance(skeleton, dict): + if isinstance(skeleton.get("expected_outcome"), str) and skeleton["expected_outcome"]: + return skeleton["expected_outcome"] + return derive_expected_outcome(features_from_skeleton(skeleton)) + return None + + +def evaluate_outcomes( + rows: list[dict[str, Any]], + *, + cfg: Config | None = None, + enable_vision: bool | None = False, + classifier_dir: Path | None = None, + extractor_dir: Path | None = None, +) -> dict[str, Any]: + """Run the analysis chain and score predicted vs gold expected_outcome.""" + cfg = cfg or Config.load() + orch = DocumentAnalysisOrchestrator( + cfg=cfg, + enable_vision=enable_vision, + classifier_dir=classifier_dir, + extractor_dir=extractor_dir, + ) + + y_true: list[str] = [] + y_pred: list[str] = [] + details: list[dict[str, Any]] = [] + skipped = 0 + + for row in rows: + gold = _gold_outcome(row) + if not gold: + skipped += 1 + continue + + # Ensure gold is visible to PredictOutcomeStage via metadata. + enriched = dict(row) + meta = dict(enriched.get("metadata") or {}) + if "skeleton" in enriched and "skeleton" not in meta: + meta["skeleton"] = enriched["skeleton"] + meta.setdefault("expected_outcome", gold) + if isinstance(enriched.get("skeleton"), dict): + sk = dict(enriched["skeleton"]) + sk.setdefault("expected_outcome", gold) + meta["skeleton"] = sk + enriched["metadata"] = meta + enriched["expected_outcome"] = gold + + ctx = orch.analyze(AnalysisDocument.from_row(enriched)) + payload = ctx.to_dict() + pred = (payload.get("outcome") or {}).get("expected_outcome") or "investigate" + y_true.append(gold) + y_pred.append(pred) + details.append( + { + "record_id": payload.get("record_id"), + "claim_id": payload.get("claim_id"), + "gold": gold, + "pred": pred, + "correct": gold == pred, + "confidence": (payload.get("outcome") or {}).get("confidence"), + "features": (payload.get("outcome") or {}).get("features"), + } + ) + + labels = list(OUTCOME_LABELS) + if not y_true: + report = { + "n": 0, + "n_skipped_no_gold": skipped, + "accuracy": None, + "macro_f1": None, + "per_class": {}, + "confusion_matrix": [], + "label_order": labels, + "details": [], + } + return report + + acc = float(accuracy_score(y_true, y_pred)) + macro_f1 = float(f1_score(y_true, y_pred, labels=labels, average="macro", zero_division=0)) + per_class = classification_report( + y_true, + y_pred, + labels=labels, + output_dict=True, + zero_division=0, + ) + cm = confusion_matrix(y_true, y_pred, labels=labels).tolist() + gold_dist = dict(Counter(y_true)) + pred_dist = dict(Counter(y_pred)) + + report = { + "n": len(y_true), + "n_skipped_no_gold": skipped, + "accuracy": acc, + "macro_f1": macro_f1, + "per_class": per_class, + "confusion_matrix": cm, + "label_order": labels, + "gold_distribution": gold_dist, + "pred_distribution": pred_dist, + "n_correct": sum(1 for d in details if d["correct"]), + "details": details, + "chain": orch.stage_names, + "metric_family": "claim_outcome_prediction", + } + return report + + +def write_outcome_report(report: dict[str, Any], out_dir: Path) -> tuple[Path, Path]: + out_dir.mkdir(parents=True, exist_ok=True) + json_path = out_dir / "outcome_prediction_report.json" + md_path = out_dir / "outcome_prediction_report.md" + # Keep JSON lean for dashboards (details can be large) + slim = {k: v for k, v in report.items() if k != "details"} + slim["n_detail_rows"] = len(report.get("details") or []) + write_json(json_path, slim) + + lines = [ + "# Claim outcome prediction report", + "", + f"- N scored: **{report.get('n')}**", + f"- Skipped (no gold): {report.get('n_skipped_no_gold')}", + f"- Accuracy: **{report.get('accuracy')}**", + f"- Macro F1: **{report.get('macro_f1')}**", + f"- Chain: `{' → '.join(report.get('chain') or [])}`", + "", + "## Label distribution (gold)", + "", + ] + for lab, n in sorted((report.get("gold_distribution") or {}).items()): + lines.append(f"- `{lab}`: {n}") + lines.extend(["", "## Per-class F1", ""]) + per = report.get("per_class") or {} + for lab in report.get("label_order") or OUTCOME_LABELS: + stats = per.get(lab) or {} + if not stats: + continue + lines.append( + f"- `{lab}`: precision={stats.get('precision', 0):.3f} " + f"recall={stats.get('recall', 0):.3f} f1={stats.get('f1-score', 0):.3f} " + f"support={stats.get('support', 0)}" + ) + lines.extend( + [ + "", + "## Notes", + "", + "Gold `expected_outcome` is a deterministic function of skeleton features " + "(complexity, injuries, damage vs deductible/reserve). Accuracy therefore " + "tracks how well upstream extraction recovers those features for the " + "predictive disposition rule — complementary to classification accuracy " + "and extraction field F1.", + "", + ] + ) + md_path.write_text("\n".join(lines), encoding="utf-8") + return json_path, md_path + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + parser = argparse.ArgumentParser( + description="Evaluate claim outcome prediction accuracy vs gold expected_outcome" + ) + parser.add_argument( + "--in", + dest="inp", + type=Path, + required=True, + help="JSONL of documents with skeleton and/or expected_outcome gold labels", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=None, + help="Report directory (default: evaluation/reports)", + ) + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--vision", action="store_true") + parser.add_argument("--no-vision", action="store_true", default=True) + parser.add_argument("--classifier-dir", type=Path, default=None) + parser.add_argument("--extractor-dir", type=Path, default=None) + parser.add_argument("--wandb", action="store_true", default=None) + parser.add_argument("--no-wandb", action="store_true") + args = parser.parse_args() + + cfg = Config.load() + rows = load_jsonl(args.inp) + if args.limit is not None: + rows = rows[: args.limit] + + enable_vision = True if args.vision else False + report = evaluate_outcomes( + rows, + cfg=cfg, + enable_vision=enable_vision, + classifier_dir=args.classifier_dir, + extractor_dir=args.extractor_dir, + ) + out_dir = args.out_dir or cfg.evaluation_reports_dir + json_path, md_path = write_outcome_report(report, out_dir) + + use_wandb = False if args.no_wandb else (True if args.wandb else None) + settings = load_wandb_settings(enabled=False if args.no_wandb else use_wandb) + with start_run( + name="outcome-prediction-eval", + job_type="eval", + config={"n": report.get("n"), "input": str(args.inp)}, + tags=["outcome", "eval"], + settings=settings, + ) as wb: + if report.get("accuracy") is not None: + wb.summary( + { + "accuracy": report["accuracy"], + "macro_f1": report["macro_f1"], + "n": report["n"], + } + ) + wb.log( + { + "eval/outcome_accuracy": report["accuracy"], + "eval/outcome_macro_f1": report["macro_f1"], + "eval/n": report["n"], + } + ) + wb.log_artifact_files( + name="outcome-prediction-report", + paths=[json_path, md_path], + artifact_type="evaluation", + metadata={ + "accuracy": report["accuracy"], + "macro_f1": report["macro_f1"], + }, + ) + + print( + json.dumps( + { + "accuracy": report.get("accuracy"), + "macro_f1": report.get("macro_f1"), + "n": report.get("n"), + }, + indent=2, + ) + ) + print(json_path) + print(md_path) + + +if __name__ == "__main__": + main() diff --git a/src/pipeline/orchestrator.py b/src/pipeline/orchestrator.py index 2c7cbb0..b83555a 100644 --- a/src/pipeline/orchestrator.py +++ b/src/pipeline/orchestrator.py @@ -2,7 +2,7 @@ Chronological document-analysis orchestrator. One analyze action chains every initiated stage in initiation order: - to_markdown → classify → extract → vision_llm → summarize + to_markdown → classify → extract → vision_llm → predict_outcome → summarize PNG/PDF (and plain text) are converted to structured markdown first so downstream LLM stages consume compact, layout-aware context instead of @@ -23,6 +23,7 @@ ExtractStage, MarkdownConvertStage, PipelineStage, + PredictOutcomeStage, SummarizeStage, VisionLLMStage, ) @@ -60,7 +61,8 @@ def __init__( ClassifyStage(cfg=self.cfg, model_dir=classifier_dir, order=1), ExtractStage(cfg=self.cfg, model_dir=extractor_dir, order=2), VisionLLMStage(cfg=self.cfg, order=3, enabled=vision_enabled), - SummarizeStage(cfg=self.cfg, order=4), + PredictOutcomeStage(cfg=self.cfg, order=4), + SummarizeStage(cfg=self.cfg, order=5), ] # Preserve initiation order; do not re-sort by name. self.stages.sort(key=lambda s: s.order) @@ -199,7 +201,7 @@ def main() -> None: parser = argparse.ArgumentParser( description=( "Run the full document-analysis chain in one action: " - "to_markdown → classify → extract → vision_llm → summarize" + "to_markdown → classify → extract → vision_llm → predict_outcome → summarize" ) ) parser.add_argument( diff --git a/src/pipeline/outcome.py b/src/pipeline/outcome.py new file mode 100644 index 0000000..3657350 --- /dev/null +++ b/src/pipeline/outcome.py @@ -0,0 +1,222 @@ +"""Claim outcome labels and predictive rules. + +`expected_outcome` is a synthetic supervisory label for tracking how well the +pipeline's extracted claim features support a downstream settlement prediction. +Gold labels are a deterministic function of skeleton features so evaluation +accuracy measures end-to-end feature recovery + decision rules (not random noise). +""" + +from __future__ import annotations + +from typing import Any + +OUTCOME_LABELS: tuple[str, ...] = ( + "pay_full", + "pay_partial", + "deny", + "investigate", + "close_without_payment", +) + +OUTCOME_DESCRIPTIONS: dict[str, str] = { + "pay_full": "Approve and pay the claim at (or near) presented exposure", + "pay_partial": "Settle for less than claimed / apply limits or comparative fault", + "deny": "Deny coverage (exclusion, condition breach, or fraud indicators)", + "investigate": "Hold decision pending further investigation / SIU / medical review", + "close_without_payment": "Close with no indemnity (below deductible / withdrawn / no coverage trigger)", +} + + +def _as_float(value: Any) -> float | None: + if value is None or value == "": + return None + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip().replace(",", "").replace("$", "") + try: + return float(text) + except ValueError: + return None + + +def _as_bool(value: Any) -> bool | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return value + lower = str(value).strip().lower() + if lower in {"1", "true", "yes", "y", "filed", "reported"}: + return True + if lower in {"0", "false", "no", "n", "none", "not filed", "not reported"}: + return False + return None + + +def features_from_skeleton(skeleton: dict[str, Any]) -> dict[str, Any]: + """Flatten skeleton fields used by the outcome decision rule.""" + loss = skeleton.get("loss_event") or {} + fin = skeleton.get("financials") or {} + return { + "narrative_complexity": skeleton.get("narrative_complexity"), + "loss_type": loss.get("loss_type"), + "injuries_reported": loss.get("injuries_reported"), + "police_report_filed": loss.get("police_report_filed"), + "estimated_damage": fin.get("estimated_damage"), + "deductible": fin.get("deductible"), + "reserve_set": fin.get("reserve_set"), + "document_type": skeleton.get("document_type"), + } + + +def features_from_extraction( + *, + fields: dict[str, Any] | None, + document_type: str | None = None, + text: str = "", + narrative_complexity: str | None = None, +) -> dict[str, Any]: + """Build predictor features from upstream pipeline payloads + text cues.""" + flat = dict(fields or {}) + lower = (text or "").lower() + + injuries = _as_bool(flat.get("injuries_reported")) + if injuries is None: + injuries = any( + k in lower + for k in ("injuries reported", "injury reported", "bodily injury", "injured") + ) and "no injuries" not in lower and "injuries: none" not in lower + + police = _as_bool(flat.get("police_report_filed")) + if police is None: + police = "police report" in lower and "no police" not in lower + + complexity = narrative_complexity + if not complexity: + if any(k in lower for k in ("fraud", "siu", "misrepresentation", "red flag")): + complexity = "fraud_flagged" + elif any(k in lower for k in ("ambiguous", "unclear liability", "conflicting")): + complexity = "ambiguous" + elif any(k in lower for k in ("clean loss", "straightforward", "no complications")): + complexity = "clean" + else: + complexity = "standard" + + return { + "narrative_complexity": complexity, + "loss_type": flat.get("loss_type"), + "injuries_reported": injuries, + "police_report_filed": police, + "estimated_damage": _as_float(flat.get("estimated_damage")), + "deductible": _as_float(flat.get("deductible")), + "reserve_set": _as_float(flat.get("reserve_set")), + "document_type": document_type or flat.get("document_type"), + } + + +def derive_expected_outcome(features: dict[str, Any]) -> str: + """ + Deterministic claim-outcome label from claim features. + + Used both when sampling synthetic skeletons (gold label) and as the core + decision rule inside the predict_outcome stage (prediction). + """ + complexity = str(features.get("narrative_complexity") or "standard") + injuries = bool(features.get("injuries_reported")) + damage = _as_float(features.get("estimated_damage")) + deductible = _as_float(features.get("deductible")) + reserve = _as_float(features.get("reserve_set")) + + if complexity == "fraud_flagged": + if damage is not None and damage >= 10_000: + return "deny" + return "investigate" + + if complexity == "ambiguous" or injuries: + return "investigate" + + if damage is not None and deductible is not None and damage <= deductible: + return "close_without_payment" + + if complexity == "clean" and (damage is None or damage < 10_000): + return "pay_full" + + if damage is not None and damage >= 50_000: + return "pay_partial" + + if ( + damage is not None + and reserve is not None + and reserve > 0 + and damage / reserve >= 1.35 + ): + return "investigate" + + if complexity == "standard" and damage is not None and damage >= 15_000: + return "pay_partial" + + if damage is None and deductible is None: + # Underwriting / certificate docs often lack loss financials. + doc = str(features.get("document_type") or "") + if doc.startswith("application") or doc in { + "certificate_evidence", + "policy_change_endorsement", + }: + return "investigate" + return "investigate" + + return "pay_full" + + +def predict_outcome( + *, + fields: dict[str, Any] | None = None, + document_type: str | None = None, + text: str = "", + narrative_complexity: str | None = None, + gold_skeleton: dict[str, Any] | None = None, +) -> dict[str, Any]: + """ + Predict expected claim outcome from pipeline features. + + Returns label, confidence, feature snapshot, and optional gold comparison + when a skeleton with `expected_outcome` (or derivable features) is supplied. + """ + features = features_from_extraction( + fields=fields, + document_type=document_type, + text=text, + narrative_complexity=narrative_complexity, + ) + label = derive_expected_outcome(features) + + # Confidence: higher when key financials are present and complexity is sharp. + present = sum( + 1 + for k in ("estimated_damage", "deductible", "loss_type", "narrative_complexity") + if features.get(k) not in (None, "") + ) + confidence = 0.35 + 0.12 * present + if features.get("narrative_complexity") in {"fraud_flagged", "ambiguous", "clean"}: + confidence += 0.1 + if features.get("estimated_damage") is None: + confidence = min(confidence, 0.55) + confidence = float(min(0.95, confidence)) + + payload: dict[str, Any] = { + "expected_outcome": label, + "outcome_label": label, + "confidence": confidence, + "features": features, + "label_set": list(OUTCOME_LABELS), + "description": OUTCOME_DESCRIPTIONS.get(label, ""), + "backend": "deterministic_rules", + } + + if gold_skeleton: + gold = gold_skeleton.get("expected_outcome") + if not gold: + gold = derive_expected_outcome(features_from_skeleton(gold_skeleton)) + payload["gold_outcome"] = gold + payload["correct"] = gold == label + + return payload diff --git a/src/pipeline/stages.py b/src/pipeline/stages.py index 2f297cb..c9abadf 100644 --- a/src/pipeline/stages.py +++ b/src/pipeline/stages.py @@ -12,6 +12,7 @@ from src.extraction.render_forms import FIELD_PATTERNS, label_words, render_page from src.pipeline.markdown_convert import approx_token_count, convert_to_markdown +from src.pipeline.outcome import OUTCOME_LABELS, predict_outcome from src.pipeline.types import AnalysisContext, StageResult from src.utils.config import Config from src.utils.io import read_json @@ -670,10 +671,80 @@ def _parse_json_object(text: str) -> dict[str, Any]: return {} +@dataclass +class PredictOutcomeStage: + """ + Predict expected claim disposition from upstream classify/extract/vision features. + + Gold `expected_outcome` on synthetic skeletons uses the same deterministic rule, + so outcome accuracy is a predictive tracking metric alongside classification / + extraction reports. + """ + + cfg: Config + order: int = 4 + name: str = "predict_outcome" + + def run(self, ctx: AnalysisContext) -> StageResult: + flags: list[str] = [] + try: + flat = dict((ctx.extraction or {}).get("fields_flat") or {}) + if ctx.vision and ctx.vision.get("refined_fields"): + flat.update({k: v for k, v in ctx.vision["refined_fields"].items() if v}) + + meta = ctx.document.metadata or {} + skeleton = meta.get("skeleton") if isinstance(meta.get("skeleton"), dict) else None + if skeleton is None and isinstance(meta.get("expected_outcome"), str): + skeleton = {"expected_outcome": meta["expected_outcome"]} + + # Prefer narrative complexity from metadata/skeleton when present. + complexity = None + if skeleton: + complexity = skeleton.get("narrative_complexity") + complexity = complexity or meta.get("narrative_complexity") + + prediction = predict_outcome( + fields=flat, + document_type=(ctx.classification or {}).get("document_type"), + text=ctx.content_for_encoder(), + narrative_complexity=complexity if isinstance(complexity, str) else None, + gold_skeleton=skeleton, + ) + + confidence = float(prediction.get("confidence") or 0.0) + if confidence < LOW_CONFIDENCE: + flags.append("low_confidence_outcome") + if prediction.get("features", {}).get("estimated_damage") is None: + flags.append("outcome_missing_damage") + if prediction.get("gold_outcome") is not None and not prediction.get("correct"): + flags.append("outcome_mismatch_vs_gold") + + return StageResult( + stage=self.name, + order=self.order, + ok=True, + confidence=confidence, + flags=flags, + payload=prediction, + ) + except Exception as exc: + logger.exception("PredictOutcome stage failed") + return StageResult( + stage=self.name, + order=self.order, + ok=False, + confidence=0.0, + flags=["predict_outcome_failed"], + error=str(exc), + payload={"label_set": list(OUTCOME_LABELS)}, + ) + + @dataclass class SummarizeStage: """ - Memo generation — reacts to markdown, classification, extraction, and vision. + Memo generation — reacts to markdown, classification, extraction, vision, + and predicted claim outcome. Uses a local generative model when configured; otherwise a deterministic template grounded only in upstream stage payloads (no skeleton peeking). @@ -681,7 +752,7 @@ class SummarizeStage: """ cfg: Config - order: int = 4 + order: int = 5 name: str = "summarize" _model: Any = None _tokenizer: Any = None @@ -717,6 +788,7 @@ def _template_memo(self, ctx: AnalysisContext) -> str: clf = ctx.classification or {} ext = ctx.extraction or {} vision = ctx.vision or {} + outcome = ctx.outcome or {} md_meta = ctx.markdown or {} flat = dict(ext.get("fields_flat") or {}) if vision.get("refined_fields"): @@ -735,6 +807,13 @@ def _template_memo(self, ctx: AnalysisContext) -> str: coverage = flat.get("coverage_type") or "unspecified coverage" state = flat.get("state") or "" doc_type = clf.get("document_type") or "unknown" + predicted = ( + outcome.get("expected_outcome") + or outcome.get("outcome_label") + or "unspecified" + ) + outcome_conf = outcome.get("confidence", "n/a") + outcome_desc = outcome.get("description") or "" flags = list(dict.fromkeys(ctx.flags)) review = "Yes — low-confidence upstream stage(s)" if any( @@ -762,6 +841,9 @@ def _template_memo(self, ctx: AnalysisContext) -> str: f"- Source classification confidence: {clf.get('confidence', 'n/a')}", f"- Extraction backend: {ext.get('backend', 'n/a')}", f"- Vision backend: {(vision or {}).get('backend', 'skipped')}", + f"- Predicted claim outcome: `{predicted}` " + f"(confidence {outcome_conf})" + + (f" — {outcome_desc}" if outcome_desc else ""), f"- Markdown backend: {md_meta.get('backend', 'n/a')} " f"(~{md_meta.get('approx_tokens', 'n/a')} tokens; " f"saved ≈{md_meta.get('token_saved_est', 'n/a')})", @@ -770,8 +852,8 @@ def _template_memo(self, ctx: AnalysisContext) -> str: "Issue: whether coverage appears supported by extracted claim facts. " "Rule: coverage turns on the policy declarations, conditions, and applicable exclusions. " f"Application: extracted fields from the inbound document at {location}. " - "Conclusion: proceed with investigation and reserve adequacy review pending human confirmation " - "of low-confidence fields.", + f"Conclusion: predicted disposition `{predicted}`; proceed with investigation " + "and reserve adequacy review pending human confirmation of low-confidence fields.", "", "Next Steps", "- Confirm coverage grant/denial points in writing", @@ -814,6 +896,8 @@ def run(self, ctx: AnalysisContext) -> StageResult: flags.append("summarize_missing_classification") if ctx.extraction is None: flags.append("summarize_missing_extraction") + if ctx.outcome is None: + flags.append("summarize_missing_outcome") if self._backend == "transformers": try: diff --git a/src/pipeline/types.py b/src/pipeline/types.py index 003e049..311dcfa 100644 --- a/src/pipeline/types.py +++ b/src/pipeline/types.py @@ -50,6 +50,31 @@ def from_row(cls, row: dict[str, Any]) -> "AnalysisDocument": pdf_path = source_path elif suffix in {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff", ".bmp"}: image_path = source_path + reserved = { + "record_id", + "text", + "claim_id", + "image_path", + "pdf_path", + "source_path", + "path", + "file_path", + "document_type", + "words", + "markdown", + "metadata", + } + meta: dict[str, Any] = {} + nested = row.get("metadata") + if isinstance(nested, dict): + meta.update(nested) + meta.update({k: v for k, v in row.items() if k not in reserved}) + # Keep gold outcome / skeleton available to predict_outcome even when + # callers only set top-level keys. + if "skeleton" in row and "skeleton" not in meta: + meta["skeleton"] = row["skeleton"] + if "expected_outcome" in row and "expected_outcome" not in meta: + meta["expected_outcome"] = row["expected_outcome"] return cls( record_id=record_id, text=text, @@ -59,25 +84,7 @@ def from_row(cls, row: dict[str, Any]) -> "AnalysisDocument": source_path=source_path, document_type_hint=row.get("document_type"), markdown=row.get("markdown"), - metadata={ - k: v - for k, v in row.items() - if k - not in { - "record_id", - "text", - "claim_id", - "image_path", - "pdf_path", - "source_path", - "path", - "file_path", - "document_type", - "words", - "skeleton", - "markdown", - } - }, + metadata=meta, ) def llm_text(self) -> str: @@ -102,6 +109,7 @@ class AnalysisContext: classification: dict[str, Any] | None = None extraction: dict[str, Any] | None = None vision: dict[str, Any] | None = None + outcome: dict[str, Any] | None = None summary: dict[str, Any] | None = None flags: list[str] = field(default_factory=list) @@ -129,6 +137,8 @@ def add(self, result: StageResult) -> None: self.extraction = result.payload elif result.stage == "vision_llm": self.vision = result.payload + elif result.stage == "predict_outcome": + self.outcome = result.payload elif result.stage == "summarize": self.summary = result.payload @@ -156,6 +166,7 @@ def to_dict(self) -> dict[str, Any]: "classification": self.classification, "extraction": self.extraction, "vision": self.vision, + "outcome": self.outcome, "summary": self.summary, "flags": list(dict.fromkeys(self.flags)), "stages": [ @@ -171,9 +182,10 @@ def to_dict(self) -> dict[str, Any]: for s in self.stages ], "memo": (self.summary or {}).get("memo"), + "expected_outcome": (self.outcome or {}).get("expected_outcome"), "low_confidence": any( s.confidence < 0.55 and s.ok for s in self.stages - if s.stage in {"classify", "extract", "to_markdown"} + if s.stage in {"classify", "extract", "to_markdown", "predict_outcome"} ), } diff --git a/tests/fixtures/sample_skeletons.jsonl b/tests/fixtures/sample_skeletons.jsonl index 92385ce..f22e6d9 100644 --- a/tests/fixtures/sample_skeletons.jsonl +++ b/tests/fixtures/sample_skeletons.jsonl @@ -1,3 +1,3 @@ -{"claim_id": "CLM-2026-763588", "document_type": "application_personal", "acord_form_number": "90", "policy": {"policy_number": "MN-455784-Z", "policyholder_name": "Jordan Lewis", "state": "MN", "coverage_type": "commercial_general_liability", "effective_date": "2025-04-07"}, "loss_event": {"date_of_loss": "2026-04-08", "loss_type": "collision", "location": "1890 Pine Blvd, MN", "description_seed": "rear-end collision at intersection, moderate speed", "police_report_filed": true, "injuries_reported": false}, "parties": {"insured": "Jordan Lewis", "claimant": "Harper Young", "adjuster_assigned": "Cameron Garcia"}, "financials": {"estimated_damage": 956.33, "deductible": 500.0, "reserve_set": 586.01}, "narrative_complexity": "standard", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2026-763588::application_personal::0"} -{"claim_id": "CLM-2023-397383", "document_type": "loss_notice", "acord_form_number": "2", "policy": {"policy_number": "CO-102260-M", "policyholder_name": "Jordan Brown", "state": "CO", "coverage_type": "commercial_general_liability", "effective_date": "2022-08-31"}, "loss_event": {"date_of_loss": "2023-02-17", "loss_type": "collision", "location": "4445 Sunset Ave, CO", "description_seed": "side-swipe in parking lot during rain", "police_report_filed": false, "injuries_reported": false}, "parties": {"insured": "Jordan Brown", "claimant": null, "adjuster_assigned": "Rowan Kim"}, "financials": {"estimated_damage": 1070.37, "deductible": 500.0, "reserve_set": 963.19}, "narrative_complexity": "fraud_flagged", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2023-397383::loss_notice::1"} -{"claim_id": "CLM-2025-623940", "document_type": "application_commercial", "acord_form_number": "125", "policy": {"policy_number": "FL-171262-V", "policyholder_name": "Riley Garcia", "state": "FL", "coverage_type": "personal_auto", "effective_date": "2024-02-21"}, "loss_event": {"date_of_loss": "2025-01-30", "loss_type": "liability_third_party", "location": "3953 River Rd, FL", "description_seed": "slip and fall on wet lobby floor alleged by visitor", "police_report_filed": false, "injuries_reported": false}, "parties": {"insured": "Riley Garcia", "claimant": null, "adjuster_assigned": "Jordan Garcia"}, "financials": {"estimated_damage": 5011.65, "deductible": 2500.0, "reserve_set": 5310.97}, "narrative_complexity": "clean", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2025-623940::application_commercial::2"} +{"claim_id": "CLM-2026-763588", "document_type": "application_personal", "acord_form_number": "90", "policy": {"policy_number": "MN-455784-Z", "policyholder_name": "Jordan Lewis", "state": "MN", "coverage_type": "commercial_general_liability", "effective_date": "2025-04-07"}, "loss_event": {"date_of_loss": "2026-04-08", "loss_type": "collision", "location": "1890 Pine Blvd, MN", "description_seed": "rear-end collision at intersection, moderate speed", "police_report_filed": true, "injuries_reported": false}, "parties": {"insured": "Jordan Lewis", "claimant": "Harper Young", "adjuster_assigned": "Cameron Garcia"}, "financials": {"estimated_damage": 956.33, "deductible": 500.0, "reserve_set": 586.01}, "narrative_complexity": "standard", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2026-763588::application_personal::0", "expected_outcome": "investigate"} +{"claim_id": "CLM-2023-397383", "document_type": "loss_notice", "acord_form_number": "2", "policy": {"policy_number": "CO-102260-M", "policyholder_name": "Jordan Brown", "state": "CO", "coverage_type": "commercial_general_liability", "effective_date": "2022-08-31"}, "loss_event": {"date_of_loss": "2023-02-17", "loss_type": "collision", "location": "4445 Sunset Ave, CO", "description_seed": "side-swipe in parking lot during rain", "police_report_filed": false, "injuries_reported": false}, "parties": {"insured": "Jordan Brown", "claimant": null, "adjuster_assigned": "Rowan Kim"}, "financials": {"estimated_damage": 1070.37, "deductible": 500.0, "reserve_set": 963.19}, "narrative_complexity": "fraud_flagged", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2023-397383::loss_notice::1", "expected_outcome": "investigate"} +{"claim_id": "CLM-2025-623940", "document_type": "application_commercial", "acord_form_number": "125", "policy": {"policy_number": "FL-171262-V", "policyholder_name": "Riley Garcia", "state": "FL", "coverage_type": "personal_auto", "effective_date": "2024-02-21"}, "loss_event": {"date_of_loss": "2025-01-30", "loss_type": "liability_third_party", "location": "3953 River Rd, FL", "description_seed": "slip and fall on wet lobby floor alleged by visitor", "police_report_filed": false, "injuries_reported": false}, "parties": {"insured": "Riley Garcia", "claimant": null, "adjuster_assigned": "Jordan Garcia"}, "financials": {"estimated_damage": 5011.65, "deductible": 2500.0, "reserve_set": 5310.97}, "narrative_complexity": "clean", "multi_doc_group_id": null, "target_outputs": {"document_text": null, "memo_text": null}, "_record_id": "CLM-2025-623940::application_commercial::2", "expected_outcome": "pay_full"} diff --git a/tests/test_discord_bot.py b/tests/test_discord_bot.py index 948c245..cd682fb 100644 --- a/tests/test_discord_bot.py +++ b/tests/test_discord_bot.py @@ -36,6 +36,12 @@ def test_compact_analysis_and_discord_summary(): "classification": {"document_type": "loss_notice", "confidence": 0.91}, "extraction": {"fields": {"claim_number": "CLM-1", "date_of_loss": "2024-01-15"}}, "vision": {"refined_fields": {"loss_type": "collision"}}, + "outcome": { + "expected_outcome": "pay_partial", + "confidence": 0.72, + "description": "Settle for less than claimed", + }, + "expected_outcome": "pay_partial", "summary": {"memo": "Short memo about the loss."}, "memo": "Short memo about the loss.", "flags": ["low_confidence_extract"], @@ -55,12 +61,14 @@ def test_compact_analysis_and_discord_summary(): assert compact["document_type"] == "loss_notice" assert compact["fields"]["claim_number"] == "CLM-1" assert compact["fields"]["loss_type"] == "collision" + assert compact["expected_outcome"] == "pay_partial" assert compact["memo"] == "Short memo about the loss." text = format_discord_summary(compact) assert "## Document analysis" in text assert "loss_notice" in text assert "CLM-1" in text + assert "pay_partial" in text assert "Short memo" in text assert "low_confidence_extract" in text @@ -69,6 +77,7 @@ def test_overlay_secrets_fills_placeholders(monkeypatch): monkeypatch.setenv("DISCORD_TOKEN", "discord-test-token") monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") monkeypatch.setenv("DISCORD_USE_OPENROUTER", "1") + monkeypatch.delenv("DISCORD_AI_MODEL", raising=False) raw = { "DISCORD_TOKEN": "Paste your Discord token here.", "AI_API_KEY": "Put your API key here.", diff --git a/tests/test_outcome_prediction.py b/tests/test_outcome_prediction.py new file mode 100644 index 0000000..3e4edee --- /dev/null +++ b/tests/test_outcome_prediction.py @@ -0,0 +1,176 @@ +"""Tests for claim outcome prediction + evaluation metrics.""" + +from __future__ import annotations + +from pathlib import Path + +from src.generation.skeleton_sampler import sample_batch +from src.pipeline.eval_outcome import evaluate_outcomes, write_outcome_report +from src.pipeline.orchestrator import analyze_document +from src.pipeline.outcome import ( + OUTCOME_LABELS, + derive_expected_outcome, + features_from_skeleton, + predict_outcome, +) +from src.utils.config import Config +from src.utils.io import read_json + + +def test_derive_expected_outcome_rules(): + assert ( + derive_expected_outcome( + { + "narrative_complexity": "fraud_flagged", + "estimated_damage": 25000, + "deductible": 500, + "injuries_reported": False, + } + ) + == "deny" + ) + assert ( + derive_expected_outcome( + { + "narrative_complexity": "ambiguous", + "estimated_damage": 2000, + "deductible": 500, + "injuries_reported": False, + } + ) + == "investigate" + ) + assert ( + derive_expected_outcome( + { + "narrative_complexity": "clean", + "estimated_damage": 400, + "deductible": 500, + "injuries_reported": False, + } + ) + == "close_without_payment" + ) + assert ( + derive_expected_outcome( + { + "narrative_complexity": "clean", + "estimated_damage": 3000, + "deductible": 500, + "injuries_reported": False, + } + ) + == "pay_full" + ) + + +def test_skeleton_sampler_sets_expected_outcome(): + cfg = Config.load() + dist = read_json(cfg.profiles_dir / "insurance_distributions.json") + schema = read_json(cfg.claim_schema_path) + skeletons = sample_batch(n=12, seed=7, dist=dist, schema=schema) + for sk in skeletons: + assert sk["expected_outcome"] in OUTCOME_LABELS + assert sk["expected_outcome"] == derive_expected_outcome(features_from_skeleton(sk)) + + +def test_predict_outcome_matches_gold_from_perfect_features(): + skeleton = { + "narrative_complexity": "standard", + "document_type": "loss_notice", + "loss_event": { + "loss_type": "collision", + "injuries_reported": False, + "police_report_filed": True, + }, + "financials": { + "estimated_damage": 20000, + "deductible": 1000, + "reserve_set": 18000, + }, + } + gold = derive_expected_outcome(features_from_skeleton(skeleton)) + skeleton["expected_outcome"] = gold + pred = predict_outcome( + fields={ + "estimated_damage": 20000, + "deductible": 1000, + "reserve_set": 18000, + "loss_type": "collision", + }, + document_type="loss_notice", + narrative_complexity="standard", + gold_skeleton=skeleton, + ) + assert pred["expected_outcome"] == gold + assert pred["correct"] is True + + +def test_pipeline_includes_predict_outcome_stage(): + cfg = Config.load() + text = ( + "AUTOMOBILE LOSS NOTICE\n" + "Claim Number: CLM-OUT-1\n" + "Date of Loss: 2024-01-15\n" + "Loss Type: collision\n" + "Estimated Damage: $3,200.00\n" + "Deductible: $500.00\n" + "Reserve Amount: $2,800.00\n" + "Injuries Reported: No\n" + "Complexity assessment: clean.\n" + ) + result = analyze_document( + text, + record_id="outcome-stage-test", + cfg=cfg, + enable_vision=False, + ) + names = [s["stage"] for s in result["stages"]] + assert names == [ + "to_markdown", + "classify", + "extract", + "vision_llm", + "predict_outcome", + "summarize", + ] + assert result["expected_outcome"] in OUTCOME_LABELS + assert result["outcome"]["expected_outcome"] == result["expected_outcome"] + assert "Predicted claim outcome" in (result.get("memo") or "") + + +def test_outcome_eval_report(tmp_path: Path): + cfg = Config.load() + dist = read_json(cfg.profiles_dir / "insurance_distributions.json") + schema = read_json(cfg.claim_schema_path) + skeletons = sample_batch(n=8, seed=3, dist=dist, schema=schema) + rows = [] + for sk in skeletons: + rows.append( + { + "record_id": sk["claim_id"], + "claim_id": sk["claim_id"], + "document_type": sk["document_type"], + "expected_outcome": sk["expected_outcome"], + "skeleton": sk, + "narrative_complexity": sk["narrative_complexity"], + "text": ( + f"LOSS NOTICE\nClaim Number: {sk['claim_id']}\n" + f"Date of Loss: {sk['loss_event']['date_of_loss']}\n" + f"Loss Type: {sk['loss_event']['loss_type']}\n" + f"Estimated Damage: ${sk['financials']['estimated_damage']}\n" + f"Deductible: ${sk['financials']['deductible']}\n" + f"Reserve Amount: ${sk['financials']['reserve_set']}\n" + f"Injuries Reported: {'Yes' if sk['loss_event']['injuries_reported'] else 'No'}\n" + f"Complexity assessment: {sk['narrative_complexity']}.\n" + ), + } + ) + report = evaluate_outcomes(rows, cfg=cfg, enable_vision=False) + assert report["n"] == 8 + assert report["accuracy"] is not None + assert 0.0 <= report["accuracy"] <= 1.0 + json_path, md_path = write_outcome_report(report, tmp_path) + assert json_path.exists() + assert md_path.exists() + assert "Claim outcome prediction" in md_path.read_text(encoding="utf-8") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 5f068d8..f537393 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -103,21 +103,24 @@ def test_analyze_single_chains_stages_in_order(): "classify", "extract", "vision_llm", + "predict_outcome", "summarize", ] - assert [s["order"] for s in result["stages"]] == [0, 1, 2, 3, 4] + assert [s["order"] for s in result["stages"]] == [0, 1, 2, 3, 4, 5] assert all(s["ok"] for s in result["stages"]) assert result["markdown"]["markdown"] assert "| Field | Value |" in result["markdown"]["markdown"] assert result["classification"]["document_type"] == "loss_notice" assert result["extraction"]["fields_flat"].get("date_of_loss") + assert result["expected_outcome"] assert result["memo"] and "ADJUSTER MEMO" in result["memo"] - # Summarize must react to prior stages including markdown + # Summarize must react to prior stages including markdown + outcome grounded = result["summary"]["grounded_in"] assert "to_markdown" in grounded assert "classify" in grounded assert "extract" in grounded assert "vision_llm" in grounded + assert "predict_outcome" in grounded assert result["summary"]["input_from"] == "markdown" assert result["vision"]["llm_input_mode"] == "markdown" @@ -174,6 +177,7 @@ def test_vision_skipped_when_disabled(): "classify", "extract", "vision_llm", + "predict_outcome", "summarize", ] ctx = orch.analyze(AnalysisDocument.from_row(load_jsonl(FIXTURES)[0])) @@ -199,14 +203,17 @@ def test_batch_runner_writes_review_queue(tmp_path: Path): "classify", "extract", "vision_llm", + "predict_outcome", "summarize", ] results = load_jsonl(tmp_path / "batch" / "batch_results.jsonl") assert len(results) == 3 assert all(r.get("memo") for r in results) + assert all(r.get("expected_outcome") for r in results) assert all((r.get("markdown") or {}).get("markdown") for r in results) assert (tmp_path / "batch" / "human_review_queue.jsonl").exists() assert (tmp_path / "batch" / "batch_summary.json").exists() + assert "outcome_metrics" in summary def test_extraction_reacts_to_classification():