diff --git a/.github/ISSUE_TEMPLATE/acquire-contexttrace-unseen-v1.md b/.github/ISSUE_TEMPLATE/acquire-contexttrace-unseen-v1.md new file mode 100644 index 0000000..4437eac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/acquire-contexttrace-unseen-v1.md @@ -0,0 +1,40 @@ +--- +name: Acquire ContextTrace-Unseen-v1 +about: Collect and freeze source/domain/time-disjoint natural RAG traces +title: "Acquire and freeze ContextTrace-Unseen-v1 with source/domain/time-disjoint natural RAG traces" +labels: research, data +assignees: "" +--- + +## Objective + +Acquire and publish the unlabeled ContextTrace-Unseen-v1 manifest before any +`semantic_core_v2` implementation or evaluation. + +## Deliverables + +- [ ] 300--500 Natural OOD traces spanning software/product documentation, + policy/regulatory documents, and support/operational knowledge bases. +- [ ] Approximately 100 temporal/source-condition traces from versioned or + authority-contrasting document pairs. +- [ ] BM25, vector, and hybrid retrieval; multiple chunking/reranking settings; + at least two pinned generator models; clean and naturally failing answers. +- [ ] Immutable source snapshots, hashes, canonical URLs, source families, + domains, and publication windows. +- [ ] Leakage audit proving separation from all calibration sources. +- [ ] Published unlabeled IDs/configuration manifest and SHA-256 before scoring. +- [ ] Independent annotation and sealed adjudication records following + `benchmarks/contexttrace_unseen_v1/ANNOTATION_MANUAL.md`. + +## Exclusions + +Do not run ContextTrace on candidates before the manifest is frozen. Do not +manually author failures, inspect sealed labels during model development, begin +TRAIL transfer, recruit human-study participants, or include 1.2 performance and +dashboard work in this issue. + +## Exit condition + +The issue closes when the public unlabeled manifest/hash and leakage report are +available and the sealed-label custodian confirms that annotations are ready for +one-time scoring after the preregistration lock. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30ce693..a0b4a8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,59 +8,97 @@ on: jobs: python: - name: Python tests + name: Full tests and coverage runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" - + cache: pip - name: Install packages run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip "setuptools>=83" python -m pip install -e "./apps/api[test]" - python -m pip install -e "./packages/contexttrace[test]" - - - name: Run tests - run: python -m pytest -q - - - name: Run ContextTrace-Bench - run: | - python benchmarks/contexttrace_bench/run_contexttrace.py \ - --mode semantic \ - --case-set all \ - --output-dir benchmarks/contexttrace_bench/out \ - --enforce-sota-gates + python -m pip install -e "./packages/contexttrace[test,quality]" + - name: Run tests with coverage gate + env: + PYTHONWARNINGS: error::ResourceWarning + run: python -m pytest -q --cov=contexttrace --cov-report=term-missing --cov-fail-under=80 + - name: Ruff correctness checks + run: python -m ruff check --select E9,F601,F63,F7,F82 packages/contexttrace/contexttrace packages/contexttrace/tests + - name: Type-check new public contracts + run: python -m mypy --follow-imports=skip --ignore-missing-imports packages/contexttrace/contexttrace/contracts.py packages/contexttrace/contexttrace/privacy.py + - name: Audit installed dependencies + run: python -m pip_audit - - name: Upload ContextTrace-Bench artifacts - if: always() - uses: actions/upload-artifact@v4 + sdk-compat: + name: SDK Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - name: contexttrace-bench - path: | - benchmarks/contexttrace_bench/out/contexttrace_bench_results.json - benchmarks/contexttrace_bench/out/results.md - benchmarks/contexttrace_bench/out/leaderboard.md - benchmarks/contexttrace_bench/out/report.html - benchmarks/contexttrace_bench/out/error_analysis.json - benchmarks/contexttrace_bench/out/error_analysis.md - benchmarks/contexttrace_bench/out/candidate_inputs.jsonl - benchmarks/contexttrace_bench/METHODOLOGY.md - benchmarks/contexttrace_bench/BASELINES.md + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e "./packages/contexttrace[test]" + - run: python -m pytest -q packages/contexttrace/tests - - name: Build SDK package - run: | - python -m pip install build - python -m build packages/contexttrace + optional-integrations: + name: Optional integration (${{ matrix.integration }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - integration: fastapi + extra: fastapi + tests: packages/contexttrace/tests/test_fastapi_middleware.py + - integration: langchain + extra: langchain + tests: packages/contexttrace/tests/test_langchain_integration.py + - integration: langgraph + extra: langgraph + tests: packages/contexttrace/tests/test_langgraph_integration.py + - integration: llamaindex + extra: llamaindex + tests: packages/contexttrace/tests/test_llamaindex_integration.py + - integration: opentelemetry + extra: opentelemetry + tests: packages/contexttrace/tests/test_opentelemetry_export.py + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e "./packages/contexttrace[test,${{ matrix.extra }}]" + - run: python -m pytest -q ${{ matrix.tests }} packages/contexttrace/tests/test_integration_concurrency.py - - name: Smoke install SDK wheel + wheel-smoke: + name: Wheel smoke (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - run: python -m pip install --upgrade pip build + - run: python -m build packages/contexttrace + - name: Install and import wheel + shell: bash run: | - python -m venv /tmp/contexttrace-smoke - /tmp/contexttrace-smoke/bin/python -m pip install --upgrade pip - /tmp/contexttrace-smoke/bin/python -m pip install packages/contexttrace/dist/contexttrace-*.whl - /tmp/contexttrace-smoke/bin/contexttrace --version - /tmp/contexttrace-smoke/bin/python -c "import contexttrace; print(contexttrace.__version__)" + python -m pip install packages/contexttrace/dist/contexttrace-*.whl + contexttrace --version + python -c "from contexttrace import ContextTrace, load_json_schema; print(ContextTrace, load_json_schema('TraceV1')['title'])" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd9494b..7937b35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,6 +103,7 @@ jobs: name: pypi url: https://pypi.org/p/contexttrace permissions: + id-token: write contents: read steps: - name: Download distribution artifacts @@ -115,6 +116,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist - password: ${{ secrets.PYPI_API_TOKEN }} - skip-existing: true - attestations: false + attestations: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bf83fc..3c3546d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] +## [1.1.0] - 2026-07-22 + ### Added +- The `1.1.0` release, validated through the `1.1.0rc1` TestPyPI candidate, + including versioned public JSON + schemas, artifact provenance, strict privacy controls, streaming-safe capture, + concurrent integration isolation, verification limits, and batch verification. +- Python 3.10--3.13, optional-integration, dependency-audit, 80% coverage, and + cross-platform wheel quality gates. +- Golden TraceV1 compatibility coverage and adversarial tests for nested + redaction, stream chunking, queue saturation, concurrency, and oversized input. - `contexttrace repair` and SDK helpers for evidence-backed, root-cause-specific repair plans, optional corpus audit, and post-fix regression commands. diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py index 7511f6b..e232528 100644 --- a/apps/api/tests/conftest.py +++ b/apps/api/tests/conftest.py @@ -36,10 +36,12 @@ def override_get_db(): app.dependency_overrides[get_db] = override_get_db app.dependency_overrides[get_llm_judge_provider] = lambda: judge_provider - with TestClient(app) as test_client: - yield test_client - - Base.metadata.drop_all(bind=engine) + try: + with TestClient(app) as test_client: + yield test_client + finally: + Base.metadata.drop_all(bind=engine) + engine.dispose() @pytest.fixture() diff --git a/benchmarks/contexttrace_bench/BASELINES.md b/benchmarks/contexttrace_bench/BASELINES.md index 4edb345..56f982e 100644 --- a/benchmarks/contexttrace_bench/BASELINES.md +++ b/benchmarks/contexttrace_bench/BASELINES.md @@ -1,5 +1,10 @@ # Baseline Comparison Runbook +> **Calibration notice:** all ContextTrace results in this file, including the +> RAGTruth 200-case sample, are calibration/development evidence for the frozen +> `semantic_v1_calibrated` verifier. They are not untouched external-test results. +> See `docs/verifier-governance.md` for the successor-verifier protocol. + This file tracks competitor and reference baseline work for ContextTrace-Bench. Rows should only be described as publishable after they cover the full benchmark case set and are scored by `run_contexttrace.py --candidate`. @@ -8,7 +13,7 @@ case set and are scored by `run_contexttrace.py --candidate`. | System | Runner or Adapter | Status | Publishable | Notes | | --- | --- | --- | --- | --- | -| ContextTrace semantic verifier | `run_contexttrace.py --mode semantic` | Ready | Yes | Local-first product path. CI enforces default quality gates. | +| ContextTrace `semantic_v1_calibrated` verifier | `run_contexttrace.py --mode semantic` | Frozen calibration | No | Local-first compatibility path. Repeatedly calibrated on repository and RAGTruth cases; not external test evidence. | | RAGAS | `run_ragas.py` | Full OpenAI-backed candidate scored | Yes | `gpt-4.1-mini`, 500/500 rows, zero row errors. Faithfulness-only baseline; diagnostic fields are `N/A`. | | DeepEval | `run_deepeval.py` | Full OpenAI-backed candidate scored | Yes | `gpt-4.1-mini`, 500/500 rows, zero row errors. Faithfulness-only baseline; diagnostic fields are `N/A`. | | RAGChecker | `run_ragchecker.py`, `adapt_candidate.py --preset ragchecker` | 200-row real-reference CRAG calibration scored | No | `gpt-4.1-mini`, 200/200 same-ID CRAG rows, real official-answer sidecar, all 11 metrics, and zero errors. The gold-answer grounding proxy remains review-pending, not publishable. | diff --git a/benchmarks/contexttrace_bench/METHODOLOGY.md b/benchmarks/contexttrace_bench/METHODOLOGY.md index 2fde605..a47801c 100644 --- a/benchmarks/contexttrace_bench/METHODOLOGY.md +++ b/benchmarks/contexttrace_bench/METHODOLOGY.md @@ -1,5 +1,14 @@ # ContextTrace-Bench Methodology +## Calibration status + +The current repository benchmark, ContextTrace-Diag-150, Naturalistic Eval v2, +and the 200-case RAGTruth sample are development/calibration sets for +`semantic_v1_calibrated`. Results on them must not be described as untouched +external-test performance. No successor-verifier rule or threshold may be +changed in response to their errors. The source/domain/time-disjoint freeze +protocol is documented in `docs/verifier-governance.md`. + ContextTrace-Bench measures ContextTrace as a verifier, not as a retriever or answer generator. A case is a portable RAG trace with a query, answer, retrieved contexts, optional citations, and expected diagnostic labels. diff --git a/benchmarks/contexttrace_bench/freeze_untouched_split.py b/benchmarks/contexttrace_bench/freeze_untouched_split.py new file mode 100644 index 0000000..ceb72c5 --- /dev/null +++ b/benchmarks/contexttrace_bench/freeze_untouched_split.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REQUIRED_SEPARATION_FIELDS = ( + "track", + "source_family", + "source_document_id", + "domain", + "publication_window", +) +DISJOINT_FIELDS = ( + "source_family", + "source_document_id", + "domain", + "publication_window", +) + + +def freeze_split(case_pack: dict[str, Any], calibration: dict[str, Any]) -> dict[str, Any]: + cases = case_pack.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError("Candidate test pack must contain a non-empty cases list.") + + calibration_cases = calibration.get("cases") or [] + if not isinstance(calibration_cases, list): + raise ValueError("Calibration pack cases must be a list.") + + seen_ids: set[str] = set() + normalized: list[dict[str, str]] = [] + for index, case in enumerate(cases): + if not isinstance(case, dict): + raise ValueError("cases[%s] must be an object." % index) + case_id = str(case.get("id") or "").strip() + if not case_id or case_id in seen_ids: + raise ValueError("Every candidate test case must have a unique non-empty id.") + seen_ids.add(case_id) + record = {"id": case_id} + for field in REQUIRED_SEPARATION_FIELDS: + value = str(case.get(field) or (case.get("metadata") or {}).get(field) or "").strip() + if not value: + raise ValueError("Case %s is missing required separation field %s." % (case_id, field)) + record[field] = value + normalized.append(record) + + calibration_values = { + field: { + str(case.get(field) or (case.get("metadata") or {}).get(field) or "").strip() + for case in calibration_cases + if isinstance(case, dict) + } + for field in DISJOINT_FIELDS + } + overlaps: dict[str, list[str]] = {} + for field in DISJOINT_FIELDS: + values = sorted({record[field] for record in normalized} & calibration_values[field]) + if values: + overlaps[field] = values + if overlaps: + raise ValueError("Candidate split overlaps calibration data: %s" % json.dumps(overlaps, sort_keys=True)) + + normalized.sort(key=lambda item: item["id"]) + canonical = json.dumps(normalized, sort_keys=True, separators=(",", ":")) + return { + "schema_version": 1, + "status": "frozen_unscored", + "frozen_at": datetime.now(timezone.utc).isoformat(), + "case_count": len(normalized), + "separation_fields": list(REQUIRED_SEPARATION_FIELDS), + "cases": normalized, + "manifest_sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + "policy": "Publish this manifest before successor-verifier implementation; score once after lock.", + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Freeze and hash a source-family/domain/publication-window-disjoint test split." + ) + parser.add_argument("--case-pack", required=True) + parser.add_argument("--calibration-pack", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + candidate = json.loads(Path(args.case_pack).read_text(encoding="utf-8")) + calibration = json.loads(Path(args.calibration_pack).read_text(encoding="utf-8")) + manifest = freeze_split(candidate, calibration) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("Frozen %s cases: %s" % (manifest["case_count"], manifest["manifest_sha256"])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/contexttrace_unseen_v1/ANNOTATION_MANUAL.md b/benchmarks/contexttrace_unseen_v1/ANNOTATION_MANUAL.md new file mode 100644 index 0000000..d5dba45 --- /dev/null +++ b/benchmarks/contexttrace_unseen_v1/ANNOTATION_MANUAL.md @@ -0,0 +1,93 @@ +# ContextTrace-Unseen-v1 annotation manual + +Annotators receive the query, answer, retrieved chunks, selected context, +citations, and source snapshots. They do not receive ContextTrace predictions. +Annotations are claim-level unless a field is explicitly trace-level. + +## Unitization + +Split the answer into minimal independently verifiable claims. Preserve qualifiers, +negation, quantities, entities, relationships, dates, and policy conditions. A +sentence containing two independently falsifiable propositions produces two +claims. Record exact answer character offsets for every claim. + +## Fields + +### Claim verdict + +- `supported`: selected evidence entails the complete claim. +- `partially_supported`: evidence entails only a proper subset or omits a material + qualifier. +- `contradicted`: evidence entails an incompatible proposition. +- `unsupported`: evidence is relevant but does not entail the claim. +- `unverifiable`: available material is insufficient, ambiguous, or unsuitable for + a reliable verdict. + +Grounding is not truth. A claim can be textually supported while the source is +stale, superseded, non-canonical, or low authority; encode that in source condition. + +### Failure label + +Choose the observable failure that best describes the answer/evidence relation: +`none`, `retrieval_miss`, `context_selection_error`, `citation_mismatch`, +`answer_overreach`, `contradiction`, `insufficient_evidence`, +`should_have_abstained`, or `source_condition_failure`. Do not infer a hidden +pipeline defect when the trace does not expose it. + +### Primary observable root cause + +Choose exactly one: `none`, `retriever_omitted_available_evidence`, +`reranker_or_selector_dropped_evidence`, `corpus_missing_evidence`, +`generator_ignored_or_exceeded_evidence`, `citation_points_to_wrong_chunk`, +`contexts_conflict`, `source_stale_or_superseded`, `source_noncanonical`, +`source_low_authority`, or `not_observable`. Use `not_observable` rather than +guessing between indistinguishable mechanisms. + +### Citation state + +Choose `not_applicable`, `correct`, `partial`, `wrong_source`, `missing`, or +`malformed`. Citation correctness concerns whether the cited span supports the +associated claim, not whether another uncited chunk supports it. + +### Source condition + +Choose one primary state: `current_canonical`, `current_noncanonical`, `stale`, +`superseded`, `low_authority`, `conflicting_authorities`, or `unknown`. Record the +specific snapshot IDs and comparison evidence used. Prefer `unknown` when +publication order, authority, or canonical status cannot be established. + +### Abstention requirement + +Choose `must_answer`, `may_answer_with_qualification`, or `must_abstain`. Use +`must_abstain` when no available evidence can safely support an answer or when +unresolved authoritative conflicts make a direct answer unsafe. + +### Minimal evidence span + +Select the shortest contiguous source span that preserves the entities, +relationship, polarity, quantity, date, and material conditions needed for the +verdict. Record source ID plus UTF-8 character start/end offsets. Multiple spans +are allowed only when no single span is sufficient. Contradiction spans must +include the conflicting proposition, not merely topical text. + +## Independent annotation and adjudication + +Two annotators independently label all headline cases; if capacity is constrained, +they must cover a preregistered stratified subset of at least 50% from every +domain, track, retriever family, generator, and source-condition category. The +remaining cases receive one annotation and are excluded from agreement claims. + +Preserve both raw annotations. An adjudicator records the selected value, whether +the disagreement was definitional or evidentiary, and a short rationale without +overwriting either original. Report agreement separately for claim boundaries, +claim verdict, failure label, primary root cause, citation state, source condition, +abstention requirement, and evidence spans. Use field-appropriate statistics +rather than a combined kappa; report exact agreement and class prevalence beside +chance-corrected measures. Report span token-F1 and character IoU for evidence. + +## Sealing + +Store raw annotations, disagreements, and adjudications in access-controlled +files unavailable to implementers. Do not export gold labels to the development +workspace until the signed release-lock record confirms the verifier version, NLI +artifact hash, thresholds, metrics, tests, and candidate output schema. diff --git a/benchmarks/contexttrace_unseen_v1/PREREGISTRATION.md b/benchmarks/contexttrace_unseen_v1/PREREGISTRATION.md new file mode 100644 index 0000000..495dd81 --- /dev/null +++ b/benchmarks/contexttrace_unseen_v1/PREREGISTRATION.md @@ -0,0 +1,29 @@ +# ContextTrace-Unseen-v1 preregistration checklist + +Complete and timestamp this file before unsealing labels. + +- [ ] Unlabeled manifest URL and SHA-256 published. +- [ ] Calibration overlap report is empty for source document, normalized content, + source family, domain, and publication window. +- [ ] `semantic_core_v2` commit and source archive SHA-256 recorded. +- [ ] Local NLI model name, immutable revision, local artifact hash, tokenizer + revision, runtime, and numerical precision recorded. +- [ ] Deterministic/NLI routing thresholds and abstention thresholds frozen. +- [ ] Candidate output JSON Schema and taxonomy version frozen. +- [ ] Claim unitization and aggregation rules frozen. +- [ ] Bootstrap seed, confidence-interval method, and paired significance tests + frozen. + +Primary metrics are failure-label macro-F1, root-cause accuracy, unverifiable F1, +dangerous false-green rate, evidence-span token-F1 and character IoU, expected +calibration error, risk-coverage/AURC, p50/p95 latency, and NLI invocation rate. + +Success gates are root-cause accuracy at least 0.75, unverifiable F1 at least +0.60, failure-label macro-F1 at least 0.15 absolute above +`semantic_v1_calibrated`, dangerous false-green rate at most 0.02, and NLI +invocation on fewer than 40% of claims. Report all metrics even if a gate fails. + +The primary comparison is paired on identical traces. Bootstrap confidence +intervals resample source families, not individual claims, to avoid treating +correlated traces from one document as independent. The untouched test is scored +once; inspected errors are retired to future calibration data. diff --git a/benchmarks/contexttrace_unseen_v1/README.md b/benchmarks/contexttrace_unseen_v1/README.md new file mode 100644 index 0000000..7d650f6 --- /dev/null +++ b/benchmarks/contexttrace_unseen_v1/README.md @@ -0,0 +1,50 @@ +# ContextTrace-Unseen-v1 + +Status: acquisition not started; no manifest is frozen and no labels exist. + +This benchmark is the next data milestone for `semantic_core_v2`. It has two +independent tracks: + +- **Natural OOD:** 300--500 traces from software/product documentation, + policy/regulatory documents, and support/operational knowledge bases. +- **Temporal/source condition:** approximately 100 traces made from versioned or + authority-contrasting document pairs. + +## Acquisition contract + +Every trace must be produced by a real RAG run and retain the complete retrieval +and generation configuration. The natural track must cross BM25, vector, and +hybrid retrieval; multiple chunking or reranking settings; at least two generator +models; and clean as well as naturally failing outputs. Failures must not be +manually written or injected after generation. + +Every candidate record must include: + +- immutable trace ID and track; +- source family, source document ID, domain, canonical URL, snapshot hash, and + publication window; +- retriever, chunker, reranker, generator provider/model/revision, prompt hash, + random seed when supported, and generation timestamp; +- retrieved chunk IDs, selected context, answer, citations, and token/latency + metadata; +- no gold diagnostic labels. + +No source document, source family, or publication window may overlap the declared +calibration registry. Near-duplicate snapshots must be detected by normalized +content hash before freezing. + +## Freeze sequence + +1. Acquire candidate sources and produce natural RAG runs without invoking either + `semantic_v1_calibrated` or `semantic_core_v2`. +2. Run leakage checks against every calibration source registry. +3. Freeze the unlabeled manifest with `freeze_untouched_split.py`. +4. Publish the sorted trace IDs, source metadata, generator/retriever configuration + hashes, and manifest SHA-256. Do not publish gold annotations. +5. Independently annotate and seal labels according to `ANNOTATION_MANUAL.md`. +6. Freeze `semantic_core_v2`, its NLI model/revision, thresholds, metrics, + statistical tests, and output schema. +7. Score once. Any inspected error becomes development data for later versions. + +The absence of `manifest.json` in this directory is intentional until real source +acquisition is complete. diff --git a/benchmarks/tests/test_freeze_untouched_split.py b/benchmarks/tests/test_freeze_untouched_split.py new file mode 100644 index 0000000..894eba7 --- /dev/null +++ b/benchmarks/tests/test_freeze_untouched_split.py @@ -0,0 +1,81 @@ +import pytest + +from benchmarks.contexttrace_bench.freeze_untouched_split import freeze_split + + +def _case(case_id, *, source_family="family-a", document="doc-a", domain="software", window="2026-Q2"): + return { + "id": case_id, + "track": "natural_ood", + "source_family": source_family, + "source_document_id": document, + "domain": domain, + "publication_window": window, + } + + +def test_freeze_split_is_sorted_and_hash_is_deterministic(): + candidate = { + "cases": [ + _case("b", source_family="family-b", document="doc-b", domain="support"), + _case("a"), + ] + } + calibration = { + "cases": [ + _case( + "cal", + source_family="calibration-family", + document="calibration-doc", + domain="finance", + window="2025-Q4", + ) + ] + } + + first = freeze_split(candidate, calibration) + second = freeze_split(candidate, calibration) + + assert first["status"] == "frozen_unscored" + assert [case["id"] for case in first["cases"]] == ["a", "b"] + assert first["manifest_sha256"] == second["manifest_sha256"] + assert len(first["manifest_sha256"]) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("source_family", "calibration-family"), + ("source_document_id", "calibration-doc"), + ("domain", "finance"), + ("publication_window", "2025-Q4"), + ], +) +def test_freeze_split_rejects_each_calibration_overlap_dimension(field, value): + candidate_case = _case( + "candidate", + source_family="new-family", + document="new-doc", + domain="support", + window="2026-Q3", + ) + candidate_case["track"] = "temporal_source_condition" + candidate_case[field] = value + calibration_case = _case( + "calibration", + source_family="calibration-family", + document="calibration-doc", + domain="finance", + window="2025-Q4", + ) + + with pytest.raises(ValueError, match="overlaps calibration"): + freeze_split({"cases": [candidate_case]}, {"cases": [calibration_case]}) + + +def test_freeze_split_requires_all_separation_fields(): + candidate = _case("candidate") + del candidate["publication_window"] + + with pytest.raises(ValueError, match="publication_window"): + freeze_split({"cases": [candidate]}, {"cases": []}) diff --git a/docs/artifact-schemas.md b/docs/artifact-schemas.md new file mode 100644 index 0000000..60d5bde --- /dev/null +++ b/docs/artifact-schemas.md @@ -0,0 +1,17 @@ +# Public artifact schemas + +ContextTrace packages JSON Schema Draft 2020-12 contracts for `TraceV1`, +`ClaimVerificationV1`, `DiagnosisV1`, `RepairPlanV1`, and `RegressionCaseV1`. +Load them without relying on repository paths: + +```python +from contexttrace import load_json_schema + +schema = load_json_schema("ClaimVerificationV1") +``` + +Every emitted artifact includes `schema_version`, `taxonomy_version`, +`verifier_version`, and `profile_id`. Readers should reject unsupported major +schema versions and record all four fields with benchmark results. The current +compatibility verifier is `semantic_v1_calibrated`; its identifier explicitly +signals that repository and RAGTruth scores are calibration evidence. diff --git a/docs/integrations/fastapi.md b/docs/integrations/fastapi.md index ab48a00..c87a618 100644 --- a/docs/integrations/fastapi.md +++ b/docs/integrations/fastapi.md @@ -23,9 +23,23 @@ app.add_middleware( ContextTraceFastAPIMiddleware, client=ct, should_trace=lambda request: request["path"] == "/query", + route_allowlist=("/query", "/v1/rag/*"), + content_type_allowlist=("application/json",), + max_capture_bytes=1_048_576, ) ``` +Request and response bodies are bounded tees: ASGI messages are forwarded as +they arrive, while at most `max_capture_bytes` is retained for extraction. +Server-sent events and attachment responses are forwarded without capturing +their bodies. Truncation and skipped-stream counters are available through +`middleware.metrics`. + +Set `background_logging=True` with `max_pending_logs=` to move persistence out +of the request path. Call `await middleware.drain()` during application shutdown +to flush queued writes. When the queue is full, traces are dropped instead of +applying unbounded backpressure, and `logging_dropped` is incremented. + The default extractor looks for: - request query: `query`, `question`, `input`, or `prompt` @@ -59,4 +73,3 @@ app.add_middleware( ``` Logging failures are swallowed by default so tracing does not break the production endpoint. Set `raise_logging_errors=True` during development. - diff --git a/docs/privacy.md b/docs/privacy.md index c1f9b2c..3cc728f 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -31,6 +31,36 @@ local_only: true log_chunk_text: false log_answer_text: false storage_path: .contexttrace/contexttrace.db +privacy: strict +hash_only: false +retention_days: 30 +metadata_allowlist: request_id,tenant_id ``` Use `log_chunk_text: false` or `log_answer_text: false` when traces should preserve metadata and metrics without storing sensitive text. + +`privacy="strict"` covers queries, chunks, answers, citation claims, metadata, +tool inputs and outputs, event names, and agent errors before they reach local or +hosted persistence. Strict mode hashes source identifiers so citation edges stay +joinable without retaining the original IDs. `hash_only=True` replaces captured +text with salted SHA-256 values. + +For selective redaction, pass regexes or custom callables: + +```python +ct = ContextTrace( + redaction_patterns=(r"[\w.+-]+@[\w.-]+",), + custom_redactors=(remove_customer_ids,), + metadata_allowlist=("request_id", "tenant_id"), + retention_days=30, +) +``` + +Local SQLite directories are created with mode `0700` and database files with +mode `0600` where the operating system supports POSIX permissions. Applications +can provide a `TextCipher` implementation through `text_cipher=`; values are +encrypted before persistence and decrypted when traces are fetched. ContextTrace +does not manage encryption keys. + +TTL cleanup runs when the local transport starts and before a new trace is +created. Deletion is permanent, so retain only the minimum operational window. diff --git a/docs/selective-diagnosis-research-plan.md b/docs/selective-diagnosis-research-plan.md new file mode 100644 index 0000000..280a9af --- /dev/null +++ b/docs/selective-diagnosis-research-plan.md @@ -0,0 +1,51 @@ +# Selective Evidence-Chain Diagnosis for Actionable RAG Debugging + +This is the successor-paper plan. The existing RAGTruth, Diag-150, Naturalistic +Eval v2, and repository results are calibration evidence for +`semantic_v1_calibrated`, not an external test set. + +## RQ1: Does fine-grained diagnosis generalize? + +Build a confidence-gated cascade: + +```text +deterministic checks -> uncertain cases only: local NLI -> still uncertain: abstain or optional judge +``` + +Do not force a root-cause label when evidence is insufficient. Pre-register and +report selective-risk curves, coverage, expected calibration error, dangerous +false-green rate, latency, and cost alongside macro-F1. Targets for the +successor verifier are root-cause accuracy at least 0.75, unverifiable F1 at +least 0.60, failure-label macro-F1 improvement of at least 0.15 absolute, and +dangerous false-green rate below 0.02. These are targets, not current results. + +The final test set must be separated from calibration data by source document, +domain, and source time window. Publish its ID/hash manifest before successor +implementation and score it once after the verifier, taxonomy, profile, and +thresholds are locked. + +## RQ2: Are diagnoses actionable for developers? + +Run a real counterbalanced developer study with three conditions: scalar RAG +metrics, an LLM-judge explanation, and the ContextTrace evidence chain plus +repair plan. Recruit approximately 20--30 participants and assign 6--8 broken +pipeline tasks per participant. + +Measure correct root cause, correct repair, time to repair, hidden regression +pass rate, unnecessary modifications, recurrence, and confidence calibration. +Analyze repeated observations with mixed-effects models or paired tests. Check +institutional human-subject requirements before recruitment. LLM-simulated +ratings may pilot the protocol but are not evidence of human actionability. + +## RQ3: Does diagnosis transfer to agent traces? + +Either narrow all claims to RAG or evaluate the agent layer on TRAIL. The transfer +representation should connect tool output to downstream claim, propagated error, +and observable root cause. Keep this extension after RQ1 so agent scope does not +mask unresolved RAG attribution errors. + +## Release rule + +Every result artifact must state its schema, taxonomy, verifier, and profile +versions. Any inspection of untouched-test errors retires that split to +calibration status for future verifier development. diff --git a/docs/verifier-governance.md b/docs/verifier-governance.md new file mode 100644 index 0000000..636b3c8 --- /dev/null +++ b/docs/verifier-governance.md @@ -0,0 +1,34 @@ +# Verifier governance + +The current semantic verifier is frozen as `semantic_v1_calibrated`. Its RAGTruth, +ContextTrace-Diag-150, Naturalistic Eval v2, and repository benchmark results are +calibration evidence, not external test evidence. + +The freeze boundary is recorded in +`contexttrace/verify/rulepacks/legacy_ragtruth_calibrated.yaml`, including the +implementation hash and the benchmark families that must not drive further rule +changes. Existing callers retain the calibrated compatibility behavior. New work +must use a new verifier version and must not inspect the untouched test labels or +errors before the implementation is locked. + +## Untouched-test protocol + +1. Collect cases from source documents, domains, and time windows absent from all + calibration sets. Do not randomly split cases derived from the same document. +2. Require every case to declare `id`, `track`, `source_family`, + `source_document_id`, `domain`, and `publication_window` before freezing. +3. Run `freeze_untouched_split.py` and publish the resulting sorted IDs and SHA-256 + manifest before implementing the successor verifier. +4. Keep gold annotations and test outputs inaccessible to implementers until the + successor verifier, thresholds, taxonomy, and profile are locked. +5. Score once. Subsequent inspection turns the split into calibration data and + requires a newly collected test split. + +Human relabeling can improve annotation quality, but it does not restore test-set +independence after implementation decisions were based on those examples. + +## Rule-pack boundary + +`generic_v1.yaml`, `temporal.yaml`, and `policy.yaml` define the intended boundary +for successor work. Domain packs are opt-in. The legacy calibrated pack is not a +source of rules for an independent evaluation model. diff --git a/packages/contexttrace/contexttrace/__init__.py b/packages/contexttrace/contexttrace/__init__.py index aa7a8f5..4a6a0d2 100644 --- a/packages/contexttrace/contexttrace/__init__.py +++ b/packages/contexttrace/contexttrace/__init__.py @@ -3,6 +3,7 @@ from contexttrace.capture_endpoint import EndpointCapture, capture_endpoint_trace, capture_response_trace from contexttrace.client import AsyncContextTrace, ContextTrace from contexttrace.config import ContextTraceConfig +from contexttrace.contracts import build_regression_case, load_json_schema from contexttrace.diagnose import diagnose_payload, diagnose_trace_file, write_diagnosis_regression_test from contexttrace.diagnose_report import DiagnoseReportGenerator from contexttrace.errors import ( @@ -16,6 +17,7 @@ from contexttrace.integrations.langgraph import ContextTraceLangGraphTracer from contexttrace.integrations.llamaindex import ContextTraceLlamaIndexCallbackHandler from contexttrace.integrations.opentelemetry import OpenTelemetryExporter, export_contexttrace_trace +from contexttrace.privacy import PrivacyPolicy, TextCipher from contexttrace.reliability import ReliabilityScore, ReliabilityScorer from contexttrace.repair import build_repair_plan, render_repair_plan, write_repair_plan from contexttrace.report import ReportGenerator @@ -35,18 +37,22 @@ "DiagnoseReportGenerator", "EndpointCapture", "OpenTelemetryExporter", + "PrivacyPolicy", "ReliabilityScore", "ReliabilityScorer", "ReportGenerator", + "TextCipher", "capture_rag_trace", "capture_endpoint_trace", "capture_response_trace", "build_repair_plan", + "build_regression_case", "diagnose_payload", "diagnose_trace_file", "write_diagnosis_regression_test", "export_contexttrace_trace", "langchain_documents_to_contexts", + "load_json_schema", "render_repair_plan", "write_repair_plan", "write_rag_trace", diff --git a/packages/contexttrace/contexttrace/_version.py b/packages/contexttrace/contexttrace/_version.py index 5becc17..6849410 100644 --- a/packages/contexttrace/contexttrace/_version.py +++ b/packages/contexttrace/contexttrace/_version.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.1.0" diff --git a/packages/contexttrace/contexttrace/client.py b/packages/contexttrace/contexttrace/client.py index 382371c..5384c32 100644 --- a/packages/contexttrace/contexttrace/client.py +++ b/packages/contexttrace/contexttrace/client.py @@ -7,6 +7,13 @@ from contexttrace.config import ContextTraceConfig, load_config from contexttrace.errors import ContextTraceConfigError from contexttrace.local import LocalTransport +from contexttrace.privacy import ( + AsyncPrivacyTransport, + PrivacyPolicy, + PrivacyTransport, + Redactor, + TextCipher, +) from contexttrace.report import ReportGenerator from contexttrace.transport import AsyncHttpTransport, AsyncTransport, HttpTransport, Transport @@ -31,6 +38,15 @@ def __init__( storage_path: Optional[str] = None, log_chunk_text: Optional[bool] = None, log_answer_text: Optional[bool] = None, + privacy: Optional[str] = None, + privacy_policy: PrivacyPolicy | None = None, + metadata_allowlist: tuple[str, ...] | None = None, + redaction_patterns: tuple[str, ...] = (), + custom_redactors: tuple[Redactor, ...] = (), + hash_only: Optional[bool] = None, + hash_salt: str = "", + retention_days: Optional[int] = None, + text_cipher: TextCipher | None = None, config_path: Optional[str] = None, ) -> None: self.config = load_config( @@ -47,12 +63,30 @@ def __init__( storage_path=storage_path, log_chunk_text=log_chunk_text, log_answer_text=log_answer_text, + privacy=privacy, + metadata_allowlist=metadata_allowlist, + hash_only=hash_only, + retention_days=retention_days, config_path=config_path, ) _configure_logging(self.config) self.project = self.config.project self.mode = self.config.mode - self._transport = transport or self._build_transport(self.config) + base_transport = transport or self._build_transport(self.config) + policy = privacy_policy or PrivacyPolicy( + profile=self.config.privacy, + metadata_allowlist=( + frozenset(self.config.metadata_allowlist) + if self.config.metadata_allowlist is not None + else (frozenset() if self.config.privacy == "strict" else None) + ), + redaction_patterns=redaction_patterns, + custom_redactors=custom_redactors, + hash_only=self.config.hash_only, + hash_salt=hash_salt, + cipher=text_cipher, + ) + self._transport = PrivacyTransport(base_transport, policy) def _build_transport(self, config: ContextTraceConfig) -> Transport: if config.mode == "local": @@ -62,6 +96,7 @@ def _build_transport(self, config: ContextTraceConfig) -> Transport: debug=config.debug, log_chunk_text=config.log_chunk_text, log_answer_text=config.log_answer_text, + retention_days=config.retention_days, ) if not config.api_key: raise ContextTraceConfigError( @@ -514,6 +549,15 @@ def __init__( storage_path: Optional[str] = None, log_chunk_text: Optional[bool] = None, log_answer_text: Optional[bool] = None, + privacy: Optional[str] = None, + privacy_policy: PrivacyPolicy | None = None, + metadata_allowlist: tuple[str, ...] | None = None, + redaction_patterns: tuple[str, ...] = (), + custom_redactors: tuple[Redactor, ...] = (), + hash_only: Optional[bool] = None, + hash_salt: str = "", + retention_days: Optional[int] = None, + text_cipher: TextCipher | None = None, config_path: Optional[str] = None, ) -> None: self.config = load_config( @@ -530,12 +574,30 @@ def __init__( storage_path=storage_path, log_chunk_text=log_chunk_text, log_answer_text=log_answer_text, + privacy=privacy, + metadata_allowlist=metadata_allowlist, + hash_only=hash_only, + retention_days=retention_days, config_path=config_path, ) _configure_logging(self.config) self.project = self.config.project self.mode = self.config.mode - self._transport = transport or self._build_transport(self.config) + base_transport = transport or self._build_transport(self.config) + policy = privacy_policy or PrivacyPolicy( + profile=self.config.privacy, + metadata_allowlist=( + frozenset(self.config.metadata_allowlist) + if self.config.metadata_allowlist is not None + else (frozenset() if self.config.privacy == "strict" else None) + ), + redaction_patterns=redaction_patterns, + custom_redactors=custom_redactors, + hash_only=self.config.hash_only, + hash_salt=hash_salt, + cipher=text_cipher, + ) + self._transport = AsyncPrivacyTransport(base_transport, policy) def _build_transport(self, config: ContextTraceConfig) -> AsyncTransport: if config.mode == "local": @@ -546,6 +608,7 @@ def _build_transport(self, config: ContextTraceConfig) -> AsyncTransport: debug=config.debug, log_chunk_text=config.log_chunk_text, log_answer_text=config.log_answer_text, + retention_days=config.retention_days, ) ) if not config.api_key: diff --git a/packages/contexttrace/contexttrace/config.py b/packages/contexttrace/contexttrace/config.py index 2d31d07..3ed1ed0 100644 --- a/packages/contexttrace/contexttrace/config.py +++ b/packages/contexttrace/contexttrace/config.py @@ -30,6 +30,10 @@ class ContextTraceConfig: storage_path: str = DEFAULT_STORAGE_PATH log_chunk_text: bool = True log_answer_text: bool = True + privacy: str = "standard" + hash_only: bool = False + retention_days: Optional[int] = None + metadata_allowlist: tuple[str, ...] | None = None eval_endpoint: Optional[str] = None judge_provider: str = "local" judge_base_url: str = "" @@ -54,6 +58,10 @@ def load_config( storage_path: Optional[str] = None, log_chunk_text: Optional[bool] = None, log_answer_text: Optional[bool] = None, + privacy: Optional[str] = None, + hash_only: Optional[bool] = None, + retention_days: Optional[int] = None, + metadata_allowlist: tuple[str, ...] | None = None, eval_endpoint: Optional[str] = None, judge_provider: Optional[str] = None, judge_base_url: Optional[str] = None, @@ -167,6 +175,36 @@ def load_config( True, ) ), + privacy=str( + _first( + privacy, + os.getenv("CONTEXTTRACE_PRIVACY"), + file_values.get("privacy"), + "standard", + ) + ), + hash_only=_as_bool( + _first( + hash_only, + os.getenv("CONTEXTTRACE_HASH_ONLY"), + file_values.get("hash_only"), + False, + ) + ), + retention_days=_optional_int( + _first( + retention_days, + os.getenv("CONTEXTTRACE_RETENTION_DAYS"), + file_values.get("retention_days"), + ) + ), + metadata_allowlist=_as_tuple( + _first( + metadata_allowlist, + os.getenv("CONTEXTTRACE_METADATA_ALLOWLIST"), + file_values.get("metadata_allowlist"), + ) + ), eval_endpoint=_first( eval_endpoint, os.getenv("CONTEXTTRACE_EVAL_ENDPOINT"), @@ -221,6 +259,10 @@ def load_config( if resolved.mode not in {"hosted", "local"}: raise ContextTraceConfigError("ContextTrace mode must be 'hosted' or 'local'.") + if resolved.privacy not in {"standard", "strict"}: + raise ContextTraceConfigError("ContextTrace privacy must be 'standard' or 'strict'.") + if resolved.retention_days is not None and resolved.retention_days < 0: + raise ContextTraceConfigError("ContextTrace retention_days must be zero or greater.") return resolved @@ -238,6 +280,10 @@ def write_default_config(path: str = CONFIG_FILE, *, overwrite: bool = False) -> "storage_path: .contexttrace/contexttrace.db", "log_chunk_text: true", "log_answer_text: true", + "privacy: standard", + "hash_only: false", + "retention_days: ''", + "metadata_allowlist: ''", "judge_provider: local", "judge_base_url: ''", "judge_api_key: ''", @@ -296,3 +342,17 @@ def _as_bool(value: Any) -> bool: if value is None: return False return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _optional_int(value: Any) -> Optional[int]: + if value is None or str(value).strip() == "": + return None + return int(value) + + +def _as_tuple(value: Any) -> tuple[str, ...] | None: + if value is None: + return None + if isinstance(value, (list, tuple, set)): + return tuple(str(item).strip() for item in value if str(item).strip()) + return tuple(part.strip() for part in str(value).split(",") if part.strip()) diff --git a/packages/contexttrace/contexttrace/contracts.py b/packages/contexttrace/contexttrace/contracts.py new file mode 100644 index 0000000..99f9ef8 --- /dev/null +++ b/packages/contexttrace/contexttrace/contracts.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import hashlib +import json +from importlib import resources +from typing import Any + + +TRACE_SCHEMA_VERSION = "1.0" +CLAIM_VERIFICATION_SCHEMA_VERSION = "1.0" +DIAGNOSIS_SCHEMA_VERSION = "1.0" +REPAIR_PLAN_SCHEMA_VERSION = "1.0" +REGRESSION_CASE_SCHEMA_VERSION = "1.0" + +TAXONOMY_VERSION = "1.0" +VERIFIER_VERSION = "semantic_v1_calibrated" +DEFAULT_PROFILE_ID = "full_v1" + +SCHEMA_FILES = { + "TraceV1": "trace-v1.schema.json", + "ClaimVerificationV1": "claim-verification-v1.schema.json", + "DiagnosisV1": "diagnosis-v1.schema.json", + "RepairPlanV1": "repair-plan-v1.schema.json", + "RegressionCaseV1": "regression-case-v1.schema.json", +} + + +def artifact_provenance(*, schema_version: str, profile_id: str = DEFAULT_PROFILE_ID) -> dict[str, str]: + """Return the required provenance fields for a public artifact.""" + + return { + "schema_version": schema_version, + "taxonomy_version": TAXONOMY_VERSION, + "verifier_version": VERIFIER_VERSION, + "profile_id": profile_id, + } + + +def verification_profile_id(profile: dict[str, Any]) -> str: + """Return a stable ID for custom verification profiles.""" + + canonical = json.dumps(profile, sort_keys=True, separators=(",", ":")) + default = { + "abstention_logic": True, + "citation_alignment": True, + "contradiction_checks": True, + "evidence_span_localization": True, + "root_cause_inference": True, + "semantic_normalization": True, + "source_assessment": True, + } + if profile == default: + return DEFAULT_PROFILE_ID + return "custom_" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12] + + +def load_json_schema(name: str) -> dict[str, Any]: + """Load one of the packaged public JSON Schemas by contract name.""" + + filename = SCHEMA_FILES.get(name) + if filename is None: + raise KeyError("Unknown ContextTrace schema: %s" % name) + resource = resources.files("contexttrace.schemas").joinpath(filename) + return json.loads(resource.read_text(encoding="utf-8")) + + +def build_regression_case( + *, + case_id: str, + trace: dict[str, Any], + expected: dict[str, Any], + profile_id: str = DEFAULT_PROFILE_ID, +) -> dict[str, Any]: + """Build a portable, versioned regression case artifact.""" + + return { + **artifact_provenance( + schema_version=REGRESSION_CASE_SCHEMA_VERSION, + profile_id=profile_id, + ), + "case_id": str(case_id), + "trace": dict(trace), + "expected": dict(expected), + } diff --git a/packages/contexttrace/contexttrace/diagnose.py b/packages/contexttrace/contexttrace/diagnose.py index 7171998..35cab1f 100644 --- a/packages/contexttrace/contexttrace/diagnose.py +++ b/packages/contexttrace/contexttrace/diagnose.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any +from contexttrace.contracts import DIAGNOSIS_SCHEMA_VERSION, artifact_provenance + from contexttrace.verify.runner import verify_trace from contexttrace.verify.schema import RAGTrace, TraceCitation, TraceContext, VerificationInputError, load_trace @@ -71,6 +73,7 @@ def diagnose_payload(payload: dict[str, Any], *, mode: str = "semantic", trace_p summary = _summary(trace_type, rag_result, agent_result, findings, failure_types) return { + **artifact_provenance(schema_version=DIAGNOSIS_SCHEMA_VERSION), "trace_path": trace_path or "", "trace_type": trace_type, "summary": summary, diff --git a/packages/contexttrace/contexttrace/integrations/fastapi.py b/packages/contexttrace/contexttrace/integrations/fastapi.py index cf8d3b2..e195cb1 100644 --- a/packages/contexttrace/contexttrace/integrations/fastapi.py +++ b/packages/contexttrace/contexttrace/integrations/fastapi.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +import fnmatch import inspect import json import time @@ -15,8 +17,8 @@ class ContextTraceFastAPIMiddleware: """ASGI middleware for tracing RAG-style FastAPI endpoints. - The middleware buffers JSON request and response bodies, extracts RAG fields, and logs a - ContextTrace trace after the endpoint completes. Custom extractors can return: + The middleware tees bounded JSON request and response bodies while forwarding ASGI + messages immediately. Custom extractors can return: query, metadata, retrieved_chunks, selected_context, answer, citations, model, and usage. """ @@ -34,6 +36,11 @@ def __init__( should_trace: Optional[ShouldTrace] = None, trace_metadata: Optional[dict[str, Any]] = None, raise_logging_errors: bool = False, + max_capture_bytes: int = 1_048_576, + content_type_allowlist: tuple[str, ...] = ("application/json",), + route_allowlist: tuple[str, ...] | None = None, + background_logging: bool = False, + max_pending_logs: int = 100, ) -> None: self.app = app self.client = client or ContextTrace( @@ -47,6 +54,24 @@ def __init__( self.should_trace = should_trace self.trace_metadata = trace_metadata or {} self.raise_logging_errors = raise_logging_errors + if max_capture_bytes < 0: + raise ValueError("max_capture_bytes must be zero or greater.") + if max_pending_logs < 1: + raise ValueError("max_pending_logs must be at least one.") + self.max_capture_bytes = max_capture_bytes + self.content_type_allowlist = tuple(item.lower() for item in content_type_allowlist) + self.route_allowlist = route_allowlist + self.background_logging = background_logging + self.max_pending_logs = max_pending_logs + self._pending_logs: set[asyncio.Task[Any]] = set() + self.metrics = { + "traces_attempted": 0, + "logging_failures": 0, + "logging_dropped": 0, + "request_capture_truncated": 0, + "response_capture_truncated": 0, + "streaming_responses_skipped": 0, + } async def __call__( self, @@ -58,29 +83,87 @@ async def __call__( await self.app(scope, receive, send) return - request_body, request_messages = await _read_request_body(receive) - request_info = _request_info(scope, request_body) + request_info = _request_info(scope, b"") + if not _route_allowed(str(request_info.get("path") or ""), self.route_allowlist): + await self.app(scope, receive, send) + return + if not _content_type_allowed(request_info.get("headers") or {}, self.content_type_allowlist): + await self.app(scope, receive, send) + return if self.should_trace and not self.should_trace(request_info): - await self.app(scope, _replay_receive(request_messages), send) + await self.app(scope, receive, send) return start_time = time.perf_counter() - response_messages: list[dict[str, Any]] = [] + request_capture = _BodyCapture(self.max_capture_bytes) + response_capture = _BodyCapture(self.max_capture_bytes) + response_start: dict[str, Any] = {} + + async def capture_receive() -> dict[str, Any]: + message = await receive() + if message.get("type") == "http.request": + request_capture.add(message.get("body", b"")) + return message async def capture_send(message: dict[str, Any]) -> None: - response_messages.append(message) + if message.get("type") == "http.response.start": + response_start.update(message) + await send(message) + if message.get("type") == "http.response.body": + headers = _headers(response_start.get("headers") or []) + if _response_capture_allowed(headers, self.content_type_allowlist): + response_capture.add(message.get("body", b"")) try: - await self.app(scope, _replay_receive(request_messages), capture_send) + await self.app(scope, capture_receive, capture_send) except BaseException as exc: + request_info = _request_info(scope, request_capture.body) + request_info["capture_truncated"] = request_capture.truncated await self._log_exception(request_info, exc, start_time) raise - response_info = _response_info(response_messages, start_time) - await self._log_trace(request_info, response_info) + request_info = _request_info(scope, request_capture.body) + request_info["capture_truncated"] = request_capture.truncated + response_info = _response_info( + response_start, + response_capture, + start_time, + content_type_allowlist=self.content_type_allowlist, + ) + if request_capture.truncated: + self.metrics["request_capture_truncated"] += 1 + if response_capture.truncated: + self.metrics["response_capture_truncated"] += 1 + if response_info.get("streaming_capture_skipped"): + self.metrics["streaming_responses_skipped"] += 1 + await self._submit_log(self._log_trace(request_info, response_info)) + + async def drain(self) -> None: + """Wait for background trace writes, normally during application shutdown.""" + + if self._pending_logs: + await asyncio.gather(*tuple(self._pending_logs), return_exceptions=True) + + async def _submit_log(self, operation: Awaitable[None]) -> None: + self.metrics["traces_attempted"] += 1 + if not self.background_logging or self.raise_logging_errors: + await operation + return + if len(self._pending_logs) >= self.max_pending_logs: + self.metrics["logging_dropped"] += 1 + operation.close() if inspect.iscoroutine(operation) else None + return + task = asyncio.create_task(operation) + self._pending_logs.add(task) + task.add_done_callback(self._background_log_done) + await asyncio.sleep(0) - for message in response_messages: - await send(message) + def _background_log_done(self, task: asyncio.Task[Any]) -> None: + self._pending_logs.discard(task) + try: + task.result() + except BaseException: + self.metrics["logging_failures"] += 1 async def _log_exception( self, @@ -107,6 +190,7 @@ async def _log_exception( latency_ms=_elapsed_ms(start_time), ) except Exception: + self.metrics["logging_failures"] += 1 if self.raise_logging_errors: raise @@ -126,6 +210,12 @@ async def _log_trace( **(response_data.get("metadata") or {}), "integration": "fastapi", "http": _http_metadata(request_info, response_info), + "capture": { + "request_truncated": bool(request_info.get("capture_truncated")), + "response_truncated": bool(response_info.get("capture_truncated")), + "streaming_capture_skipped": bool(response_info.get("streaming_capture_skipped")), + "max_capture_bytes": self.max_capture_bytes, + }, } with self.client.trace(query=str(query), metadata=metadata) as trace: @@ -152,6 +242,7 @@ async def _log_trace( if citations: trace.log_citations(citations) except Exception: + self.metrics["logging_failures"] += 1 if self.raise_logging_errors: raise @@ -185,38 +276,8 @@ def default_response_extractor(response: dict[str, Any], request: Optional[dict[ } -async def _read_request_body( - receive: Callable[[], Awaitable[dict[str, Any]]], -) -> tuple[bytes, list[dict[str, Any]]]: - body_parts: list[bytes] = [] - messages: list[dict[str, Any]] = [] - while True: - message = await receive() - messages.append(message) - if message.get("type") != "http.request": - break - body_parts.append(message.get("body", b"")) - if not message.get("more_body", False): - break - return b"".join(body_parts), messages - - -def _replay_receive(messages: list[dict[str, Any]]) -> Callable[[], Awaitable[dict[str, Any]]]: - pending = list(messages) - - async def receive() -> dict[str, Any]: - if pending: - return pending.pop(0) - return {"type": "http.request", "body": b"", "more_body": False} - - return receive - - def _request_info(scope: dict[str, Any], body: bytes) -> dict[str, Any]: - headers = { - key.decode("latin1").lower(): value.decode("latin1") - for key, value in scope.get("headers", []) - } + headers = _headers(scope.get("headers", [])) return { "method": scope.get("method"), "path": scope.get("path"), @@ -227,29 +288,79 @@ def _request_info(scope: dict[str, Any], body: bytes) -> dict[str, Any]: } -def _response_info(messages: list[dict[str, Any]], start_time: float) -> dict[str, Any]: - status_code = None - headers: dict[str, str] = {} - body_parts: list[bytes] = [] - for message in messages: - if message.get("type") == "http.response.start": - status_code = message.get("status") - headers = { - key.decode("latin1").lower(): value.decode("latin1") - for key, value in message.get("headers", []) - } - if message.get("type") == "http.response.body": - body_parts.append(message.get("body", b"")) - body = b"".join(body_parts) +def _response_info( + start_message: dict[str, Any], + capture: "_BodyCapture", + start_time: float, + *, + content_type_allowlist: tuple[str, ...], +) -> dict[str, Any]: + headers = _headers(start_message.get("headers") or []) + body = capture.body + capture_allowed = _response_capture_allowed(headers, content_type_allowlist) return { - "status_code": status_code, + "status_code": start_message.get("status"), "headers": headers, "body": body, "json": _decode_json(body), "latency_ms": _elapsed_ms(start_time), + "capture_truncated": capture.truncated, + "streaming_capture_skipped": not capture_allowed, } +class _BodyCapture: + def __init__(self, limit: int) -> None: + self.limit = limit + self.parts: list[bytes] = [] + self.size = 0 + self.truncated = False + + @property + def body(self) -> bytes: + return b"".join(self.parts) + + def add(self, value: Any) -> None: + chunk = bytes(value or b"") + remaining = max(0, self.limit - self.size) + if len(chunk) > remaining: + self.truncated = True + if remaining: + captured = chunk[:remaining] + self.parts.append(captured) + self.size += len(captured) + + +def _headers(raw_headers: Any) -> dict[str, str]: + return { + key.decode("latin1").lower(): value.decode("latin1") + for key, value in raw_headers + } + + +def _route_allowed(path: str, allowlist: tuple[str, ...] | None) -> bool: + if allowlist is None: + return True + return any(fnmatch.fnmatch(path, pattern) for pattern in allowlist) + + +def _content_type_allowed(headers: dict[str, str], allowlist: tuple[str, ...]) -> bool: + content_type = str(headers.get("content-type") or "").split(";", 1)[0].strip().lower() + if not content_type: + return True + return any(content_type == allowed or (allowed.endswith("/*") and content_type.startswith(allowed[:-1])) for allowed in allowlist) + + +def _response_capture_allowed(headers: dict[str, str], allowlist: tuple[str, ...]) -> bool: + content_type = str(headers.get("content-type") or "").split(";", 1)[0].strip().lower() + disposition = str(headers.get("content-disposition") or "").lower() + if content_type == "text/event-stream" or "attachment" in disposition: + return False + if content_type.endswith("+json"): + return True + return _content_type_allowed(headers, allowlist) + + async def _call_extractor(extractor: Extractor, *args: Any) -> dict[str, Any]: try: value = extractor(*args) diff --git a/packages/contexttrace/contexttrace/integrations/langchain.py b/packages/contexttrace/contexttrace/integrations/langchain.py index 52e9650..dfe2d72 100644 --- a/packages/contexttrace/contexttrace/integrations/langchain.py +++ b/packages/contexttrace/contexttrace/integrations/langchain.py @@ -2,6 +2,8 @@ import time from collections.abc import Iterable as RuntimeIterable +from contextvars import ContextVar +from dataclasses import dataclass, field from typing import Any, Callable, Dict, Iterable, Optional from contexttrace.client import ContextTrace @@ -19,6 +21,20 @@ MetadataExtractor = Callable[[Any, Dict[str, Any]], Dict[str, Any]] +@dataclass +class _RunState: + trace: Any = None + query: Optional[str] = None + retrieved_chunks: list[dict[str, Any]] = field(default_factory=list) + start_time: Optional[float] = None + retriever_start_time: Optional[float] = None + llm_model: Optional[str] = None + llm_usage: dict[str, Any] = field(default_factory=dict) + answer_logged: bool = False + tool_start_times: dict[str, float] = field(default_factory=dict) + tool_names: dict[str, str] = field(default_factory=dict) + + class ContextTraceCallbackHandler(BaseCallbackHandler): # type: ignore[misc] def __init__( self, @@ -51,16 +67,11 @@ def __init__( self.document_converter = document_converter or langchain_document_to_chunk self.metadata_extractor = metadata_extractor self.log_agent_events = log_agent_events - self.trace = None - self.query: Optional[str] = None - self.retrieved_chunks: list[dict[str, Any]] = [] - self.start_time: Optional[float] = None - self.retriever_start_time: Optional[float] = None - self.llm_model: Optional[str] = None - self.llm_usage: dict[str, Any] = {} - self.answer_logged = False - self._tool_start_times: dict[str, float] = {} - self._tool_names: dict[str, str] = {} + self._default_state = _RunState() + self._current_state: ContextVar[_RunState | None] = ContextVar( + "contexttrace_langchain_run_state", default=None + ) + self._run_states: dict[str, _RunState] = {} def on_chain_start( self, @@ -68,6 +79,7 @@ def on_chain_start( inputs: Any, **kwargs: Any, ) -> None: + self._activate(kwargs, new_root=not kwargs.get("parent_run_id")) query = self.query_extractor(inputs) if query: self._ensure_trace( @@ -82,6 +94,7 @@ def on_retriever_start( query: str, **kwargs: Any, ) -> None: + self._activate(kwargs) self.retriever_start_time = time.perf_counter() self._ensure_trace( query=query, @@ -90,6 +103,7 @@ def on_retriever_start( ) def on_retriever_end(self, documents: Iterable[Any], **kwargs: Any) -> None: + self._activate(kwargs) chunks = [self.document_converter(document, index) for index, document in enumerate(documents)] self.retrieved_chunks = chunks @@ -121,6 +135,7 @@ def on_retriever_end(self, documents: Iterable[Any], **kwargs: Any) -> None: ) def on_llm_start(self, serialized: dict[str, Any], prompts: list[str], **kwargs: Any) -> None: + self._activate(kwargs) model = _serialized_name(serialized) if model: self.llm_model = model @@ -133,10 +148,12 @@ def on_llm_start(self, serialized: dict[str, Any], prompts: list[str], **kwargs: ) def on_llm_end(self, response: Any, **kwargs: Any) -> None: + self._activate(kwargs) self.llm_usage = _extract_token_usage(response) self.llm_model = _extract_model(response) or self.llm_model def on_chain_end(self, outputs: Any, **kwargs: Any) -> None: + state = self._activate(kwargs) answer = self.answer_extractor(outputs) if not answer: return @@ -167,8 +184,11 @@ def on_chain_end(self, outputs: Any, **kwargs: Any) -> None: if citations: self.trace.log_citations(citations) self.answer_logged = True + if not kwargs.get("parent_run_id"): + self._release_state(state) def on_chain_error(self, error: BaseException, **kwargs: Any) -> None: + state = self._activate(kwargs) if self.trace is not None and self.log_agent_events: self.trace.log_agent_error( str(error), @@ -189,8 +209,11 @@ def on_chain_error(self, error: BaseException, **kwargs: Any) -> None: }, ) self.answer_logged = True + if not kwargs.get("parent_run_id"): + self._release_state(state) def on_tool_start(self, serialized: dict[str, Any], input_str: str, **kwargs: Any) -> None: + self._activate(kwargs) if not self.log_agent_events: return if self.trace is None: @@ -212,6 +235,7 @@ def on_tool_start(self, serialized: dict[str, Any], input_str: str, **kwargs: An ) def on_tool_end(self, output: Any, **kwargs: Any) -> None: + self._activate(kwargs) if not self.log_agent_events or self.trace is None: return run_id = str(kwargs.get("run_id") or "langchain_tool") @@ -224,6 +248,7 @@ def on_tool_end(self, output: Any, **kwargs: Any) -> None: ) def on_tool_error(self, error: BaseException, **kwargs: Any) -> None: + self._activate(kwargs) if not self.log_agent_events or self.trace is None: return run_id = str(kwargs.get("run_id") or "langchain_tool") @@ -275,6 +300,101 @@ def _merge_metadata(self, source: Any, base: dict[str, Any]) -> dict[str, Any]: merged.update(extracted) return merged + def _activate(self, kwargs: dict[str, Any], *, new_root: bool = False) -> _RunState: + run_id = str(kwargs.get("run_id")) if kwargs.get("run_id") is not None else "" + parent_id = str(kwargs.get("parent_run_id")) if kwargs.get("parent_run_id") is not None else "" + state = self._run_states.get(parent_id) or self._run_states.get(run_id) + current = self._current_state.get() + if state is None and new_root and (run_id or current is None or current.answer_logged): + state = _RunState() + if state is None: + state = current or self._default_state + if run_id: + self._run_states[run_id] = state + if parent_id: + self._run_states[parent_id] = state + self._current_state.set(state) + return state + + def _release_state(self, state: _RunState) -> None: + for run_id in [key for key, value in self._run_states.items() if value is state]: + self._run_states.pop(run_id, None) + + def _state(self) -> _RunState: + return self._current_state.get() or self._default_state + + @property + def trace(self) -> Any: + return self._state().trace + + @trace.setter + def trace(self, value: Any) -> None: + self._state().trace = value + + @property + def query(self) -> Optional[str]: + return self._state().query + + @query.setter + def query(self, value: Optional[str]) -> None: + self._state().query = value + + @property + def retrieved_chunks(self) -> list[dict[str, Any]]: + return self._state().retrieved_chunks + + @retrieved_chunks.setter + def retrieved_chunks(self, value: list[dict[str, Any]]) -> None: + self._state().retrieved_chunks = value + + @property + def start_time(self) -> Optional[float]: + return self._state().start_time + + @start_time.setter + def start_time(self, value: Optional[float]) -> None: + self._state().start_time = value + + @property + def retriever_start_time(self) -> Optional[float]: + return self._state().retriever_start_time + + @retriever_start_time.setter + def retriever_start_time(self, value: Optional[float]) -> None: + self._state().retriever_start_time = value + + @property + def llm_model(self) -> Optional[str]: + return self._state().llm_model + + @llm_model.setter + def llm_model(self, value: Optional[str]) -> None: + self._state().llm_model = value + + @property + def llm_usage(self) -> dict[str, Any]: + return self._state().llm_usage + + @llm_usage.setter + def llm_usage(self, value: dict[str, Any]) -> None: + self._state().llm_usage = value + + @property + def answer_logged(self) -> bool: + return self._state().answer_logged + + @answer_logged.setter + def answer_logged(self, value: bool) -> None: + self._state().answer_logged = value + + @property + def _tool_start_times(self) -> dict[str, float]: + return self._state().tool_start_times + + @property + def _tool_names(self) -> dict[str, str]: + return self._state().tool_names + def langchain_document_to_chunk(document: Any, index: int = 0) -> dict[str, Any]: metadata = getattr(document, "metadata", None) or {} diff --git a/packages/contexttrace/contexttrace/integrations/langgraph.py b/packages/contexttrace/contexttrace/integrations/langgraph.py index 321be86..54cf4a3 100644 --- a/packages/contexttrace/contexttrace/integrations/langgraph.py +++ b/packages/contexttrace/contexttrace/integrations/langgraph.py @@ -2,12 +2,21 @@ import inspect import time +from contextvars import ContextVar +from dataclasses import dataclass, field from functools import wraps from typing import Any, Callable, Optional from contexttrace.client import ContextTrace, TraceSession +@dataclass +class _RunState: + trace: Optional[TraceSession] = None + query: Optional[str] = None + node_starts: dict[str, float] = field(default_factory=dict) + + class ContextTraceLangGraphTracer: """Beta LangGraph adapter for logging graph nodes, tools, memory, and errors.""" @@ -27,11 +36,20 @@ def __init__( client = ContextTrace(**kwargs) self.client = client self.trace_metadata = trace_metadata or {} - self.trace: Optional[TraceSession] = None - self.query: Optional[str] = None - self._node_starts: dict[str, float] = {} + self._default_state = _RunState() + self._current_state: ContextVar[_RunState | None] = ContextVar( + "contexttrace_langgraph_run_state", default=None + ) + self._run_states: dict[str, _RunState] = {} - def start_trace(self, query: str, *, metadata: Optional[dict[str, Any]] = None) -> TraceSession: + def start_trace( + self, + query: str, + *, + metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, + ) -> TraceSession: + state = self._activate(run_id, create=True) if self.trace is not None: return self.trace self.query = query @@ -41,6 +59,8 @@ def start_trace(self, query: str, *, metadata: Optional[dict[str, Any]] = None) "integration": "langgraph", } self.trace = self.client.trace(query=query, metadata=trace_metadata).__enter__() + if run_id: + self._run_states[str(run_id)] = state return self.trace def end_trace( @@ -48,7 +68,9 @@ def end_trace( *, answer: Optional[str] = None, metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, ) -> Optional[TraceSession]: + state = self._activate(run_id) if self.trace is None: return None if answer: @@ -61,6 +83,7 @@ def end_trace( ) trace = self.trace self.trace = None + self._release_state(state) return trace def on_node_start( @@ -70,7 +93,9 @@ def on_node_start( *, event_type: str = "planner_step", metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, ) -> None: + self._activate(run_id) trace = self._ensure_trace(input_json) self._node_starts[name] = time.perf_counter() trace.log_agent_event( @@ -87,7 +112,9 @@ def on_node_end( *, event_type: str = "planner_step", metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, ) -> None: + self._activate(run_id) trace = self._ensure_trace(output_json) trace.log_agent_event( event_type=event_type, @@ -97,7 +124,15 @@ def on_node_end( latency_ms=_elapsed_ms(self._node_starts.get(name)), ) - def on_tool_start(self, name: str, input_json: Any = None, *, metadata: Optional[dict[str, Any]] = None) -> None: + def on_tool_start( + self, + name: str, + input_json: Any = None, + *, + metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, + ) -> None: + self._activate(run_id) self._node_starts[name] = time.perf_counter() self._ensure_trace(input_json).log_tool_call(name, input_json=input_json, metadata=metadata) @@ -108,7 +143,9 @@ def on_tool_end( *, input_json: Any = None, metadata: Optional[dict[str, Any]] = None, + run_id: str | None = None, ) -> None: + self._activate(run_id) self._ensure_trace(output_json).log_tool_result( name, input_json=input_json, @@ -117,7 +154,15 @@ def on_tool_end( latency_ms=_elapsed_ms(self._node_starts.get(name)), ) - def on_error(self, name: str, error: BaseException, *, input_json: Any = None) -> None: + def on_error( + self, + name: str, + error: BaseException, + *, + input_json: Any = None, + run_id: str | None = None, + ) -> None: + self._activate(run_id) self._ensure_trace(input_json).log_agent_error( str(error), name=name, @@ -169,6 +214,48 @@ def _ensure_trace(self, value: Any = None) -> TraceSession: query = _query_from_value(value) or self.query or "langgraph run" return self.start_trace(query) + def _activate(self, run_id: str | None, *, create: bool = False) -> _RunState: + key = str(run_id) if run_id else "" + state = self._run_states.get(key) if key else None + current = self._current_state.get() + if state is None and create and ( + key or current is None or (current.trace is None and current.query is not None) + ): + state = _RunState() + if state is None: + state = current or self._default_state + if key: + self._run_states[key] = state + self._current_state.set(state) + return state + + def _release_state(self, state: _RunState) -> None: + for run_id in [key for key, value in self._run_states.items() if value is state]: + self._run_states.pop(run_id, None) + + def _state(self) -> _RunState: + return self._current_state.get() or self._default_state + + @property + def trace(self) -> Optional[TraceSession]: + return self._state().trace + + @trace.setter + def trace(self, value: Optional[TraceSession]) -> None: + self._state().trace = value + + @property + def query(self) -> Optional[str]: + return self._state().query + + @query.setter + def query(self, value: Optional[str]) -> None: + self._state().query = value + + @property + def _node_starts(self) -> dict[str, float]: + return self._state().node_starts + def _elapsed_ms(start_time: Optional[float]) -> Optional[int]: if start_time is None: diff --git a/packages/contexttrace/contexttrace/integrations/llamaindex.py b/packages/contexttrace/contexttrace/integrations/llamaindex.py index ddd5488..af4e23c 100644 --- a/packages/contexttrace/contexttrace/integrations/llamaindex.py +++ b/packages/contexttrace/contexttrace/integrations/llamaindex.py @@ -1,6 +1,8 @@ from __future__ import annotations import time +from contextvars import ContextVar +from dataclasses import dataclass, field from typing import Any, Callable, Dict, Iterable, Optional from contexttrace.client import ContextTrace @@ -16,6 +18,17 @@ NodeConverter = Callable[[Any, int], Dict[str, Any]] +@dataclass +class _RunState: + trace: Any = None + query: Optional[str] = None + start_time: Optional[float] = None + retrieve_start_time: Optional[float] = None + retrieved_chunks: list[dict[str, Any]] = field(default_factory=list) + source_chunks: list[dict[str, Any]] = field(default_factory=list) + answer_logged: bool = False + + class ContextTraceLlamaIndexCallbackHandler(BaseCallbackHandler): # type: ignore[misc] def __init__( self, @@ -53,13 +66,11 @@ def __init__( self.query_extractor = query_extractor or _extract_query self.response_extractor = response_extractor or _extract_response_text self.node_converter = node_converter or llamaindex_node_to_chunk - self.trace = None - self.query: Optional[str] = None - self.start_time: Optional[float] = None - self.retrieve_start_time: Optional[float] = None - self.retrieved_chunks: list[dict[str, Any]] = [] - self.source_chunks: list[dict[str, Any]] = [] - self.answer_logged = False + self._default_state = _RunState() + self._current_state: ContextVar[_RunState | None] = ContextVar( + "contexttrace_llamaindex_run_state", default=None + ) + self._event_states: dict[str, _RunState] = {} def start_trace(self, trace_id: Optional[str] = None) -> None: return None @@ -79,6 +90,7 @@ def on_event_start( parent_id: str = "", **kwargs: Any, ) -> str: + self._activate(event_id=event_id, parent_id=parent_id, new_root=_event_matches(event_type, "query")) if _event_matches(event_type, "query"): query = self.query_extractor(payload) if query: @@ -98,14 +110,20 @@ def on_event_end( event_id: str = "", **kwargs: Any, ) -> None: + state = self._activate(event_id=event_id, parent_id=str(kwargs.get("parent_id") or "")) if _event_matches(event_type, "retrieve", "retriever"): self._handle_retrieval_end(event_type, payload, event_id, kwargs) return if _event_matches(event_type, "query", "synthesize", "response"): self._handle_response_end(event_type, payload, event_id, kwargs) + if _event_matches(event_type, "query") and self.answer_logged: + self._release_state(state) def trace_query(self, query: str, *, metadata: Optional[dict[str, Any]] = None) -> None: + current = self._current_state.get() + if current is None or current.answer_logged: + self._current_state.set(_RunState()) self._ensure_trace(query=query, event="manual_query", metadata=metadata or {}) def trace_retrieved_nodes(self, nodes: Iterable[Any], *, metadata: Optional[dict[str, Any]] = None) -> None: @@ -234,6 +252,85 @@ def _latency_ms(self) -> int: return 0 return int((time.perf_counter() - self.start_time) * 1000) + def _activate(self, *, event_id: str = "", parent_id: str = "", new_root: bool = False) -> _RunState: + state = self._event_states.get(event_id) + if state is None and not new_root: + state = self._event_states.get(parent_id) + current = self._current_state.get() + if state is None and new_root: + state = _RunState() + if state is None: + state = current or self._default_state + if event_id: + self._event_states[event_id] = state + if parent_id: + self._event_states[parent_id] = state + self._current_state.set(state) + return state + + def _release_state(self, state: _RunState) -> None: + for event_id in [key for key, value in self._event_states.items() if value is state]: + self._event_states.pop(event_id, None) + + def _state(self) -> _RunState: + return self._current_state.get() or self._default_state + + @property + def trace(self) -> Any: + return self._state().trace + + @trace.setter + def trace(self, value: Any) -> None: + self._state().trace = value + + @property + def query(self) -> Optional[str]: + return self._state().query + + @query.setter + def query(self, value: Optional[str]) -> None: + self._state().query = value + + @property + def start_time(self) -> Optional[float]: + return self._state().start_time + + @start_time.setter + def start_time(self, value: Optional[float]) -> None: + self._state().start_time = value + + @property + def retrieve_start_time(self) -> Optional[float]: + return self._state().retrieve_start_time + + @retrieve_start_time.setter + def retrieve_start_time(self, value: Optional[float]) -> None: + self._state().retrieve_start_time = value + + @property + def retrieved_chunks(self) -> list[dict[str, Any]]: + return self._state().retrieved_chunks + + @retrieved_chunks.setter + def retrieved_chunks(self, value: list[dict[str, Any]]) -> None: + self._state().retrieved_chunks = value + + @property + def source_chunks(self) -> list[dict[str, Any]]: + return self._state().source_chunks + + @source_chunks.setter + def source_chunks(self, value: list[dict[str, Any]]) -> None: + self._state().source_chunks = value + + @property + def answer_logged(self) -> bool: + return self._state().answer_logged + + @answer_logged.setter + def answer_logged(self, value: bool) -> None: + self._state().answer_logged = value + def llamaindex_node_to_chunk(node_or_node_with_score: Any, index: int = 0) -> dict[str, Any]: node = getattr(node_or_node_with_score, "node", node_or_node_with_score) diff --git a/packages/contexttrace/contexttrace/local.py b/packages/contexttrace/contexttrace/local.py index 567e8cf..065ecea 100644 --- a/packages/contexttrace/contexttrace/local.py +++ b/packages/contexttrace/contexttrace/local.py @@ -20,17 +20,24 @@ def __init__( debug: bool = False, log_chunk_text: bool = True, log_answer_text: bool = True, + retention_days: Optional[int] = None, ) -> None: self.storage_path = storage_path or str(Path(store_dir) / "contexttrace.db") self.store = SQLiteTraceStore(self.storage_path) + self.retention_days = retention_days + if retention_days is not None: + self.store.cleanup_expired(retention_days=retention_days) self.debug = debug self.log_chunk_text = log_chunk_text self.log_answer_text = log_answer_text + self.trace_restorer = lambda value: value def post(self, path: str, payload: Optional[dict[str, Any]] = None) -> dict[str, Any]: payload = payload or {} self._debug("POST", path, payload) if path == "/v1/traces/start": + if self.retention_days is not None: + self.store.cleanup_expired(retention_days=self.retention_days) trace = self.store.create_trace( project=payload["project"], query=payload["query"], @@ -87,7 +94,7 @@ def post(self, path: str, payload: Optional[dict[str, Any]] = None) -> dict[str, return self.store.add_agent_event(trace_id, payload) if action == "evaluate": - trace = self.store.get_trace(trace_id) + trace = self.trace_restorer(self.store.get_trace(trace_id)) evaluation = _evaluate_trace(trace) self.store.save_evaluation(trace_id, evaluation) return evaluation diff --git a/packages/contexttrace/contexttrace/privacy.py b/packages/contexttrace/contexttrace/privacy.py new file mode 100644 index 0000000..d23d21f --- /dev/null +++ b/packages/contexttrace/contexttrace/privacy.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from typing import Any, Callable, Protocol + + +class TextCipher(Protocol): + """Optional application-provided encryption-at-rest integration.""" + + def encrypt(self, plaintext: str) -> str: + ... + + def decrypt(self, ciphertext: str) -> str: + ... + + +Redactor = Callable[[str], str] + + +@dataclass(frozen=True) +class PrivacyPolicy: + """Sanitize trace payloads before they reach local or hosted persistence.""" + + profile: str = "standard" + metadata_allowlist: frozenset[str] | None = None + redaction_patterns: tuple[str, ...] = () + custom_redactors: tuple[Redactor, ...] = () + hash_only: bool = False + hash_salt: str = "" + cipher: TextCipher | None = field(default=None, repr=False, compare=False) + + def __post_init__(self) -> None: + if self.profile not in {"standard", "strict"}: + raise ValueError("Privacy profile must be 'standard' or 'strict'.") + for pattern in self.redaction_patterns: + re.compile(pattern) + + @classmethod + def strict( + cls, + *, + hash_only: bool = False, + hash_salt: str = "", + metadata_allowlist: frozenset[str] | None = None, + redaction_patterns: tuple[str, ...] = (), + custom_redactors: tuple[Redactor, ...] = (), + cipher: TextCipher | None = None, + ) -> "PrivacyPolicy": + return cls( + profile="strict", + metadata_allowlist=frozenset() if metadata_allowlist is None else metadata_allowlist, + redaction_patterns=redaction_patterns, + custom_redactors=custom_redactors, + hash_only=hash_only, + hash_salt=hash_salt, + cipher=cipher, + ) + + def sanitize(self, path: str, payload: dict[str, Any]) -> dict[str, Any]: + value = _copy(payload) + if path == "/v1/traces/start": + value["query"] = self.protect_text(value.get("query"), field="query") + value["metadata"] = self.protect_metadata(value.get("metadata")) + return value + + action = path.rstrip("/").rsplit("/", 1)[-1] + if action in {"retrieval", "context"}: + value["metadata"] = self.protect_metadata(value.get("metadata")) + if value.get("chunk_ids") is not None: + value["chunk_ids"] = [self.protect_identifier(item) for item in value.get("chunk_ids") or []] + for chunk in value.get("chunks") or []: + if not isinstance(chunk, dict): + continue + for identifier_key in ("chunk_id", "id", "source_chunk_id"): + if identifier_key in chunk: + chunk[identifier_key] = self.protect_identifier(chunk.get(identifier_key)) + chunk["content"] = self.protect_text( + chunk.get("content") or chunk.get("text") or chunk.get("page_content"), + field="chunk text", + ) + if "source" in chunk: + chunk["source"] = self.protect_text(chunk.get("source"), field="source") + chunk["metadata"] = self.protect_metadata(chunk.get("metadata")) + elif action == "answer": + value["answer"] = self.protect_text(value.get("answer"), field="answer text") + value["metadata"] = self.protect_metadata(value.get("metadata")) + elif action == "citations": + for citation in value.get("citations") or []: + if not isinstance(citation, dict): + continue + citation["claim"] = self.protect_text(citation.get("claim"), field="citation claim") + for identifier_key in ("source_chunk_id", "source_id", "chunk_id"): + if identifier_key in citation: + citation[identifier_key] = self.protect_identifier(citation.get(identifier_key)) + if "metadata" in citation: + citation["metadata"] = self.protect_metadata(citation.get("metadata")) + elif action == "agent-events": + value["name"] = self.protect_text(value.get("name"), field="agent event name") + value["input_json"] = self.protect_value(value.get("input_json"), field="tool input") + value["output_json"] = self.protect_value(value.get("output_json"), field="tool output") + value["error_message"] = self.protect_text(value.get("error_message"), field="agent error") + if "metadata" in value: + value["metadata"] = self.protect_metadata(value.get("metadata")) + if "metadata_json" in value: + value["metadata_json"] = self.protect_metadata(value.get("metadata_json")) + return value + + def protect_text(self, value: Any, *, field: str) -> Any: + if value is None: + return None + text = str(value) + if self.hash_only: + digest = hashlib.sha256((self.hash_salt + text).encode("utf-8")).hexdigest() + return "[sha256:%s]" % digest + if self.profile == "strict": + text = "[%s redacted]" % field + else: + for pattern in self.redaction_patterns: + text = re.sub(pattern, "[redacted]", text) + for redactor in self.custom_redactors: + text = str(redactor(text)) + if self.cipher is not None: + return "enc:" + self.cipher.encrypt(text) + return text + + def protect_identifier(self, value: Any) -> Any: + if value is None or self.profile != "strict": + return value + digest = hashlib.sha256((self.hash_salt + str(value)).encode("utf-8")).hexdigest() + return "id_sha256_%s" % digest + + def protect_value(self, value: Any, *, field: str) -> Any: + if isinstance(value, dict): + return {str(key): self.protect_value(item, field=field) for key, item in value.items()} + if isinstance(value, list): + return [self.protect_value(item, field=field) for item in value] + if isinstance(value, tuple): + return [self.protect_value(item, field=field) for item in value] + if isinstance(value, str): + return self.protect_text(value, field=field) + return value + + def protect_metadata(self, value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + allowed = self.metadata_allowlist + result: dict[str, Any] = {} + for key, item in value.items(): + name = str(key) + if allowed is not None and name not in allowed: + continue + result[name] = self.protect_value(item, field="metadata") + return result + + def restore(self, value: Any) -> Any: + if isinstance(value, dict): + return {key: self.restore(item) for key, item in value.items()} + if isinstance(value, list): + return [self.restore(item) for item in value] + if isinstance(value, str) and value.startswith("enc:") and self.cipher is not None: + return self.cipher.decrypt(value[4:]) + return value + + +class PrivacyTransport: + def __init__(self, transport: Any, policy: PrivacyPolicy) -> None: + self.transport = transport + self.policy = policy + _attach_restorer(transport, policy) + + def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + result = self.transport.post(path, self.policy.sanitize(path, payload or {})) + return self.policy.restore(result) + + def get(self, path: str) -> dict[str, Any]: + return self.policy.restore(self.transport.get(path)) + + def close(self) -> Any: + close = getattr(self.transport, "close", None) + return close() if close else None + + +class AsyncPrivacyTransport: + def __init__(self, transport: Any, policy: PrivacyPolicy) -> None: + self.transport = transport + self.policy = policy + _attach_restorer(transport, policy) + + async def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + result = await self.transport.post(path, self.policy.sanitize(path, payload or {})) + return self.policy.restore(result) + + async def get(self, path: str) -> dict[str, Any]: + return self.policy.restore(await self.transport.get(path)) + + async def close(self) -> Any: + close = getattr(self.transport, "close", None) + if close is None: + return None + result = close() + if hasattr(result, "__await__"): + return await result + return result + + +def _copy(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _copy(item) for key, item in value.items()} + if isinstance(value, list): + return [_copy(item) for item in value] + if isinstance(value, tuple): + return [_copy(item) for item in value] + return value + + +def _attach_restorer(transport: Any, policy: PrivacyPolicy) -> None: + target = getattr(transport, "_transport", transport) + if hasattr(target, "trace_restorer"): + target.trace_restorer = policy.restore diff --git a/packages/contexttrace/contexttrace/repair.py b/packages/contexttrace/contexttrace/repair.py index c0b6ade..8f744ea 100644 --- a/packages/contexttrace/contexttrace/repair.py +++ b/packages/contexttrace/contexttrace/repair.py @@ -4,12 +4,14 @@ from pathlib import Path from typing import Any +from contexttrace.contracts import REPAIR_PLAN_SCHEMA_VERSION, artifact_provenance + from contexttrace.diagnose import DiagnoseInputError, diagnose_trace_file from contexttrace.verify.qa import qa_trace from contexttrace.verify.schema import VerificationInputError, load_trace_file -REPAIR_SCHEMA_VERSION = "0.1" +REPAIR_SCHEMA_VERSION = REPAIR_PLAN_SCHEMA_VERSION _ACTION_TEMPLATES: dict[str, list[tuple[str, str]]] = { "retrieval_miss": [ @@ -200,7 +202,7 @@ def build_repair_plan( ) ) return { - "schema_version": REPAIR_SCHEMA_VERSION, + **artifact_provenance(schema_version=REPAIR_SCHEMA_VERSION), "status": "repair_required" if repair_required else "no_repair_needed", "trace_path": str(source_path), "trace_type": trace_type, diff --git a/packages/contexttrace/contexttrace/schemas/__init__.py b/packages/contexttrace/contexttrace/schemas/__init__.py new file mode 100644 index 0000000..1c31b77 --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/__init__.py @@ -0,0 +1 @@ +"""Packaged JSON Schemas for ContextTrace public artifacts.""" diff --git a/packages/contexttrace/contexttrace/schemas/claim-verification-v1.schema.json b/packages/contexttrace/contexttrace/schemas/claim-verification-v1.schema.json new file mode 100644 index 0000000..be8061c --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/claim-verification-v1.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contexttrace.dev/schemas/claim-verification-v1.schema.json", + "title": "ClaimVerificationV1", + "type": "object", + "required": ["schema_version", "taxonomy_version", "verifier_version", "profile_id", "query", "answer", "summary", "claims", "abstention", "diagnostics"], + "properties": { + "schema_version": {"const": "1.0"}, + "taxonomy_version": {"type": "string", "minLength": 1}, + "verifier_version": {"type": "string", "minLength": 1}, + "profile_id": {"type": "string", "minLength": 1}, + "query": {"type": "string"}, + "answer": {"type": "string"}, + "summary": {"type": "object"}, + "claims": {"type": "array", "items": {"type": "object"}}, + "abstention": {"type": "object"}, + "diagnostics": {"type": "object"}, + "metadata": {"type": "object"}, + "verification_profile": {"type": "object"} + }, + "additionalProperties": true +} diff --git a/packages/contexttrace/contexttrace/schemas/diagnosis-v1.schema.json b/packages/contexttrace/contexttrace/schemas/diagnosis-v1.schema.json new file mode 100644 index 0000000..c101e98 --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/diagnosis-v1.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contexttrace.dev/schemas/diagnosis-v1.schema.json", + "title": "DiagnosisV1", + "type": "object", + "required": ["schema_version", "taxonomy_version", "verifier_version", "profile_id", "trace_type", "summary", "findings", "next_actions"], + "properties": { + "schema_version": {"const": "1.0"}, + "taxonomy_version": {"type": "string", "minLength": 1}, + "verifier_version": {"type": "string", "minLength": 1}, + "profile_id": {"type": "string", "minLength": 1}, + "trace_path": {"type": "string"}, + "trace_type": {"type": "string"}, + "summary": {"type": "object"}, + "findings": {"type": "array", "items": {"type": "object"}}, + "rag": {"type": ["object", "null"]}, + "agent": {"type": ["object", "null"]}, + "next_actions": {"type": "array"} + }, + "additionalProperties": true +} diff --git a/packages/contexttrace/contexttrace/schemas/regression-case-v1.schema.json b/packages/contexttrace/contexttrace/schemas/regression-case-v1.schema.json new file mode 100644 index 0000000..419fa73 --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/regression-case-v1.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contexttrace.dev/schemas/regression-case-v1.schema.json", + "title": "RegressionCaseV1", + "type": "object", + "required": ["schema_version", "taxonomy_version", "verifier_version", "profile_id", "case_id", "trace", "expected"], + "properties": { + "schema_version": {"const": "1.0"}, + "taxonomy_version": {"type": "string", "minLength": 1}, + "verifier_version": {"type": "string", "minLength": 1}, + "profile_id": {"type": "string", "minLength": 1}, + "case_id": {"type": "string", "minLength": 1}, + "trace": {"type": "object"}, + "expected": {"type": "object"} + }, + "additionalProperties": false +} diff --git a/packages/contexttrace/contexttrace/schemas/repair-plan-v1.schema.json b/packages/contexttrace/contexttrace/schemas/repair-plan-v1.schema.json new file mode 100644 index 0000000..03968e6 --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/repair-plan-v1.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contexttrace.dev/schemas/repair-plan-v1.schema.json", + "title": "RepairPlanV1", + "type": "object", + "required": ["schema_version", "taxonomy_version", "verifier_version", "profile_id", "status", "primary_root_cause", "evidence", "actions", "verification"], + "properties": { + "schema_version": {"const": "1.0"}, + "taxonomy_version": {"type": "string", "minLength": 1}, + "verifier_version": {"type": "string", "minLength": 1}, + "profile_id": {"type": "string", "minLength": 1}, + "status": {"type": "string"}, + "trace_type": {"type": "string"}, + "primary_root_cause": {"type": "string"}, + "corpus_audited": {"type": "boolean"}, + "evidence": {"type": "array", "items": {"type": "object"}}, + "actions": {"type": "array", "items": {"type": "object"}}, + "verification": {"type": "object"}, + "diagnostic_summary": {"type": "object"} + }, + "additionalProperties": true +} diff --git a/packages/contexttrace/contexttrace/schemas/trace-v1.schema.json b/packages/contexttrace/contexttrace/schemas/trace-v1.schema.json new file mode 100644 index 0000000..6ebec7e --- /dev/null +++ b/packages/contexttrace/contexttrace/schemas/trace-v1.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contexttrace.dev/schemas/trace-v1.schema.json", + "title": "TraceV1", + "type": "object", + "required": ["schema_version", "taxonomy_version", "verifier_version", "profile_id", "query", "answer", "contexts"], + "properties": { + "schema_version": {"const": "1.0"}, + "taxonomy_version": {"type": "string", "minLength": 1}, + "verifier_version": {"type": "string", "minLength": 1}, + "profile_id": {"type": "string", "minLength": 1}, + "query": {"type": "string", "minLength": 1}, + "answer": {"type": "string", "minLength": 1}, + "contexts": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "text"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "text": {"type": "string", "minLength": 1}, + "metadata": {"type": "object"} + }, + "additionalProperties": false + } + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "required": ["claim", "source_id"], + "properties": { + "claim": {"type": "string", "minLength": 1}, + "source_id": {"type": "string", "minLength": 1}, + "metadata": {"type": "object"} + }, + "additionalProperties": false + } + }, + "metadata": {"type": "object"} + }, + "additionalProperties": false +} diff --git a/packages/contexttrace/contexttrace/storage/sqlite_store.py b/packages/contexttrace/contexttrace/storage/sqlite_store.py index 655273c..2d00b5d 100644 --- a/packages/contexttrace/contexttrace/storage/sqlite_store.py +++ b/packages/contexttrace/contexttrace/storage/sqlite_store.py @@ -1,11 +1,14 @@ from __future__ import annotations import json +import os import sqlite3 import time import uuid +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Optional +from typing import Any, Iterator, Optional SCHEMA_VERSION = 1 @@ -15,7 +18,31 @@ class SQLiteTraceStore: def __init__(self, path: str = ".contexttrace/contexttrace.db") -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.parent.chmod(0o700) self._init_db() + self._secure_permissions() + + def cleanup_expired(self, *, retention_days: int) -> int: + """Delete traces and dependent rows older than the configured TTL.""" + + if retention_days < 0: + raise ValueError("retention_days must be zero or greater.") + cutoff_time = datetime.now(timezone.utc) - timedelta(days=retention_days) + cutoff = cutoff_time.strftime("%Y-%m-%dT%H:%M:%SZ") + operator = "<=" if retention_days == 0 else "<" + with self._connect() as db: + rows = db.execute( + "SELECT id FROM traces WHERE created_at %s ?" % operator, + (cutoff,), + ).fetchall() + trace_ids = [str(row["id"]) for row in rows] + for trace_id in trace_ids: + for table in ("chunks", "answers", "citation_checks", "failure_reports", "agent_events"): + db.execute("DELETE FROM %s WHERE trace_id = ?" % table, (trace_id,)) + db.execute("DELETE FROM eval_questions WHERE trace_id = ?", (trace_id,)) + db.execute("DELETE FROM traces WHERE id = ?", (trace_id,)) + self._secure_permissions() + return len(trace_ids) def create_trace(self, *, project: str, query: str, metadata: dict[str, Any]) -> dict[str, Any]: trace_id = _new_id("trace") @@ -465,10 +492,35 @@ def _init_db(self) -> None: (str(SCHEMA_VERSION),), ) - def _connect(self) -> sqlite3.Connection: + @contextmanager + def _connect(self) -> Iterator[sqlite3.Connection]: db = sqlite3.connect(str(self.path)) db.row_factory = sqlite3.Row - return db + try: + self.path.chmod(0o600) + except OSError: + pass + try: + yield db + db.commit() + except BaseException: + db.rollback() + raise + finally: + db.close() + self._secure_permissions() + + def _secure_permissions(self) -> None: + try: + self.path.parent.chmod(0o700) + except OSError: + pass + for candidate in self.path.parent.glob(self.path.name + "*"): + try: + if candidate.is_file(): + os.chmod(candidate, 0o600) + except OSError: + pass def _set_status(self, db: sqlite3.Connection, trace_id: str, status: str) -> None: db.execute( diff --git a/packages/contexttrace/contexttrace/verify/__init__.py b/packages/contexttrace/contexttrace/verify/__init__.py index 2fcd1bc..1925eaf 100644 --- a/packages/contexttrace/contexttrace/verify/__init__.py +++ b/packages/contexttrace/contexttrace/verify/__init__.py @@ -1,4 +1,4 @@ -from contexttrace.verify.runner import verify_trace, verify_trace_file +from contexttrace.verify.runner import VerificationLimits, verify_trace, verify_trace_file, verify_traces from contexttrace.verify.audit import audit_failures, audit_trace, audit_trace_file, audit_trace_with_corpus, load_corpus from contexttrace.verify.audit_benchmark import run_audit_benchmark from contexttrace.verify.compare import compare_failures, compare_trace_files, compare_verifications @@ -82,6 +82,7 @@ "TraceCitation", "TraceContext", "VerificationInputError", + "VerificationLimits", "audit_failures", "audit_trace", "audit_trace_file", @@ -116,6 +117,7 @@ "truth_status", "verify_trace", "verify_trace_file", + "verify_traces", "write_judge_calibration_report", "write_nli_calibration_report", ] diff --git a/packages/contexttrace/contexttrace/verify/citations.py b/packages/contexttrace/contexttrace/verify/citations.py index b5e5a8f..ad8fda0 100644 --- a/packages/contexttrace/contexttrace/verify/citations.py +++ b/packages/contexttrace/contexttrace/verify/citations.py @@ -113,9 +113,24 @@ def _source_fully_supports_claim( ) or ( allow_supported_score_fallback and not critical_missing - and is_supported_match(claim_text, match) + and ( + is_supported_match(claim_text, match) + or ( + float(getattr(match, "score", 0.0) or 0.0) >= 0.4 + and len(getattr(match, "matched_terms", []) or []) >= 2 + ) + ) ) - return is_supported_match(claim_text, match) + if is_supported_match(claim_text, match): + return True + # If this is already the verifier's best source, permit a conservative + # paraphrase alignment fallback. This does not rescue a different-source + # citation and still rejects conflicting facts. + return bool( + allow_supported_score_fallback + and float(getattr(match, "score", 0.0) or 0.0) >= 0.4 + and len(getattr(match, "matched_terms", []) or []) >= 2 + ) def _fact_type(fact: object) -> str: diff --git a/packages/contexttrace/contexttrace/verify/claims.py b/packages/contexttrace/contexttrace/verify/claims.py index 63ba49e..5c92505 100644 --- a/packages/contexttrace/contexttrace/verify/claims.py +++ b/packages/contexttrace/contexttrace/verify/claims.py @@ -14,7 +14,7 @@ def to_dict(self) -> dict[str, str]: _WHITESPACE_RE = re.compile(r"\s+") -_COMPOUND_SPLIT_RE = re.compile(r"\s+(?:and|but)\s+", re.IGNORECASE) +_COMPOUND_SPLIT_RE = re.compile(r"\s*(?:;|\b(?:and|but)\b)\s*", re.IGNORECASE) _FILLER_EXACT = { "thanks", @@ -96,7 +96,9 @@ def extract_claims(answer: str) -> list[Claim]: def _normalize_answer_for_claim_splitting(answer: str) -> str: - normalized = _WHITESPACE_RE.sub(" ", str(answer or "")).strip() + normalized = str(answer or "") + normalized = re.sub(r"(?:^|\n)\s*(?:[-*\u2022]|\d+[.)])\s+", ". ", normalized) + normalized = _WHITESPACE_RE.sub(" ", normalized).strip().lstrip(". ") normalized = re.sub(r"\s+\*\s+", ". ", normalized) normalized = re.sub(r"\s+(Step\s+\d+\s*:)", r". \1", normalized, flags=re.IGNORECASE) normalized = re.sub(r"\s+(Total\s+[^:]{1,40}:)", r". \1", normalized, flags=re.IGNORECASE) diff --git a/packages/contexttrace/contexttrace/verify/evidence.py b/packages/contexttrace/contexttrace/verify/evidence.py index 9e8422e..9fcb0e2 100644 --- a/packages/contexttrace/contexttrace/verify/evidence.py +++ b/packages/contexttrace/contexttrace/verify/evidence.py @@ -839,7 +839,6 @@ def _token_mode(mode: str) -> str: "resumption": "resume", "russian": "russia", "stopping": "stop", - "stopped": "stop", "miscalculated": "miscalculate", "miscalculates": "miscalculate", "miscalculation": "miscalculate", diff --git a/packages/contexttrace/contexttrace/verify/facts.py b/packages/contexttrace/contexttrace/verify/facts.py index 79abcaa..5c928f3 100644 --- a/packages/contexttrace/contexttrace/verify/facts.py +++ b/packages/contexttrace/contexttrace/verify/facts.py @@ -7,6 +7,7 @@ from functools import lru_cache from contexttrace.verify.evidence import has_unnegated_exact_surface_match +from contexttrace.verify.semantic_normalization import extract_normalized_dates, normalize_semantic_text TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]+") @@ -379,6 +380,9 @@ def compare_facts(claim_text: str, evidence_text: str, *, mode: str = "lexical") closed_list_conflict = _relative_pronoun_list_conflict(claim_text, evidence_text) if closed_list_conflict is not None: conflicting_details.append(closed_list_conflict) + passage_conflict = _passage_attribution_conflict(claim_text, evidence_text, mode=mode) + if passage_conflict is not None: + conflicting_details.append(passage_conflict) return FactMatch( required_facts=[fact.text for fact in required_details], @@ -2648,6 +2652,16 @@ def _version_conflict(claim_text: str, evidence_text: str, *, mode: str) -> Requ def _numeric_conflict(claim_text: str, evidence_text: str, *, mode: str) -> RequiredFact | None: + claim_dates = extract_normalized_dates(claim_text) + evidence_dates = extract_normalized_dates(evidence_text) + if ( + claim_dates + and evidence_dates + and claim_dates.isdisjoint(evidence_dates) + and _anchor_overlap(claim_text, evidence_text, mode=mode) >= 0.45 + ): + return RequiredFact(text=", ".join(sorted(claim_dates)), type="date") + claim_ports = set(PORT_RE.findall(str(claim_text or ""))) evidence_ports = set(PORT_RE.findall(str(evidence_text or ""))) if ( @@ -2669,6 +2683,42 @@ def _numeric_conflict(claim_text: str, evidence_text: str, *, mode: str) -> Requ return None +def _passage_attribution_conflict(claim_text: str, evidence_text: str, *, mode: str) -> RequiredFact | None: + attribution = re.search( + r"\bpassages?\s+(?P\d+(?:\s*(?:&|and|,)\s*\d+)*)", + str(claim_text or ""), + flags=re.IGNORECASE, + ) + if not attribution: + return None + passage_ids = re.findall(r"\d+", attribution.group("ids")) + if len(passage_ids) < 2: + return None + blocks: dict[str, str] = {} + matches = list(re.finditer(r"\bpassage\s+(?P\d+)\s*:", str(evidence_text or ""), flags=re.IGNORECASE)) + for index, match in enumerate(matches): + end = matches[index + 1].start() if index + 1 < len(matches) else len(str(evidence_text or "")) + blocks[match.group("id")] = str(evidence_text or "")[match.end() : end] + if not all(passage_id in blocks for passage_id in passage_ids): + return None + core_claim = re.sub(r"\([^)]*\bpassages?\s+[^)]*\)", "", str(claim_text or ""), flags=re.IGNORECASE) + core_claim = re.sub(r"\bpassages?\s+\d+(?:\s*(?:&|and|,)\s*\d+)*", "", core_claim, flags=re.IGNORECASE) + core_claim = _clean(core_claim).strip(" .,:;-") + if not core_claim: + return None + unsupported = [ + passage_id + for passage_id in passage_ids + if _token_overlap(core_claim, blocks[passage_id], mode=mode) < 0.65 + ] + if unsupported: + return RequiredFact( + text="claim attribution is not supported by passage(s) %s" % ", ".join(unsupported), + type="attribution", + ) + return None + + def _content_numbers(text: object) -> set[str]: numbers = _raw_content_numbers(text) numbers.update(_derived_content_numbers(_normalize_content_number_text(text))) @@ -3864,8 +3914,7 @@ def _clean_token(token: str) -> str: def _semantic_text(text: str) -> str: - value = _normalize_negation_text(text).lower() - value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + value = normalize_semantic_text(_normalize_negation_text(text)) value = re.sub(r"\b(\d+)\s*'\s*(\d+)\s*(?:\"|in\b|inch(?:es)?\b)?", r"\1 feet \2 inches", value) value = re.sub(r"(?<=\d),(?=\d{3}\b)", "", value) value = re.sub(r"\b\d+\.(?=[a-z])", " ", value) diff --git a/packages/contexttrace/contexttrace/verify/root_cause.py b/packages/contexttrace/contexttrace/verify/root_cause.py index 8f70ef9..5984b12 100644 --- a/packages/contexttrace/contexttrace/verify/root_cause.py +++ b/packages/contexttrace/contexttrace/verify/root_cause.py @@ -14,6 +14,7 @@ STALE_CONTEXT = "stale_context" LOW_AUTHORITY_SOURCE = "low_authority_source" INSUFFICIENT_CONTEXT = "insufficient_context" +CORPUS_GAP = "corpus_gap" SHOULD_HAVE_ABSTAINED = "should_have_abstained" @@ -53,6 +54,7 @@ def primary_root_cause(claims: list[dict[str, Any]]) -> str: PARTIAL_CONTEXT_SUPPORT, WRONG_SOURCE_CITED, MISSING_CITED_SOURCE, + CORPUS_GAP, INSUFFICIENT_CONTEXT, ] return max( @@ -75,7 +77,7 @@ def diagnose_claim(claim: dict[str, Any], abstention: dict[str, Any]) -> dict[st source_status = _string(claim.get("source_status")) source_assessment = claim.get("source_assessment") if isinstance(claim.get("source_assessment"), dict) else {} - if verdict == "supported" and source_status == "grounded_but_conflicted": + if source_status in {"grounded_but_conflicted", "conflicting_source"}: conflict = _first_source_signal(source_assessment.get("stronger_conflicting_sources") or source_assessment.get("conflicting_sources") or []) return _diagnosis( label=CONFLICTING_CONTEXTS, @@ -89,7 +91,7 @@ def diagnose_claim(claim: dict[str, Any], abstention: dict[str, Any]) -> dict[st closest_evidence=closest_evidence, ) - if verdict == "supported" and source_status == "grounded_but_stale": + if source_status in {"grounded_but_stale", "stale_source", "stale_or_version_conflicted"}: newer = _first_source_signal(source_assessment.get("newer_related_sources") or []) reason = "The claim is grounded, but the supporting source appears stale or explicitly marked stale." if newer: @@ -206,10 +208,24 @@ def diagnose_claim(claim: dict[str, Any], abstention: dict[str, Any]) -> dict[st closest_context_id=closest_context_id, closest_evidence=closest_evidence, ) + best_source = source_assessment.get("best_source") or {} + authoritative_current_gap = bool( + best_source.get("canonical") + and not best_source.get("stale") + and bool(abstention.get("should_abstain")) + ) return _diagnosis( - label=INSUFFICIENT_CONTEXT, - reason="The closest retrieved context overlaps with the claim but is too weak or ambiguous.", - suggested_fix="Retrieve more specific context or require the answer to qualify the claim.", + label=CORPUS_GAP if authoritative_current_gap else INSUFFICIENT_CONTEXT, + reason=( + "The closest current canonical context is topically relevant, but the requested fact is absent from the available corpus." + if authoritative_current_gap + else "The closest retrieved context overlaps with the claim but is too weak or ambiguous." + ), + suggested_fix=( + "Expand the source corpus or abstain until an authoritative source contains the requested fact." + if authoritative_current_gap + else "Retrieve more specific context or require the answer to qualify the claim." + ), missing_fact=missing_fact, closest_context_id=closest_context_id, closest_evidence=closest_evidence, diff --git a/packages/contexttrace/contexttrace/verify/rulepacks/generic_v1.yaml b/packages/contexttrace/contexttrace/verify/rulepacks/generic_v1.yaml new file mode 100644 index 0000000..9707119 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/rulepacks/generic_v1.yaml @@ -0,0 +1,12 @@ +id: generic_v1 +version: 1.0.0 +enabled_by_default: true +status: scaffold +scope: + - lexical_identity + - numeric_consistency + - explicit_negation +notes: >- + New generic rules belong here after validation on development data. The frozen + semantic_v1_calibrated implementation remains the compatibility path until a + rule-pack loader is released. diff --git a/packages/contexttrace/contexttrace/verify/rulepacks/legacy_ragtruth_calibrated.yaml b/packages/contexttrace/contexttrace/verify/rulepacks/legacy_ragtruth_calibrated.yaml new file mode 100644 index 0000000..df845a9 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/rulepacks/legacy_ragtruth_calibrated.yaml @@ -0,0 +1,21 @@ +id: legacy_ragtruth_calibrated +version: 1.0.0 +verifier_version: semantic_v1_calibrated +enabled_by_default: false +status: frozen +implementation: contexttrace.verify.facts +implementation_sha256: 4fa507db2126423c8d0787811e78d0d6ffecf5d7603828b6a6d9b23ca8207bc4 +frozen_at: 2026-07-22 +calibration_sets: + - RAGTruth stratified 200-case sample + - ContextTrace-Diag-150 + - Naturalistic Eval v2 +known_narrow_rules: + - jack_pine_fire_cone_paraphrase + - dermaroller_domain_conflict + - amputation_causality_conflict + - bounded_character_and_review_summaries +change_policy: >- + Do not change this implementation in response to errors from any listed + calibration set. A behavior change requires a new verifier_version and an + untouched, pre-registered test manifest. diff --git a/packages/contexttrace/contexttrace/verify/rulepacks/policy.yaml b/packages/contexttrace/contexttrace/verify/rulepacks/policy.yaml new file mode 100644 index 0000000..79d2423 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/rulepacks/policy.yaml @@ -0,0 +1,8 @@ +id: policy +version: 1.0.0 +enabled_by_default: false +status: scaffold +scope: + - authority_ranking + - explicit_abstention_requirements + - source_allowlists diff --git a/packages/contexttrace/contexttrace/verify/rulepacks/temporal.yaml b/packages/contexttrace/contexttrace/verify/rulepacks/temporal.yaml new file mode 100644 index 0000000..e0a6912 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/rulepacks/temporal.yaml @@ -0,0 +1,8 @@ +id: temporal +version: 1.0.0 +enabled_by_default: false +status: scaffold +scope: + - source_freshness + - version_precedence + - time_sensitive_queries diff --git a/packages/contexttrace/contexttrace/verify/runner.py b/packages/contexttrace/contexttrace/verify/runner.py index b128bb2..e641162 100644 --- a/packages/contexttrace/contexttrace/verify/runner.py +++ b/packages/contexttrace/contexttrace/verify/runner.py @@ -1,9 +1,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path from typing import Any +from contexttrace.contracts import ( + CLAIM_VERIFICATION_SCHEMA_VERSION, + artifact_provenance, + verification_profile_id, +) + from contexttrace.verify.abstention import judge_abstention from contexttrace.verify.citations import ( CITATION_OK, @@ -24,6 +30,7 @@ root_cause_summary, ) from contexttrace.verify.schema import RAGTrace, TraceContext, load_trace_file +from contexttrace.verify.semantic_normalization import semantic_normalization from contexttrace.verify.source_trust import attach_source_assessments from contexttrace.verify.statuses import attach_grounding_statuses from contexttrace.verify.verdicts import classify_claim @@ -37,6 +44,7 @@ class VerificationProfile: source_assessment: bool = True root_cause_inference: bool = True evidence_span_localization: bool = True + semantic_normalization: bool = True def to_dict(self) -> dict[str, bool]: return asdict(self) @@ -45,6 +53,21 @@ def to_dict(self) -> dict[str, bool]: FULL_VERIFICATION_PROFILE = VerificationProfile() +@dataclass(frozen=True) +class VerificationLimits: + """Optional explicit bounds for production verification workloads.""" + + max_contexts: int | None = None + max_context_chars: int | None = None + max_answer_chars: int | None = None + + def __post_init__(self) -> None: + for name in ("max_contexts", "max_context_chars", "max_answer_chars"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError("%s must be zero or greater." % name) + + def verify_trace_file( path: str | Path, *, @@ -52,6 +75,7 @@ def verify_trace_file( judge: ClaimJudge | None = None, nli: ClaimJudge | None = None, profile: VerificationProfile | None = None, + limits: VerificationLimits | None = None, ) -> dict[str, Any]: return verify_trace( load_trace_file(path), @@ -59,6 +83,7 @@ def verify_trace_file( judge=judge, nli=nli, profile=profile, + limits=limits, ) @@ -69,9 +94,102 @@ def verify_trace( judge: ClaimJudge | None = None, nli: ClaimJudge | None = None, profile: VerificationProfile | None = None, + limits: VerificationLimits | None = None, ) -> dict[str, Any]: mode = _normalize_mode(mode) profile = profile or FULL_VERIFICATION_PROFILE + trace, truncation = _apply_limits(trace, limits) + with semantic_normalization(profile.semantic_normalization): + result = _verify_trace_with_profile(trace, mode=mode, judge=judge, nli=nli, profile=profile) + result["truncation"] = truncation + return result + + +def verify_traces( + traces: list[RAGTrace], + *, + mode: str = "lexical", + judge: ClaimJudge | None = None, + nli: ClaimJudge | None = None, + profile: VerificationProfile | None = None, + limits: VerificationLimits | None = None, +) -> list[dict[str, Any]]: + """Verify a batch with one stable profile and explicit workload limits.""" + + return [ + verify_trace( + trace, + mode=mode, + judge=judge, + nli=nli, + profile=profile, + limits=limits, + ) + for trace in traces + ] + + +def _apply_limits( + trace: RAGTrace, + limits: VerificationLimits | None, +) -> tuple[RAGTrace, dict[str, Any]]: + original_context_count = len(trace.contexts) + original_context_chars = sum(len(context.text) for context in trace.contexts) + original_answer_chars = len(trace.answer) + if limits is None: + return trace, { + "applied": False, + "contexts_original": original_context_count, + "contexts_used": original_context_count, + "context_chars_original": original_context_chars, + "context_chars_used": original_context_chars, + "answer_chars_original": original_answer_chars, + "answer_chars_used": original_answer_chars, + } + + contexts = list(trace.contexts) + if limits.max_contexts is not None: + contexts = contexts[: limits.max_contexts] + if limits.max_context_chars is not None: + remaining = limits.max_context_chars + bounded: list[TraceContext] = [] + for context in contexts: + if remaining <= 0: + break + text = context.text[:remaining] + metadata = dict(context.metadata) + if len(text) < len(context.text): + metadata["contexttrace_truncated"] = True + if text: + bounded.append(replace(context, text=text, metadata=metadata)) + remaining -= len(text) + contexts = bounded + answer = trace.answer + if limits.max_answer_chars is not None: + answer = answer[: limits.max_answer_chars] + limited = replace(trace, contexts=contexts, answer=answer) + used_context_chars = sum(len(context.text) for context in contexts) + return limited, { + "applied": True, + "contexts_original": original_context_count, + "contexts_used": len(contexts), + "context_chars_original": original_context_chars, + "context_chars_used": used_context_chars, + "answer_chars_original": original_answer_chars, + "answer_chars_used": len(answer), + "contexts_truncated": len(contexts) < original_context_count or used_context_chars < original_context_chars, + "answer_truncated": len(answer) < original_answer_chars, + } + + +def _verify_trace_with_profile( + trace: RAGTrace, + *, + mode: str, + judge: ClaimJudge | None, + nli: ClaimJudge | None, + profile: VerificationProfile, +) -> dict[str, Any]: evidence_mode = _evidence_mode(mode) judge = _resolve_judge(mode=mode, judge=judge) nli = _resolve_nli(mode=mode, nli=nli) @@ -105,6 +223,7 @@ def verify_trace( verifications = _apply_judge_citation_statuses(trace, claims, verifications, judge, mode=evidence_mode) elif nli is not None: verifications = _apply_judge_citation_statuses(trace, claims, verifications, nli, mode=evidence_mode) + verifications = _refine_authoritative_corpus_gaps(verifications, trace) abstention = ( judge_abstention( query=trace.query, @@ -136,6 +255,7 @@ def verify_trace( } for claim in base_claim_results ] + abstention = _augment_abstention_with_source_status(abstention, claim_results) if profile.root_cause_inference: claim_results = attach_root_causes(claim_results, abstention) claim_results = attach_grounding_statuses(claim_results, trace) @@ -154,6 +274,10 @@ def verify_trace( } ) return { + **artifact_provenance( + schema_version=CLAIM_VERIFICATION_SCHEMA_VERSION, + profile_id=verification_profile_id(profile.to_dict()), + ), "query": trace.query, "answer": trace.answer, "summary": summary, @@ -429,6 +553,8 @@ def _augment_diagnostics_with_source_status( source_statuses = {str(claim.get("source_status") or "") for claim in claims} if "grounded_but_conflicted" in source_statuses: failure_types.append("source_conflict") + if any(_has_direct_polarity_conflict(claim) for claim in claims): + failure_types.append("contradicted_answer") if "grounded_but_stale" in source_statuses: failure_types.append("stale_source") if "grounded_by_low_authority_source" in source_statuses: @@ -447,6 +573,96 @@ def _augment_diagnostics_with_source_status( } +def _refine_authoritative_corpus_gaps(verifications: list[Any], trace: RAGTrace) -> list[Any]: + """Separate absent facts in an authoritative topical source from unrelated misses.""" + contexts = {context.id: context for context in trace.contexts} + refined = [] + for verification in verifications: + context = contexts.get(verification.best_context_id) + metadata = dict(getattr(context, "metadata", {}) or {}) + freshness = str( + metadata.get("freshness") + or metadata.get("freshness_status") + or metadata.get("source_status") + or "" + ).strip().lower() + explicitly_incomplete = freshness == "incomplete" or metadata.get("source_status") == "incomplete" + is_gap = bool( + verification.verdict == "unsupported" + and float(verification.best_score or 0.0) >= 0.15 + and verification.matched_terms + and metadata.get("canonical") is True + and not metadata.get("stale") + and (freshness in {"current", "fresh", "active", "latest"} or explicitly_incomplete) + and verification.missing_facts + ) + if is_gap: + refined.append( + replace( + verification, + verdict="unverifiable", + confidence=round(max(0.55, float(verification.best_score or 0.0)), 3), + reason=( + "A current canonical context is topically relevant, but it does not contain " + "the requested fact; absence is not evidence that the claim is false." + ), + ) + ) + else: + refined.append(verification) + return refined + + +def _augment_abstention_with_source_status( + abstention: dict[str, object], + claims: list[dict[str, Any]], +) -> dict[str, object]: + unsafe = set() + for claim in claims: + status = str(claim.get("source_status") or "") + assessment = claim.get("source_assessment") if isinstance(claim.get("source_assessment"), dict) else {} + if status == "grounded_but_conflicted": + unsafe.add(status) + elif status == "grounded_but_stale" and assessment.get("query_requests_current"): + unsafe.add(status) + best_metadata = ((assessment.get("best_source") or {}).get("metadata") or {}) + if status == "incomplete" and bool(best_metadata.get("requires_abstention")): + unsafe.add("incomplete_context") + if not unsafe: + return abstention + return { + **abstention, + "should_abstain": True, + "reason": ( + "The answer is textually grounded, but the selected evidence is stale or conflicts " + "with a stronger retrieved source." + ), + "source_safety_override": sorted(unsafe), + } + + +def _has_direct_polarity_conflict(claim: dict[str, Any]) -> bool: + assessment = claim.get("source_assessment") if isinstance(claim.get("source_assessment"), dict) else {} + claim_text = str(claim.get("claim") or "").lower() + pairs = ( + ("enable", "disable"), + ("allow", "prohibit"), + ("permit", "forbid"), + ("require", "optional"), + ("increase", "decrease"), + ("accept", "reject"), + ) + for signal in assessment.get("stronger_conflicting_sources") or []: + evidence = str((signal or {}).get("evidence") or "").lower() + if any( + (left in claim_text and right in evidence) + or (right in claim_text and left in evidence) + for left, right in pairs + ): + return True + return False + + def _suggested_fix(failure_types: list[str]) -> str: if "should_have_abstained" in failure_types: return ( diff --git a/packages/contexttrace/contexttrace/verify/schema.py b/packages/contexttrace/contexttrace/verify/schema.py index 94e5c07..1a82569 100644 --- a/packages/contexttrace/contexttrace/verify/schema.py +++ b/packages/contexttrace/contexttrace/verify/schema.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Any +from contexttrace.contracts import DEFAULT_PROFILE_ID, TAXONOMY_VERSION, TRACE_SCHEMA_VERSION, VERIFIER_VERSION + class VerificationInputError(ValueError): """Raised when a portable verification trace cannot be loaded.""" @@ -45,9 +47,17 @@ class RAGTrace: contexts: list[TraceContext] citations: list[TraceCitation] = field(default_factory=list) metadata: dict[str, Any] = field(default_factory=dict) + schema_version: str = TRACE_SCHEMA_VERSION + taxonomy_version: str = TAXONOMY_VERSION + verifier_version: str = VERIFIER_VERSION + profile_id: str = DEFAULT_PROFILE_ID def to_dict(self) -> dict[str, Any]: payload: dict[str, Any] = { + "schema_version": self.schema_version, + "taxonomy_version": self.taxonomy_version, + "verifier_version": self.verifier_version, + "profile_id": self.profile_id, "query": self.query, "answer": self.answer, "contexts": [context.to_dict() for context in self.contexts], @@ -106,6 +116,10 @@ def load_trace(payload: Any, *, source: str = "trace") -> RAGTrace: contexts=contexts, citations=citations, metadata=metadata, + schema_version=str(payload.get("schema_version") or TRACE_SCHEMA_VERSION), + taxonomy_version=str(payload.get("taxonomy_version") or TAXONOMY_VERSION), + verifier_version=str(payload.get("verifier_version") or VERIFIER_VERSION), + profile_id=str(payload.get("profile_id") or DEFAULT_PROFILE_ID), ) diff --git a/packages/contexttrace/contexttrace/verify/semantic_core/__init__.py b/packages/contexttrace/contexttrace/verify/semantic_core/__init__.py new file mode 100644 index 0000000..748fbc0 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/semantic_core/__init__.py @@ -0,0 +1,6 @@ +"""Successor-verifier boundary for generic semantic verification logic. + +The calibrated implementation remains frozen in :mod:`contexttrace.verify.facts`. +New generic logic belongs here only after the untouched-test manifest is published +and the successor verifier receives a distinct version identifier. +""" diff --git a/packages/contexttrace/contexttrace/verify/semantic_normalization.py b/packages/contexttrace/contexttrace/verify/semantic_normalization.py new file mode 100644 index 0000000..76f7be3 --- /dev/null +++ b/packages/contexttrace/contexttrace/verify/semantic_normalization.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import re +import unicodedata +from contextlib import contextmanager +from contextvars import ContextVar +from datetime import datetime +from typing import Iterator + + +ENTITY_ALIASES = { + "u.s.": "united states", + "u.s": "united states", + "usa": "united states", + "uk": "united kingdom", + "u.k.": "united kingdom", + "nyc": "new york city", +} + +NUMBER_WORDS = { + "zero": "0", + "one": "1", + "two": "2", + "three": "3", + "four": "4", + "five": "5", + "six": "6", + "seven": "7", + "eight": "8", + "nine": "9", + "ten": "10", + "eleven": "11", + "twelve": "12", + "thirteen": "13", + "fourteen": "14", + "fifteen": "15", + "sixteen": "16", + "seventeen": "17", + "eighteen": "18", + "nineteen": "19", + "twenty": "20", +} + +MONTHS = { + name.lower(): index + for index, name in enumerate( + ( + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", + ), + start=1, + ) +} +MONTH_PATTERN = "|".join(MONTHS) +_ENABLED: ContextVar[bool] = ContextVar("contexttrace_semantic_normalization_enabled", default=True) + + +@contextmanager +def semantic_normalization(enabled: bool) -> Iterator[None]: + token = _ENABLED.set(bool(enabled)) + try: + yield + finally: + _ENABLED.reset(token) + + +def normalize_semantic_text(text: object) -> str: + value = unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode("ascii") + value = value.lower() + if not _ENABLED.get(): + return value + value = re.sub(r"\bcan(?:not|'t)\b", "not", value) + value = re.sub(r"\b([a-z]+)n't\b", r"\1 not", value) + for alias, canonical in ENTITY_ALIASES.items(): + value = re.sub(r"\b%s\b" % re.escape(alias.strip(".")), canonical, value) + value = re.sub(r"\bgreater than\b|\bhigher than\b", "more than", value) + value = re.sub(r"\blower than\b|\bfewer than\b", "less than", value) + for word, number in NUMBER_WORDS.items(): + value = re.sub(r"\b%s\b" % word, number, value) + value = _normalize_written_dates(value) + value = re.sub(r"(?<=\d),(?=\d{3}\b)", "", value) + return re.sub(r"\s+", " ", value).strip() + + +def extract_normalized_dates(text: object) -> set[str]: + normalized = normalize_semantic_text(text) + return set(re.findall(r"\b\d{4}-\d{2}-\d{2}\b", normalized)) + + +def _normalize_written_dates(value: str) -> str: + pattern = re.compile( + rf"\b(?P{MONTH_PATTERN})\s+(?P\d{{1,2}})(?:st|nd|rd|th)?[,]?\s+(?P\d{{4}})\b", + flags=re.IGNORECASE, + ) + + def replace(match: re.Match[str]) -> str: + month = MONTHS[match.group("month").lower()] + try: + return datetime(int(match.group("year")), month, int(match.group("day"))).strftime("%Y-%m-%d") + except ValueError: + return match.group(0) + + return pattern.sub(replace, value) diff --git a/packages/contexttrace/contexttrace/verify/source_trust.py b/packages/contexttrace/contexttrace/verify/source_trust.py index 6cbab26..d9970a3 100644 --- a/packages/contexttrace/contexttrace/verify/source_trust.py +++ b/packages/contexttrace/contexttrace/verify/source_trust.py @@ -12,6 +12,7 @@ GROUNDING_SOURCE_UNKNOWN = "freshness_unknown" SUPPORTED_BY_CANONICAL_SOURCE = "supported_by_canonical_source" +CURRENT_CANONICAL_SOURCE = "current_canonical_source" GROUNDED_BUT_STALE = "grounded_but_stale" GROUNDED_BUT_CONFLICTED = "grounded_but_conflicted" GROUNDED_BY_LOW_AUTHORITY_SOURCE = "grounded_by_low_authority_source" @@ -68,7 +69,12 @@ def source_assessment( for context in trace.contexts ] supporting = [signal for signal in context_signals if signal["verdict"] == "supported"] - conflicting = [signal for signal in context_signals if signal["verdict"] == "contradicted"] + conflicting = [ + signal + for signal in context_signals + if signal["verdict"] == "contradicted" + or _direct_polarity_conflict(claim_text, str(signal.get("evidence") or "")) + ] newer_sources = _newer_related_sources(best_signal, context_signals) stronger_conflicts = _stronger_conflicts(best_signal, conflicting) explicit_status = _metadata_status(best_signal["metadata"] if best_signal else {}) @@ -81,7 +87,12 @@ def source_assessment( "has_conflict": bool(conflicting), "has_stronger_conflict": bool(stronger_conflicts), "has_newer_related_source": bool(newer_sources), + "has_direct_polarity_conflict": any( + _direct_polarity_conflict(claim_text, str(signal.get("evidence") or "")) + for signal in stronger_conflicts + ), "explicit_source_status": explicit_status, + "query_requests_current": _query_requests_current(trace.query), } @@ -92,17 +103,38 @@ def source_status_from_assessment(claim: dict[str, Any], assessment: dict[str, A if not best: return "no_source" if verdict != "supported": - if assessment.get("has_stronger_conflict") or assessment.get("has_conflict"): + # A current source contradicting the *answer* is not itself a source conflict. + # Reserve this label for disagreement among retrieved sources. + if assessment.get("has_stronger_conflict"): return CONFLICTING_SOURCE + if explicit_status in {"stale", "stale_source", GROUNDED_BUT_STALE}: + return STALE_SOURCE + if explicit_status == "incomplete": + return "incomplete" + if explicit_status in {"current", "fresh", "active", "latest"} and ( + bool(best.get("canonical")) or float(best.get("authority_score") or 0.0) >= HIGH_AUTHORITY_THRESHOLD + ): + return CURRENT_CANONICAL_SOURCE if explicit_status: return explicit_status + if bool(best.get("canonical")) or float(best.get("authority_score") or 0.0) >= HIGH_AUTHORITY_THRESHOLD: + return CURRENT_CANONICAL_SOURCE return GROUNDING_SOURCE_UNKNOWN - if explicit_status in {GROUNDED_BUT_STALE, STALE_SOURCE, "stale_or_version_conflicted"}: + explicitly_stale = explicit_status in { + GROUNDED_BUT_STALE, + STALE_SOURCE, + "stale_or_version_conflicted", + } or bool(best.get("stale")) + if explicitly_stale and assessment.get("query_requests_current"): return GROUNDED_BUT_STALE - if bool(best.get("stale")) or assessment.get("has_newer_related_source"): + if assessment.get("has_direct_polarity_conflict"): + return GROUNDED_BUT_CONFLICTED + if assessment.get("has_newer_related_source"): return GROUNDED_BUT_STALE if assessment.get("has_stronger_conflict"): return GROUNDED_BUT_CONFLICTED + if explicitly_stale: + return GROUNDED_BUT_STALE if float(best.get("authority_score") or 0.0) < LOW_AUTHORITY_THRESHOLD: return GROUNDED_BY_LOW_AUTHORITY_SOURCE if bool(best.get("canonical")) or float(best.get("authority_score") or 0.0) >= HIGH_AUTHORITY_THRESHOLD: @@ -112,6 +144,42 @@ def source_status_from_assessment(claim: dict[str, Any], assessment: dict[str, A return GROUNDING_SOURCE_UNKNOWN +def _query_requests_current(query: str) -> bool: + return bool( + re.search( + r"\b(?:current|currently|latest|newest|now|today|active|in force|effective)\b", + str(query or ""), + flags=re.IGNORECASE, + ) + ) + + +def _direct_polarity_conflict(claim_text: str, evidence_text: str) -> bool: + claim = str(claim_text or "").lower() + evidence = str(evidence_text or "").lower() + pairs = ( + ("enable", "disable"), + ("allow", "prohibit"), + ("permit", "forbid"), + ("require", "optional"), + ("increase", "decrease"), + ("accept", "reject"), + ) + paired = any( + (left in claim and right in evidence) + or (right in claim and left in evidence) + for left, right in pairs + ) + lifecycle_conflict = bool( + re.search(r"\b(?:use|should use|can use)\b", claim) + and re.search( + r"\b(?:removed|retired|deprecated|renamed|moved|no longer|end.of.support|recommends? (?:migrating|migration))\b", + evidence, + ) + ) + return paired or lifecycle_conflict + + def _context_signal(context: Any, claim_text: str, *, mode: str) -> dict[str, Any]: metadata = dict(getattr(context, "metadata", {}) or {}) match = score_claim_against_context(claim_text, context, mode=mode) diff --git a/packages/contexttrace/contexttrace/verify/spans.py b/packages/contexttrace/contexttrace/verify/spans.py index addb8fc..33a591d 100644 --- a/packages/contexttrace/contexttrace/verify/spans.py +++ b/packages/contexttrace/contexttrace/verify/spans.py @@ -104,6 +104,15 @@ def _passage_context_spans(context: TraceContext) -> list[EvidenceSpan]: if sentence_start == 0 and sentence_end == len(block_text.strip()): continue _append_evidence_span(spans, context, raw_start + sentence_start, raw_start + sentence_end) + sentence_text = block_text[sentence_start:sentence_end] + prefix = re.match(r"(?i)^\s*passage\s*\d+\s*:\s*", sentence_text) + if prefix and prefix.end() < len(sentence_text): + _append_evidence_span( + spans, + context, + raw_start + sentence_start + prefix.end(), + raw_start + sentence_end, + ) return spans diff --git a/packages/contexttrace/contexttrace/verify/verdicts.py b/packages/contexttrace/contexttrace/verify/verdicts.py index 16fa8e8..a63814e 100644 --- a/packages/contexttrace/contexttrace/verify/verdicts.py +++ b/packages/contexttrace/contexttrace/verify/verdicts.py @@ -202,8 +202,13 @@ def classify_claim( and has_contexts and is_contradicted(claim.text, contradiction_evidence, match.score, mode=fact_mode) ) + opposed_predicate = bool( + contradiction_checks + and has_contexts + and _opposed_predicate_conflict(claim.text, contradiction_evidence, mode=fact_mode) + ) fully_fact_supported = bool(fact_match.required_facts and not fact_match.missing_facts and not fact_match.conflicting_facts) - if fact_match.conflicting_facts or (contradicted and not fully_fact_supported): + if fact_match.conflicting_facts or opposed_predicate or (contradicted and not fully_fact_supported): verdict = "contradicted" confidence = max(0.66, min(0.98, match.score + 0.12)) reason = ( @@ -305,6 +310,12 @@ def _fact_evidence_text(match: EvidenceMatch) -> str: def _needs_full_context_conflict_scan(claim_text: str, mode: str) -> bool: if mode != "semantic": return False + if re.search( + r"\bpassages?\s+\d+(?:\s*(?:&|and|,)\s*\d+)+", + str(claim_text or ""), + flags=re.IGNORECASE, + ): + return True return bool( re.search( r"\blost\s+(?:her|his|their)\s+foot\s+in\s+(?:the\s+)?(?:bombing|blast|attack)\b", @@ -315,6 +326,10 @@ def _needs_full_context_conflict_scan(claim_text: str, mode: str) -> bool: def is_contradicted(claim_text: str, evidence_text: str, score: float, *, mode: str = "lexical") -> bool: + if _denied_existence_conflicts_with_lifecycle_claim(claim_text, evidence_text, mode=mode): + return True + if _opposed_predicate_conflict(claim_text, evidence_text, mode=mode): + return True if score < 0.50: return False if has_unnegated_exact_surface_match(claim_text, evidence_text): @@ -349,6 +364,102 @@ def is_contradicted(claim_text: str, evidence_text: str, score: float, *, mode: return False +def _opposed_predicate_conflict(claim_text: str, evidence_text: str, *, mode: str) -> bool: + """Catch a small set of explicit verbal antonyms with a shared subject/object. + + Token overlap makes pairs such as ``enables``/``disables`` look nearly + identical. Requiring substantial non-predicate overlap keeps this rule + narrow and avoids treating unrelated uses of an antonym as evidence. + """ + claim = str(claim_text or "") + evidence = str(evidence_text or "") + if _explicit_replacement_conflict(claim, evidence, mode=mode): + return True + predicate_pairs = ( + (r"\benabl(?:e|es|ed|ing)\b", r"\bdisabl(?:e|es|ed|ing)\b"), + (r"\bactivat(?:e|es|ed|ing)\b", r"\bdeactivat(?:e|es|ed|ing)\b"), + (r"\bpermit(?:s|ted|ting)?\b", r"\bprohibit(?:s|ed|ing)?\b"), + ) + opposed = any( + (re.search(left, claim, flags=re.IGNORECASE) and re.search(right, evidence, flags=re.IGNORECASE)) + or (re.search(right, claim, flags=re.IGNORECASE) and re.search(left, evidence, flags=re.IGNORECASE)) + for left, right in predicate_pairs + ) + return bool( + opposed + and not has_unnegated_exact_surface_match(claim, evidence) + and _core_overlap(claim, evidence, mode=mode) >= 0.50 + ) + + +def _explicit_replacement_conflict(claim_text: str, evidence_text: str, *, mode: str) -> bool: + """Detect guidance that explicitly replaces the object recommended by a claim. + + Mention-only matching otherwise mistakes ``use New instead of Old`` as + support for ``use Old``. The rule requires an action-oriented claim, an + explicit replacement construction, and shared topical context. + """ + claim = str(claim_text or "") + evidence = str(evidence_text or "") + if not re.search( + r"\b(?:use|uses|using|construct|constructs|call|calls|import|imports|target|targets)\b", + claim, + flags=re.IGNORECASE, + ): + return False + replaced: list[str] = [] + patterns = ( + r"\binstead\s+of\s+(?P[A-Za-z][A-Za-z0-9_.-]*)", + r"\brenamed\s+(?P[A-Za-z][A-Za-z0-9_.-]*)\s+to\s+[A-Za-z][A-Za-z0-9_.-]*", + r"\breplac(?:e|es|ed|ing)\s+(?P[A-Za-z][A-Za-z0-9_.-]*)\s+with\s+[A-Za-z][A-Za-z0-9_.-]*", + ) + for pattern in patterns: + replaced.extend(match.group("old") for match in re.finditer(pattern, evidence, flags=re.IGNORECASE)) + if not replaced: + return False + claim_tokens = { + token.lower().strip(".-") + for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]*", claim) + } + return bool( + any(value.lower().strip(".") in claim_tokens for value in replaced) + and _core_overlap(claim, evidence, mode=mode) >= 0.35 + ) + + +def _denied_existence_conflicts_with_lifecycle_claim( + claim_text: str, + evidence_text: str, + *, + mode: str, +) -> bool: + """Detect false-premise claims whose event presupposes an explicitly denied entity. + + Discontinuing, retiring, or cancelling a thing entails that it previously existed. + A current source saying that it was never offered/created/operated is therefore a + contradiction even when ordinary token overlap is below the general threshold. + """ + claim = str(claim_text or "") + evidence = str(evidence_text or "") + lifecycle_event = re.search( + r"\b(?:discontinued|retired|sunset|sunsetted|cancelled|canceled|terminated|" + r"shut\s+down|phased\s+out|withdrew|withdrawn|removed)\b", + claim, + flags=re.IGNORECASE, + ) + denied_existence = re.search( + r"\b(?:never|not)\s+(?:previously\s+|ever\s+)?" + r"(?:offered|created|launched|operated|provided|supported|sold|issued|maintained)\b", + evidence, + flags=re.IGNORECASE, + ) + return bool( + lifecycle_event + and denied_existence + and _core_overlap(claim, evidence, mode=mode) >= 0.30 + ) + + def _has_negation(text: str) -> bool: normalized_text = _neutralize_non_negating_phrases(str(text or "")) normalized_text = re.sub(r"\bcan(?:'|\u2019)?t\b", "cannot", normalized_text, flags=re.IGNORECASE) diff --git a/packages/contexttrace/pyproject.toml b/packages/contexttrace/pyproject.toml index 3c230d3..df73fb9 100644 --- a/packages/contexttrace/pyproject.toml +++ b/packages/contexttrace/pyproject.toml @@ -1,14 +1,14 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=83", "wheel"] build-backend = "setuptools.build_meta" [project] name = "contexttrace" -version = "1.0.0" +version = "1.1.0" description = "Local-first evidence-chain debugger for RAG and AI agent claim grounding, citation checks, root-cause diagnosis, and regression tests." readme = "README.md" -requires-python = ">=3.8" -license = { text = "MIT" } +requires-python = ">=3.10" +license = "MIT" authors = [ { name = "ContextTrace contributors" }, ] @@ -26,10 +26,7 @@ keywords = [ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -103,12 +100,23 @@ all = [ "onnxruntime>=1.17", ] test = [ + "jsonschema>=4.22", "pytest>=8.0", ] +quality = [ + "build>=1.2", + "mypy>=1.10", + "pip-audit>=2.7", + "pytest-cov>=5.0", + "ruff>=0.5", +] [tool.setuptools.packages.find] where = ["."] include = ["contexttrace*"] [tool.setuptools.package-data] -contexttrace = ["py.typed", "verify/*.json"] +contexttrace = ["py.typed", "verify/*.json", "verify/rulepacks/*.yaml", "schemas/*.json"] + +[tool.ruff.lint.per-file-ignores] +"contexttrace/verify/facts.py" = ["F601"] diff --git a/packages/contexttrace/tests/fixtures/trace-v1.0.json b/packages/contexttrace/tests/fixtures/trace-v1.0.json new file mode 100644 index 0000000..c83b54c --- /dev/null +++ b/packages/contexttrace/tests/fixtures/trace-v1.0.json @@ -0,0 +1,30 @@ +{ + "schema_version": "1.0", + "taxonomy_version": "1.0", + "verifier_version": "semantic_v1_calibrated", + "profile_id": "full_v1", + "query": "What is the documented refund window?", + "answer": "The documented refund window is 30 days.", + "contexts": [ + { + "id": "refund-policy-v1#window", + "text": "Customers may request a refund within 30 days of purchase.", + "metadata": { + "source": "refund-policy-v1", + "publication_window": "2025-Q1" + } + } + ], + "citations": [ + { + "claim": "The documented refund window is 30 days.", + "source_id": "refund-policy-v1#window", + "metadata": { + "citation_index": 1 + } + } + ], + "metadata": { + "fixture": "trace-v1.0" + } +} diff --git a/packages/contexttrace/tests/test_capture_endpoint.py b/packages/contexttrace/tests/test_capture_endpoint.py index 8938daa..e83a559 100644 --- a/packages/contexttrace/tests/test_capture_endpoint.py +++ b/packages/contexttrace/tests/test_capture_endpoint.py @@ -99,6 +99,7 @@ def test_capture_endpoint_cli_writes_trace_and_verification_report(tmp_path, cap finally: server.shutdown() thread.join(timeout=2) + server.server_close() output = capsys.readouterr().out assert exit_code == 0 diff --git a/packages/contexttrace/tests/test_claim_decomposition.py b/packages/contexttrace/tests/test_claim_decomposition.py new file mode 100644 index 0000000..3cf9779 --- /dev/null +++ b/packages/contexttrace/tests/test_claim_decomposition.py @@ -0,0 +1,17 @@ +from contexttrace.verify.claims import extract_claims + + +def test_extracts_numbered_list_as_atomic_claims(): + claims = extract_claims("1. Alpha launched in 2024. 2. Beta launched in 2025.") + + assert [claim.text for claim in claims] == [ + "Alpha launched in 2024.", + "Beta launched in 2025.", + ] + + +def test_splits_independent_semicolon_clauses_without_splitting_entity_list(): + claims = extract_claims("Alpha is current; Beta is deprecated.") + + assert [claim.text for claim in claims] == ["Alpha is current.", "Beta is deprecated."] + assert len(extract_claims("The supported regions are France, Spain, and Italy.")) == 1 diff --git a/packages/contexttrace/tests/test_contracts_privacy.py b/packages/contexttrace/tests/test_contracts_privacy.py new file mode 100644 index 0000000..09218a0 --- /dev/null +++ b/packages/contexttrace/tests/test_contracts_privacy.py @@ -0,0 +1,201 @@ +import json +import sqlite3 +import stat +from contextlib import closing + +from contexttrace import ( + ContextTrace, + PrivacyPolicy, + build_regression_case, + capture_rag_trace, + load_json_schema, +) +from contexttrace.diagnose import diagnose_payload +from contexttrace.repair import build_repair_plan +from contexttrace.verify.runner import VerificationLimits, verify_trace, verify_traces + + +PROVENANCE = {"schema_version", "taxonomy_version", "verifier_version", "profile_id"} + + +class ReverseCipher: + def encrypt(self, plaintext): + return plaintext[::-1] + + def decrypt(self, ciphertext): + return ciphertext[::-1] + + +def _trace_payload(): + return { + "query": "What is the policy?", + "answer": "Refunds are available within 30 days.", + "contexts": [{"id": "policy", "text": "Refunds are available within 30 days."}], + } + + +def test_public_artifacts_have_packaged_v1_contracts(tmp_path): + trace = capture_rag_trace( + query="What is the policy?", + answer="Refunds are available within 30 days.", + contexts=[{"id": "policy", "text": "Refunds are available within 30 days."}], + ) + verification = verify_trace(trace, mode="semantic") + diagnosis = diagnose_payload(trace.to_dict()) + regression = build_regression_case( + case_id="policy_1", + trace=trace.to_dict(), + expected={"status": "passed"}, + ) + trace_path = tmp_path / "trace.json" + trace_path.write_text(json.dumps(trace.to_dict()), encoding="utf-8") + repair = build_repair_plan(trace_path) + + artifacts = { + "TraceV1": trace.to_dict(), + "ClaimVerificationV1": verification, + "DiagnosisV1": diagnosis, + "RepairPlanV1": repair, + "RegressionCaseV1": regression, + } + for name, artifact in artifacts.items(): + schema = load_json_schema(name) + assert schema["title"] == name + assert PROVENANCE <= artifact.keys() + assert artifact["schema_version"] == "1.0" + assert artifact["verifier_version"] == "semantic_v1_calibrated" + + +def test_strict_privacy_covers_query_claim_metadata_and_agent_values(tmp_path): + ct = ContextTrace( + project="private", + storage_path=str(tmp_path / "trace.db"), + privacy="strict", + ) + with ct.trace(query="patient@example.com", metadata={"patient": "Ada"}) as trace: + trace.log_retrieval( + [{"chunk_id": "secret-id", "content": "Patient diagnosis", "source": "/private/a"}] + ) + trace.log_context(chunk_ids=["secret-id"]) + trace.log_answer("Sensitive answer", metadata={"ticket": "abc"}) + trace.log_citations([{"claim": "Sensitive claim", "source_chunk_id": "secret-id"}]) + trace.log_tool_result( + "patient_lookup", + input_json={"email": "patient@example.com"}, + output_json={"diagnosis": "sensitive"}, + ) + + fetched = trace.fetch() + assert fetched["query"] == "[query redacted]" + assert fetched["metadata"] == {} + assert fetched["chunks"][0]["content"] == "[chunk text redacted]" + assert fetched["chunks"][0]["chunk_id"].startswith("id_sha256_") + assert fetched["chunks"][0]["selected"] is True + assert fetched["answer"]["answer"] == "[answer text redacted]" + assert fetched["citation_checks"][0]["claim"] == "[citation claim redacted]" + assert fetched["agent_events"][0]["input_json"]["email"] == "[tool input redacted]" + assert fetched["agent_events"][0]["output_json"]["diagnosis"] == "[tool output redacted]" + + +def test_regex_redaction_cipher_ttl_and_secure_permissions(tmp_path): + database = tmp_path / "private" / "trace.db" + policy = PrivacyPolicy( + redaction_patterns=(r"[\w.+-]+@[\w.-]+",), + cipher=ReverseCipher(), + ) + ct = ContextTrace( + project="private", + storage_path=str(database), + privacy_policy=policy, + retention_days=0, + ) + with ct.trace(query="Email alice@example.com") as first: + first.log_retrieval([{"chunk_id": "c1", "content": "Contact alice@example.com"}]) + first.log_answer("Sent to alice@example.com") + assert first.fetch()["query"] == "Email [redacted]" + assert first.fetch()["answer"]["answer"] == "Sent to [redacted]" + + with closing(sqlite3.connect(database)) as db: + raw_query = db.execute("SELECT query FROM traces").fetchone()[0] + assert raw_query.startswith("enc:") + assert "alice@example.com" not in raw_query + + with ct.trace(query="Second trace") as second: + second.log_answer("Second answer") + assert [item["id"] for item in ct.list_traces()] == [second.trace_id] + assert stat.S_IMODE(database.stat().st_mode) == 0o600 + assert stat.S_IMODE(database.parent.stat().st_mode) == 0o700 + + +def test_batch_verification_reports_explicit_truncation(): + trace = capture_rag_trace( + query="q", + answer="A supported answer.", + contexts=[ + {"id": "a", "text": "A supported answer."}, + {"id": "b", "text": "Additional evidence."}, + ], + ) + results = verify_traces( + [trace, trace], + mode="semantic", + limits=VerificationLimits(max_contexts=1, max_context_chars=8), + ) + assert len(results) == 2 + assert results[0]["truncation"]["applied"] is True + assert results[0]["truncation"]["contexts_used"] == 1 + assert results[0]["truncation"]["context_chars_used"] == 8 + assert results[0]["truncation"]["contexts_truncated"] is True + + +def test_nested_redaction_recurses_and_honors_metadata_allowlist(): + policy = PrivacyPolicy( + metadata_allowlist=frozenset({"safe"}), + redaction_patterns=(r"[\w.+-]+@[\w.-]+", r"token-[a-z0-9]+"), + custom_redactors=(lambda text: text.replace("Ada", "[name]"),), + ) + sanitized = policy.sanitize( + "/v1/traces/t1/agent-events", + { + "name": "lookup", + "input_json": { + "users": [ + {"email": "ada@example.com", "credentials": ("token-secret", "Ada")}, + ] + }, + "output_json": {"nested": {"contact": "ada@example.com"}}, + "metadata": { + "safe": {"owner": "Ada", "email": "ada@example.com"}, + "secret": "token-private", + }, + }, + ) + + assert sanitized["input_json"]["users"][0] == { + "email": "[redacted]", + "credentials": ["[redacted]", "[name]"], + } + assert sanitized["output_json"]["nested"]["contact"] == "[redacted]" + assert sanitized["metadata"] == { + "safe": {"owner": "[name]", "email": "[redacted]"}, + } + + +def test_verification_limits_oversized_answer_and_context_without_mutating_input(): + trace = capture_rag_trace( + query="q", + answer="answer-" * 100, + contexts=[{"id": "large", "text": "context-" * 100}], + ) + original = trace.to_dict() + + result = verify_trace( + trace, + mode="semantic", + limits=VerificationLimits(max_answer_chars=17, max_context_chars=19), + ) + + assert result["truncation"]["applied"] is True + assert result["truncation"]["answer_chars_used"] == 17 + assert result["truncation"]["context_chars_used"] == 19 + assert trace.to_dict() == original diff --git a/packages/contexttrace/tests/test_contradiction_detection.py b/packages/contexttrace/tests/test_contradiction_detection.py new file mode 100644 index 0000000..2250aa4 --- /dev/null +++ b/packages/contexttrace/tests/test_contradiction_detection.py @@ -0,0 +1,60 @@ +from contexttrace.verify.facts import compare_facts +from contexttrace.verify.verdicts import is_contradicted + + +def test_date_mismatch_is_a_structured_conflict(): + result = compare_facts( + "The release occurred on June 3, 2024.", + "The release occurred on June 4, 2024.", + mode="semantic", + ) + + assert any(fact.type == "date" for fact in result.conflicting_fact_details) + + +def test_numeric_and_negation_conflicts_are_detected(): + assert is_contradicted("The limit is 30 days.", "The limit is 14 days.", 0.9, mode="semantic") + assert is_contradicted("The feature is enabled.", "The feature is not enabled.", 0.9, mode="semantic") + + +def test_explicit_opposed_predicates_are_detected(): + assert is_contradicted( + "Atlas enables public access by default.", + "Atlas disables public access by default.", + 0.77, + mode="semantic", + ) + assert not is_contradicted( + "Atlas enables public access by default.", + "Borealis disables guest checkout.", + 0.77, + mode="semantic", + ) + + +def test_multi_passage_attribution_requires_support_from_every_named_passage(): + result = compare_facts( + "Constipation is a symptom (passage 2 & 3).", + "passage 2: The cause is unknown. passage 3: Symptoms include constipation.", + mode="semantic", + ) + + assert any(fact.type == "attribution" for fact in result.conflicting_fact_details) + + +def test_explicit_denial_of_existence_rejects_lifecycle_false_premise(): + assert is_contradicted( + "Orion discontinued its public token because adoption fell.", + "Orion has never offered a public token.", + 0.36, + mode="semantic", + ) + + +def test_low_overlap_negation_without_lifecycle_entailment_is_not_a_conflict(): + assert not is_contradicted( + "Orion publishes an annual security report.", + "Orion has never offered a public token.", + 0.36, + mode="semantic", + ) diff --git a/packages/contexttrace/tests/test_endpoint_eval_local.py b/packages/contexttrace/tests/test_endpoint_eval_local.py index 39e59f6..8f4d652 100644 --- a/packages/contexttrace/tests/test_endpoint_eval_local.py +++ b/packages/contexttrace/tests/test_endpoint_eval_local.py @@ -59,6 +59,7 @@ def test_endpoint_eval_creates_local_traces_and_report(tmp_path): finally: server.shutdown() thread.join(timeout=2) + server.server_close() assert result.questions_tested == 1 assert result.failure_rate == 0.0 diff --git a/packages/contexttrace/tests/test_evidence_span_selection.py b/packages/contexttrace/tests/test_evidence_span_selection.py new file mode 100644 index 0000000..5761028 --- /dev/null +++ b/packages/contexttrace/tests/test_evidence_span_selection.py @@ -0,0 +1,18 @@ +from contexttrace.verify.schema import TraceContext +from contexttrace.verify.spans import split_context_spans + + +def test_sentence_spans_are_minimal_and_preserve_offsets(): + context = TraceContext(id="ctx", text="Alpha is supported. Beta is contradicted.", metadata={}) + spans = split_context_spans(context) + + assert [span.text for span in spans] == ["Alpha is supported.", "Beta is contradicted."] + assert [(span.start_char, span.end_char) for span in spans] == [(0, 19), (20, 41)] + + +def test_passage_context_exposes_block_and_sentence_spans_for_multispan_support(): + context = TraceContext(id="ctx", text="Passage 1: Alpha is supported. Beta is supported.", metadata={}) + texts = [span.text for span in split_context_spans(context)] + + assert "Alpha is supported." in texts + assert "Beta is supported." in texts diff --git a/packages/contexttrace/tests/test_fastapi_middleware.py b/packages/contexttrace/tests/test_fastapi_middleware.py index 9363fc3..f03716d 100644 --- a/packages/contexttrace/tests/test_fastapi_middleware.py +++ b/packages/contexttrace/tests/test_fastapi_middleware.py @@ -164,3 +164,213 @@ async def send(message): assert transport.calls[0][2]["metadata"]["custom"] is True assert transport.calls[2][2]["answer"] == "Custom answer." + +def test_fastapi_middleware_tees_streaming_response_before_logging(): + order = [] + + class OrderedTransport(FakeTransport): + def post(self, path, payload=None): + order.append(("log", path)) + return super().post(path, payload) + + async def streaming_app(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"application/json")]}) + await send({"type": "http.response.body", "body": b'{"answer":"streamed ', "more_body": True}) + await send({"type": "http.response.body", "body": b'answer"}', "more_body": False}) + + transport = OrderedTransport() + middleware = ContextTraceFastAPIMiddleware(streaming_app, client=ContextTrace(transport=transport)) + sent = [] + + async def receive(): + return {"type": "http.request", "body": b'{"query":"q"}', "more_body": False} + + async def send(message): + order.append(("send", message["type"])) + sent.append(message) + + asyncio.run(middleware({"type": "http", "method": "POST", "path": "/query", "headers": []}, receive, send)) + + assert [item["body"] for item in sent if item["type"] == "http.response.body"] == [ + b'{"answer":"streamed ', + b'answer"}', + ] + assert max(index for index, item in enumerate(order) if item[0] == "send") < min( + index for index, item in enumerate(order) if item[0] == "log" + ) + assert transport.calls[-1][2]["answer"] == "streamed answer" + + +def test_fastapi_middleware_bounds_capture_and_skips_sse_and_disallowed_routes(): + transport = FakeTransport() + + async def sse_app(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/event-stream")]}) + await send({"type": "http.response.body", "body": b"data: secret\n\n", "more_body": False}) + + middleware = ContextTraceFastAPIMiddleware( + sse_app, + client=ContextTrace(transport=transport), + max_capture_bytes=8, + route_allowlist=("/events",), + ) + + async def receive(): + return {"type": "http.request", "body": b'{"query":"a long secret"}', "more_body": False} + + async def send(message): + return None + + asyncio.run(middleware({"type": "http", "method": "POST", "path": "/events", "headers": []}, receive, send)) + assert middleware.metrics["request_capture_truncated"] == 1 + assert middleware.metrics["streaming_responses_skipped"] == 1 + assert not any(call[1].endswith("/answer") for call in transport.calls) + + before = len(transport.calls) + asyncio.run(middleware({"type": "http", "method": "POST", "path": "/health", "headers": []}, receive, send)) + assert len(transport.calls) == before + + +def test_fastapi_middleware_reassembles_json_across_adversarial_chunk_boundaries(): + transport = FakeTransport() + + async def chunked_app(scope, receive, send): + request_parts = [] + while True: + message = await receive() + request_parts.append(message.get("body", b"")) + if not message.get("more_body"): + break + assert json.loads(b"".join(request_parts))["query"] == "split request" + response = json.dumps({"answer": "split response"}).encode("utf-8") + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"application/json")]}) + for index, byte in enumerate(response): + await send( + { + "type": "http.response.body", + "body": bytes([byte]), + "more_body": index < len(response) - 1, + } + ) + + request = b'{"query":"split request"}' + messages = [ + {"type": "http.request", "body": request[:1], "more_body": True}, + {"type": "http.request", "body": request[1:9], "more_body": True}, + {"type": "http.request", "body": request[9:], "more_body": False}, + ] + + async def receive(): + return messages.pop(0) + + async def send(message): + return None + + middleware = ContextTraceFastAPIMiddleware(chunked_app, client=ContextTrace(transport=transport)) + asyncio.run(middleware({"type": "http", "method": "POST", "path": "/query", "headers": []}, receive, send)) + + assert transport.calls[0][2]["query"] == "split request" + assert next(payload for _, path, payload in transport.calls if path.endswith("/answer"))["answer"] == "split response" + assert middleware.metrics["request_capture_truncated"] == 0 + assert middleware.metrics["response_capture_truncated"] == 0 + + +def test_fastapi_middleware_drops_background_log_when_queue_is_saturated(): + transport = FakeTransport() + + async def minimal_app(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"application/json")]}) + await send({"type": "http.response.body", "body": b'{"answer":"ok"}', "more_body": False}) + + async def scenario(): + release = asyncio.Event() + extractor_started = asyncio.Event() + + async def blocked_request_extractor(request): + extractor_started.set() + await release.wait() + return {"query": request["json"]["query"]} + + middleware = ContextTraceFastAPIMiddleware( + minimal_app, + client=ContextTrace(transport=transport), + request_extractor=blocked_request_extractor, + background_logging=True, + max_pending_logs=1, + ) + + async def invoke(query): + used = False + + async def receive(): + nonlocal used + assert not used + used = True + return { + "type": "http.request", + "body": json.dumps({"query": query}).encode("utf-8"), + "more_body": False, + } + + async def send(message): + return None + + await middleware( + {"type": "http", "method": "POST", "path": "/query", "headers": []}, + receive, + send, + ) + + await invoke("first") + await extractor_started.wait() + await invoke("second") + assert middleware.metrics["logging_dropped"] == 1 + assert len(middleware._pending_logs) == 1 + release.set() + await middleware.drain() + return middleware + + middleware = asyncio.run(scenario()) + assert middleware.metrics["traces_attempted"] == 2 + assert middleware.metrics["logging_failures"] == 0 + assert len([call for call in transport.calls if call[1] == "/v1/traces/start"]) == 1 + + +def test_fastapi_middleware_truncates_oversized_request_and_response_independently(): + transport = FakeTransport() + + async def oversized_app(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"application/json")]}) + await send( + { + "type": "http.response.body", + "body": json.dumps({"answer": "x" * 1024}).encode("utf-8"), + "more_body": False, + } + ) + + async def receive(): + return { + "type": "http.request", + "body": json.dumps({"query": "y" * 1024}).encode("utf-8"), + "more_body": False, + } + + async def send(message): + return None + + middleware = ContextTraceFastAPIMiddleware( + oversized_app, + client=ContextTrace(transport=transport), + max_capture_bytes=32, + ) + asyncio.run(middleware({"type": "http", "method": "POST", "path": "/oversized", "headers": []}, receive, send)) + + assert middleware.metrics["request_capture_truncated"] == 1 + assert middleware.metrics["response_capture_truncated"] == 1 + assert transport.calls[0][2]["query"] == "/oversized" + assert not any(path.endswith("/answer") for _, path, _ in transport.calls) diff --git a/packages/contexttrace/tests/test_frozen_verifier.py b/packages/contexttrace/tests/test_frozen_verifier.py new file mode 100644 index 0000000..0f882f8 --- /dev/null +++ b/packages/contexttrace/tests/test_frozen_verifier.py @@ -0,0 +1,17 @@ +import hashlib +from pathlib import Path + +from contexttrace.contracts import VERIFIER_VERSION + + +FROZEN_FACTS_SHA256 = "4fa507db2126423c8d0787811e78d0d6ffecf5d7603828b6a6d9b23ca8207bc4" + + +def test_semantic_v1_calibrated_implementation_is_frozen(): + implementation = Path(__file__).parents[1] / "contexttrace" / "verify" / "facts.py" + digest = hashlib.sha256(implementation.read_bytes()).hexdigest() + assert VERIFIER_VERSION == "semantic_v1_calibrated" + assert digest == FROZEN_FACTS_SHA256, ( + "The calibrated verifier changed. Do not update this hash in response to calibration-set errors; " + "create a new verifier version and preregister an untouched test manifest." + ) diff --git a/packages/contexttrace/tests/test_integration_concurrency.py b/packages/contexttrace/tests/test_integration_concurrency.py new file mode 100644 index 0000000..02e5bf7 --- /dev/null +++ b/packages/contexttrace/tests/test_integration_concurrency.py @@ -0,0 +1,100 @@ +import asyncio + +from contexttrace import ( + ContextTrace, + ContextTraceCallbackHandler, + ContextTraceLangGraphTracer, + ContextTraceLlamaIndexCallbackHandler, +) + + +class RunTransport: + def __init__(self): + self.calls = [] + self.count = 0 + + def post(self, path, payload=None): + payload = payload or {} + if path == "/v1/traces/start": + self.count += 1 + trace_id = "trace_%s" % self.count + self.calls.append((path, payload, trace_id)) + return {"trace_id": trace_id, "project_id": "project"} + self.calls.append((path, payload, None)) + return {"accepted": 1} + + def get(self, path): + return {} + + +def _answer_routes(transport): + return { + payload["answer"]: path + for path, payload, _ in transport.calls + if path.endswith("/answer") + } + + +def test_langchain_handler_separates_interleaved_run_ids(): + transport = RunTransport() + handler = ContextTraceCallbackHandler(client=ContextTrace(transport=transport)) + + async def run(run_id, query, answer): + handler.on_chain_start({"name": "qa"}, {"query": query}, run_id=run_id) + await asyncio.sleep(0) + handler.on_chain_end({"answer": answer}, run_id=run_id) + + async def main(): + await asyncio.gather(run("a", "query a", "answer a"), run("b", "query b", "answer b")) + + asyncio.run(main()) + routes = _answer_routes(transport) + assert routes["answer a"] != routes["answer b"] + + +def test_llamaindex_handler_separates_interleaved_event_ids(): + transport = RunTransport() + handler = ContextTraceLlamaIndexCallbackHandler(client=ContextTrace(transport=transport)) + + async def run(event_id, query, answer): + handler.on_event_start("query", {"query_str": query}, event_id=event_id, parent_id="shared-root") + await asyncio.sleep(0) + handler.on_event_end("query", {"response": answer}, event_id=event_id) + + async def main(): + await asyncio.gather(run("a", "query a", "answer a"), run("b", "query b", "answer b")) + + asyncio.run(main()) + routes = _answer_routes(transport) + assert routes["answer a"] != routes["answer b"] + + +def test_langgraph_tracer_separates_explicit_run_ids(): + transport = RunTransport() + tracer = ContextTraceLangGraphTracer(client=ContextTrace(transport=transport)) + tracer.start_trace("query a", run_id="a") + tracer.start_trace("query b", run_id="b") + tracer.end_trace(answer="answer a", run_id="a") + tracer.end_trace(answer="answer b", run_id="b") + routes = _answer_routes(transport) + assert routes["answer a"] != routes["answer b"] + + +def test_langchain_handler_isolates_many_adversarially_interleaved_runs(): + transport = RunTransport() + handler = ContextTraceCallbackHandler(client=ContextTrace(transport=transport)) + run_count = 32 + + async def run(index): + run_id = "run-%02d" % index + handler.on_chain_start({"name": "qa"}, {"query": "query %02d" % index}, run_id=run_id) + await asyncio.sleep(0 if index % 2 else 0.001) + handler.on_chain_end({"answer": "answer %02d" % index}, run_id=run_id) + + async def main(): + await asyncio.gather(*(run(index) for index in range(run_count))) + + asyncio.run(main()) + routes = _answer_routes(transport) + assert len(routes) == run_count + assert len(set(routes.values())) == run_count diff --git a/packages/contexttrace/tests/test_package_smoke.py b/packages/contexttrace/tests/test_package_smoke.py index 73b915c..2f9dd04 100644 --- a/packages/contexttrace/tests/test_package_smoke.py +++ b/packages/contexttrace/tests/test_package_smoke.py @@ -5,7 +5,7 @@ def test_package_exports_core_public_api(): - assert contexttrace.__version__ == "1.0.0" + assert contexttrace.__version__ == "1.1.0" assert ContextTrace is not None assert AsyncContextTrace is not None assert ReliabilityScorer().score( diff --git a/packages/contexttrace/tests/test_schema_compatibility.py b/packages/contexttrace/tests/test_schema_compatibility.py new file mode 100644 index 0000000..e4c38f7 --- /dev/null +++ b/packages/contexttrace/tests/test_schema_compatibility.py @@ -0,0 +1,38 @@ +import json +from pathlib import Path + +from jsonschema import Draft202012Validator + +from contexttrace import load_json_schema +from contexttrace.verify.schema import load_trace, load_trace_file + + +FIXTURE = Path(__file__).parent / "fixtures" / "trace-v1.0.json" + + +def test_trace_v1_golden_round_trip_is_stable(): + golden = json.loads(FIXTURE.read_text(encoding="utf-8")) + + loaded = load_trace_file(FIXTURE) + + assert loaded.to_dict() == golden + + +def test_trace_v1_golden_conforms_to_packaged_schema(): + golden = json.loads(FIXTURE.read_text(encoding="utf-8")) + schema = load_json_schema("TraceV1") + + Draft202012Validator.check_schema(schema) + Draft202012Validator(schema).validate(golden) + + +def test_trace_v1_loader_preserves_declared_provenance(): + golden = json.loads(FIXTURE.read_text(encoding="utf-8")) + golden["taxonomy_version"] = "1.0-fixture" + golden["verifier_version"] = "semantic_v1_fixture" + golden["profile_id"] = "compatibility_fixture" + + emitted = load_trace(golden, source="compatibility fixture").to_dict() + assert emitted["taxonomy_version"] == "1.0-fixture" + assert emitted["verifier_version"] == "semantic_v1_fixture" + assert emitted["profile_id"] == "compatibility_fixture" diff --git a/packages/contexttrace/tests/test_semantic_normalization.py b/packages/contexttrace/tests/test_semantic_normalization.py new file mode 100644 index 0000000..cd38be2 --- /dev/null +++ b/packages/contexttrace/tests/test_semantic_normalization.py @@ -0,0 +1,14 @@ +from contexttrace.verify.semantic_normalization import extract_normalized_dates, normalize_semantic_text + + +def test_normalizes_entities_numbers_comparatives_and_negation(): + normalized = normalize_semantic_text("The U.S. can't admit fewer than five entities.") + + assert "united states" in normalized + assert "not" in normalized + assert "less than 5" in normalized + + +def test_normalizes_written_date_without_changing_invalid_date(): + assert extract_normalized_dates("Released June 3, 2024") == {"2024-06-03"} + assert extract_normalized_dates("Released February 31, 2024") == set() diff --git a/packages/contexttrace/tests/test_suite.py b/packages/contexttrace/tests/test_suite.py index 4a32769..e298191 100644 --- a/packages/contexttrace/tests/test_suite.py +++ b/packages/contexttrace/tests/test_suite.py @@ -121,6 +121,7 @@ def test_run_suite_passes_when_saved_failure_is_fixed(tmp_path): finally: server.shutdown() thread.join(timeout=2) + server.server_close() assert result["summary"]["status"] == "passed" assert result["summary"]["passed"] == 1 @@ -140,6 +141,7 @@ def test_run_suite_fails_when_failure_still_reproduces(tmp_path): finally: server.shutdown() thread.join(timeout=2) + server.server_close() assert result["summary"]["status"] == "failed" assert result["summary"]["failed"] == 1 @@ -159,6 +161,7 @@ def test_suite_report_generation(tmp_path): finally: server.shutdown() thread.join(timeout=2) + server.server_close() written = SuiteReportGenerator().generate(result, path=str(report_path)) @@ -250,6 +253,7 @@ def test_suite_cli_create_run_and_report(tmp_path, capsys): finally: server.shutdown() thread.join(timeout=2) + server.server_close() output = capsys.readouterr().out assert exit_code == 0 diff --git a/packages/contexttrace/tests/test_verify.py b/packages/contexttrace/tests/test_verify.py index fe0277e..5fefa8a 100644 --- a/packages/contexttrace/tests/test_verify.py +++ b/packages/contexttrace/tests/test_verify.py @@ -677,6 +677,94 @@ def test_canonical_supported_source_wins_over_lower_authority_conflict(): assert claim["source_assessment"]["conflicting_sources"][0]["context_id"] == "policy_old" +def test_direct_stronger_conflict_overrides_stale_support_and_forces_abstention(): + result = verify_trace( + RAGTrace( + query="Is public access enabled by default?", + answer="Orion enables public access by default.", + contexts=[ + TraceContext( + id="old_guide", + text="Orion enables public access by default.", + metadata={"stale": True, "source_authority": "secondary", "source_group": "orion"}, + ), + TraceContext( + id="current_control", + text="Orion disables public access by default.", + metadata={"canonical": True, "source_authority": "official", "source_group": "orion"}, + ), + ], + ), + mode="semantic", + ) + + assert result["claims"][0]["source_status"] == "grounded_but_conflicted" + assert result["claims"][0]["root_cause"]["label"] == "conflicting_contexts" + assert result["abstention"]["should_abstain"] is True + assert "source_conflict" in result["summary"]["failure_types"] + assert "contradicted_answer" in result["summary"]["failure_types"] + + +def test_current_query_preserves_stale_classification_when_versions_conflict(): + result = verify_trace( + RAGTrace( + query="What is Orion's current cancellation window?", + answer="Orion allows cancellation within 30 days.", + contexts=[ + TraceContext( + id="old_policy", + text="Orion allows cancellation within 30 days.", + metadata={"stale": True, "source_authority": "secondary", "source_group": "orion"}, + ), + TraceContext( + id="current_policy", + text="Orion allows cancellation within 14 days.", + metadata={"canonical": True, "source_authority": "official", "source_group": "orion"}, + ), + ], + ), + mode="semantic", + ) + + assert result["claims"][0]["source_status"] == "grounded_but_stale" + assert result["claims"][0]["root_cause"]["label"] == "stale_context" + assert result["abstention"]["should_abstain"] is True + + +def test_missing_fact_in_current_canonical_context_is_unverifiable_corpus_gap(): + result = verify_trace( + RAGTrace( + query="What was Orion's private incident count?", + answer="Orion recorded 42 private incidents.", + contexts=[ + TraceContext( + id="operations_summary", + text="Orion publishes regional uptime summaries.", + metadata={"canonical": True, "freshness": "current", "source_authority": "official"}, + ) + ], + ), + mode="semantic", + ) + + assert result["claims"][0]["verdict"] == "unverifiable" + assert result["claims"][0]["root_cause"]["label"] == "corpus_gap" + assert result["abstention"]["should_abstain"] is True + + +def test_missing_fact_without_explicit_canonical_current_metadata_stays_unsupported(): + result = verify_trace( + RAGTrace( + query="What was Orion's private incident count?", + answer="Orion recorded 42 private incidents.", + contexts=[TraceContext(id="operations_summary", text="Orion publishes regional uptime summaries.")], + ), + mode="semantic", + ) + + assert result["claims"][0]["verdict"] == "unsupported" + + def test_low_authority_supported_source_is_flagged(): result = verify_trace( RAGTrace( diff --git a/paper/references.bib b/paper/references.bib index edd98f2..01eec8e 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -1,6 +1,6 @@ @inproceedings{lewis2020rag, title = {Retrieval-Augmented Generation for Knowledge-Intensive {NLP} Tasks}, - author = {Lewis, Patrick and Perez, Ethan and Piktus, Aleksandra and Petroni, Fabio and Karpukhin, Vladimir and Goyal, Naman and K{"u}ttler, Heinrich and Lewis, Mike and Yih, Wen-tau and Rockt{"a}schel, Tim and Riedel, Sebastian and Kiela, Douwe}, + author = {Lewis, Patrick and Perez, Ethan and Piktus, Aleksandra and Petroni, Fabio and Karpukhin, Vladimir and Goyal, Naman and K{\"u}ttler, Heinrich and Lewis, Mike and Yih, Wen-tau and Rockt{\"a}schel, Tim and Riedel, Sebastian and Kiela, Douwe}, booktitle = {Advances in Neural Information Processing Systems 33}, year = {2020}, url = {https://proceedings.neurips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html} @@ -93,9 +93,45 @@ @misc{opentelemetry2026 } @misc{openinference2026, - title = {{OpenInference}: Semantic Conventions for AI Observability}, + title = {{OpenInference}: Semantic Conventions for {AI} Observability}, author = {{Arize AI}}, year = {2026}, howpublished = {Open-source specification}, url = {https://github.com/Arize-ai/openinference} } + +@misc{cohen2025ragxplain, + title = {{RAGXplain}: From Explainable Evaluation to Actionable Guidance of {RAG} Pipelines}, + author = {Cohen, Dvir and Burg, Lin and Barkan, Gilad}, + year = {2025}, + eprint = {2505.13538}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2505.13538} +} + +@misc{cheng2025ragtrace, + title = {{RAGTrace}: Understanding and Refining Retrieval-Generation Dynamics in Retrieval-Augmented Generation}, + author = {Cheng, Sizhe and Li, Jiaping and Wang, Huanchen and Ma, Yuxin}, + year = {2025}, + eprint = {2508.06056}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2508.06056} +} + +@misc{deshpande2025trail, + title = {{TRAIL}: Trace Reasoning and Agentic Issue Localization}, + author = {Deshpande, Darshan and Gangal, Varun and Mehta, Hersh and Krishnan, Jitin and Kannappan, Anand and Qian, Rebecca}, + year = {2025}, + eprint = {2505.08638}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2505.08638} +} + +@misc{narita2026enterprise, + title = {Overcoming the ``Impracticality'' of {RAG}: Proposing a Real-World Benchmark and Multi-Dimensional Diagnostic Framework}, + author = {Narita, Kenichirou and Peng, Siqi and Fukui, Taku and Yamada, Moyuru and Munakata, Satoshi and Takahashi, Satoru}, + year = {2026}, + eprint = {2604.02640}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2604.02640} +} diff --git a/paper/sections/02_related_work.tex b/paper/sections/02_related_work.tex index 67975de..258aa73 100644 --- a/paper/sections/02_related_work.tex +++ b/paper/sections/02_related_work.tex @@ -11,12 +11,28 @@ \section{Related Work} \paragraph{Diagnostic benchmarks.} RAGChecker introduces fine-grained retriever and generator diagnostics -\citep{ru2024ragchecker}. RAGTruth supplies word-level hallucination annotations +\citep{ru2024ragchecker}, the closest existing analogue to our contract, but +its diagnostics characterize retrieval and generation quality and do not model +source condition -- staleness, authority, or canonical-source status -- so a +RAGChecker-clean case can still cite a source that is stale or superseded. +RAGTruth supplies word-level hallucination annotations across RAG tasks \citep{niu2024ragtruth}, while CRAG evaluates factual and temporally dynamic retrieval settings \citep{yang2024crag}. ContextTrace uses RAGTruth as external calibration evidence and CRAG as a secondary track; it -adds an explicit serialized chain from claims to repairs. The current external -annotations remain review-pending. +adds an explicit serialized chain from claims to repairs. + +Recent systems overlap more directly with actionable diagnosis. RAGXplain uses +LLM reasoning to translate metric assessments into explanations and repair +recommendations \citep{cohen2025ragxplain}. RAGTrace is an interactive system +for exploring retrieval--generation dynamics and locating failures +\citep{cheng2025ragtrace}. A 2026 enterprise benchmark proposes a +multi-dimensional taxonomy spanning reasoning, retrieval difficulty, source +structure, and operational explainability \citep{narita2026enterprise}. +ContextTrace differs by emitting deterministic, replayable, local +claim--evidence--source-condition records and CI regression targets, rather than +LLM-generated recommendations or primarily interactive exploration. These are +complementary system designs; our experiments do not establish superior human +actionability. \paragraph{Model judges and observability.} LLM judges can approximate human preferences \citep{zheng2023judge}, but order, @@ -26,3 +42,9 @@ \section{Related Work} \citep{opentelemetry2026}, and OpenInference extends semantic conventions to AI workloads \citep{openinference2026}. ContextTrace accepts such trace-like records but targets semantic evidence diagnosis rather than telemetry transport. +TRAIL further shows that issue localization in agent workflows is a distinct +and difficult task, providing 148 human-annotated OpenTelemetry-structured +traces and a fine-grained error taxonomy \citep{deshpande2025trail}. We treat +TRAIL as the appropriate external transfer benchmark for future agent-trace +claims; the present evaluation is predominantly RAG and does not establish +agent-trace generalization. diff --git a/release/v1.1.0.md b/release/v1.1.0.md new file mode 100644 index 0000000..f5c2959 --- /dev/null +++ b/release/v1.1.0.md @@ -0,0 +1,15 @@ +# ContextTrace v1.1.0 + +ContextTrace 1.1.0 freezes the calibrated semantic verifier as +`semantic_v1_calibrated` and establishes the contracts needed for a genuinely +untouched evaluation track. + +This release adds versioned public artifact schemas, strict privacy controls, +bounded and streaming-safe integrations, concurrent run isolation, explicit +verification limits, golden v1 trace compatibility coverage, and adversarial +release tests. It also raises the coverage gate to 80% and expands Python, +optional-integration, dependency-audit, and cross-platform wheel checks. + +The exact release artifacts were first published and smoke-tested as +`1.1.0rc1` on TestPyPI. Existing RAGTruth, Diag-150, and Naturalistic Eval v2 +results remain calibration evidence and are not independent test results. diff --git a/release/v1.1.0rc1.md b/release/v1.1.0rc1.md new file mode 100644 index 0000000..be2394d --- /dev/null +++ b/release/v1.1.0rc1.md @@ -0,0 +1,14 @@ +# ContextTrace v1.1.0rc1 + +This release candidate freezes the existing semantic verifier as +`semantic_v1_calibrated` and prepares ContextTrace for a genuinely untouched +evaluation track. + +It adds versioned public artifact schemas, strict privacy controls, bounded and +streaming-safe integrations, concurrent run isolation, explicit verification +limits, and expanded release quality gates. Existing RAGTruth, Diag-150, and +Naturalistic Eval v2 results are calibration evidence and must not be presented +as independent test results. + +Publish this candidate to TestPyPI and complete the clean-install smoke test +before changing the package version to final `1.1.0`.