Freeze calibrated verifier and prepare Unseen-v1 evaluation

.github/ISSUE_TEMPLATE/acquire-contexttrace-unseen-v1.md

@@ -0,0 +1,40 @@
+---
+name: Acquire ContextTrace-Unseen-v1
+about: Collect and freeze source/domain/time-disjoint natural RAG traces
+title: "Acquire and freeze ContextTrace-Unseen-v1 with source/domain/time-disjoint natural RAG traces"
+labels: research, data
+assignees: ""
+---
+
+## Objective
+
+Acquire and publish the unlabeled ContextTrace-Unseen-v1 manifest before any
+`semantic_core_v2` implementation or evaluation.
+
+## Deliverables
+
+- [ ] 300--500 Natural OOD traces spanning software/product documentation,
+  policy/regulatory documents, and support/operational knowledge bases.
+- [ ] Approximately 100 temporal/source-condition traces from versioned or
+  authority-contrasting document pairs.
+- [ ] BM25, vector, and hybrid retrieval; multiple chunking/reranking settings;
+  at least two pinned generator models; clean and naturally failing answers.
+- [ ] Immutable source snapshots, hashes, canonical URLs, source families,
+  domains, and publication windows.
+- [ ] Leakage audit proving separation from all calibration sources.
+- [ ] Published unlabeled IDs/configuration manifest and SHA-256 before scoring.
+- [ ] Independent annotation and sealed adjudication records following
+  `benchmarks/contexttrace_unseen_v1/ANNOTATION_MANUAL.md`.
+
+## Exclusions
+
+Do not run ContextTrace on candidates before the manifest is frozen. Do not
+manually author failures, inspect sealed labels during model development, begin
+TRAIL transfer, recruit human-study participants, or include 1.2 performance and
+dashboard work in this issue.
+
+## Exit condition
+
+The issue closes when the public unlabeled manifest/hash and leakage report are
+available and the sealed-label custodian confirms that annotations are ready for
+one-time scoring after the preregistration lock.

.github/workflows/ci.yml

@@ -8,59 +8,97 @@ on:
 
 jobs:
   python:
-    name: Python tests
+    name: Full tests and coverage
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-
+          cache: pip
       - name: Install packages
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip "setuptools>=83"
           python -m pip install -e "./apps/api[test]"
-          python -m pip install -e "./packages/contexttrace[test]"
-
-      - name: Run tests
-        run: python -m pytest -q
-
-      - name: Run ContextTrace-Bench
-        run: |
-          python benchmarks/contexttrace_bench/run_contexttrace.py \
-            --mode semantic \
-            --case-set all \
-            --output-dir benchmarks/contexttrace_bench/out \
-            --enforce-sota-gates
+          python -m pip install -e "./packages/contexttrace[test,quality]"
+      - name: Run tests with coverage gate
+        env:
+          PYTHONWARNINGS: error::ResourceWarning
+        run: python -m pytest -q --cov=contexttrace --cov-report=term-missing --cov-fail-under=80
+      - name: Ruff correctness checks
+        run: python -m ruff check --select E9,F601,F63,F7,F82 packages/contexttrace/contexttrace packages/contexttrace/tests
+      - name: Type-check new public contracts
+        run: python -m mypy --follow-imports=skip --ignore-missing-imports packages/contexttrace/contexttrace/contracts.py packages/contexttrace/contexttrace/privacy.py
+      - name: Audit installed dependencies
+        run: python -m pip_audit
 
-      - name: Upload ContextTrace-Bench artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
+  sdk-compat:
+    name: SDK Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          name: contexttrace-bench
-          path: |
-            benchmarks/contexttrace_bench/out/contexttrace_bench_results.json
-            benchmarks/contexttrace_bench/out/results.md
-            benchmarks/contexttrace_bench/out/leaderboard.md
-            benchmarks/contexttrace_bench/out/report.html
-            benchmarks/contexttrace_bench/out/error_analysis.json
-            benchmarks/contexttrace_bench/out/error_analysis.md
-            benchmarks/contexttrace_bench/out/candidate_inputs.jsonl
-            benchmarks/contexttrace_bench/METHODOLOGY.md
-            benchmarks/contexttrace_bench/BASELINES.md
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -e "./packages/contexttrace[test]"
+      - run: python -m pytest -q packages/contexttrace/tests
 
-      - name: Build SDK package
-        run: |
-          python -m pip install build
-          python -m build packages/contexttrace
+  optional-integrations:
+    name: Optional integration (${{ matrix.integration }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - integration: fastapi
+            extra: fastapi
+            tests: packages/contexttrace/tests/test_fastapi_middleware.py
+          - integration: langchain
+            extra: langchain
+            tests: packages/contexttrace/tests/test_langchain_integration.py
+          - integration: langgraph
+            extra: langgraph
+            tests: packages/contexttrace/tests/test_langgraph_integration.py
+          - integration: llamaindex
+            extra: llamaindex
+            tests: packages/contexttrace/tests/test_llamaindex_integration.py
+          - integration: opentelemetry
+            extra: opentelemetry
+            tests: packages/contexttrace/tests/test_opentelemetry_export.py
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -e "./packages/contexttrace[test,${{ matrix.extra }}]"
+      - run: python -m pytest -q ${{ matrix.tests }} packages/contexttrace/tests/test_integration_concurrency.py
 
-      - name: Smoke install SDK wheel
+  wheel-smoke:
+    name: Wheel smoke (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - run: python -m pip install --upgrade pip build
+      - run: python -m build packages/contexttrace
+      - name: Install and import wheel
+        shell: bash
         run: |
-          python -m venv /tmp/contexttrace-smoke
-          /tmp/contexttrace-smoke/bin/python -m pip install --upgrade pip
-          /tmp/contexttrace-smoke/bin/python -m pip install packages/contexttrace/dist/contexttrace-*.whl
-          /tmp/contexttrace-smoke/bin/contexttrace --version
-          /tmp/contexttrace-smoke/bin/python -c "import contexttrace; print(contexttrace.__version__)"
+          python -m pip install packages/contexttrace/dist/contexttrace-*.whl
+          contexttrace --version
+          python -c "from contexttrace import ContextTrace, load_json_schema; print(ContextTrace, load_json_schema('TraceV1')['title'])"

.github/workflows/release.yml

@@ -103,6 +103,7 @@ jobs:
       name: pypi
       url: https://pypi.org/p/contexttrace
     permissions:
+      id-token: write
       contents: read
     steps:
       - name: Download distribution artifacts
@@ -115,6 +116,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: dist
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          skip-existing: true
-          attestations: false
+          attestations: true

CHANGELOG.md

@@ -6,8 +6,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
 
 ## [Unreleased]
 
+## [1.1.0] - 2026-07-22
+
 ### Added
 
+- The `1.1.0` release, validated through the `1.1.0rc1` TestPyPI candidate,
+  including versioned public JSON
+  schemas, artifact provenance, strict privacy controls, streaming-safe capture,
+  concurrent integration isolation, verification limits, and batch verification.
+- Python 3.10--3.13, optional-integration, dependency-audit, 80% coverage, and
+  cross-platform wheel quality gates.
+- Golden TraceV1 compatibility coverage and adversarial tests for nested
+  redaction, stream chunking, queue saturation, concurrency, and oversized input.
 - `contexttrace repair` and SDK helpers for evidence-backed, root-cause-specific
   repair plans, optional corpus audit, and post-fix regression commands.
 

apps/api/tests/conftest.py

@@ -36,10 +36,12 @@ def override_get_db():
 
     app.dependency_overrides[get_db] = override_get_db
     app.dependency_overrides[get_llm_judge_provider] = lambda: judge_provider
-    with TestClient(app) as test_client:
-        yield test_client
-
-    Base.metadata.drop_all(bind=engine)
+    try:
+        with TestClient(app) as test_client:
+            yield test_client
+    finally:
+        Base.metadata.drop_all(bind=engine)
+        engine.dispose()
 
 
 @pytest.fixture()

benchmarks/contexttrace_bench/BASELINES.md

@@ -1,5 +1,10 @@
 # Baseline Comparison Runbook
 
+> **Calibration notice:** all ContextTrace results in this file, including the
+> RAGTruth 200-case sample, are calibration/development evidence for the frozen
+> `semantic_v1_calibrated` verifier. They are not untouched external-test results.
+> See `docs/verifier-governance.md` for the successor-verifier protocol.
+
 This file tracks competitor and reference baseline work for ContextTrace-Bench.
 Rows should only be described as publishable after they cover the full benchmark
 case set and are scored by `run_contexttrace.py --candidate`.
@@ -8,7 +13,7 @@ case set and are scored by `run_contexttrace.py --candidate`.
 
 | System | Runner or Adapter | Status | Publishable | Notes |
 | --- | --- | --- | --- | --- |
-| ContextTrace semantic verifier | `run_contexttrace.py --mode semantic` | Ready | Yes | Local-first product path. CI enforces default quality gates. |
+| ContextTrace `semantic_v1_calibrated` verifier | `run_contexttrace.py --mode semantic` | Frozen calibration | No | Local-first compatibility path. Repeatedly calibrated on repository and RAGTruth cases; not external test evidence. |
 | RAGAS | `run_ragas.py` | Full OpenAI-backed candidate scored | Yes | `gpt-4.1-mini`, 500/500 rows, zero row errors. Faithfulness-only baseline; diagnostic fields are `N/A`. |
 | DeepEval | `run_deepeval.py` | Full OpenAI-backed candidate scored | Yes | `gpt-4.1-mini`, 500/500 rows, zero row errors. Faithfulness-only baseline; diagnostic fields are `N/A`. |
 | RAGChecker | `run_ragchecker.py`, `adapt_candidate.py --preset ragchecker` | 200-row real-reference CRAG calibration scored | No | `gpt-4.1-mini`, 200/200 same-ID CRAG rows, real official-answer sidecar, all 11 metrics, and zero errors. The gold-answer grounding proxy remains review-pending, not publishable. |

benchmarks/contexttrace_bench/METHODOLOGY.md

@@ -1,5 +1,14 @@
 # ContextTrace-Bench Methodology
 
+## Calibration status
+
+The current repository benchmark, ContextTrace-Diag-150, Naturalistic Eval v2,
+and the 200-case RAGTruth sample are development/calibration sets for
+`semantic_v1_calibrated`. Results on them must not be described as untouched
+external-test performance. No successor-verifier rule or threshold may be
+changed in response to their errors. The source/domain/time-disjoint freeze
+protocol is documented in `docs/verifier-governance.md`.
+
 ContextTrace-Bench measures ContextTrace as a verifier, not as a retriever or
 answer generator. A case is a portable RAG trace with a query, answer, retrieved
 contexts, optional citations, and expected diagnostic labels.

benchmarks/contexttrace_bench/freeze_untouched_split.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+REQUIRED_SEPARATION_FIELDS = (
+    "track",
+    "source_family",
+    "source_document_id",
+    "domain",
+    "publication_window",
+)
+DISJOINT_FIELDS = (
+    "source_family",
+    "source_document_id",
+    "domain",
+    "publication_window",
+)
+
+
+def freeze_split(case_pack: dict[str, Any], calibration: dict[str, Any]) -> dict[str, Any]:
+    cases = case_pack.get("cases")
+    if not isinstance(cases, list) or not cases:
+        raise ValueError("Candidate test pack must contain a non-empty cases list.")
+
+    calibration_cases = calibration.get("cases") or []
+    if not isinstance(calibration_cases, list):
+        raise ValueError("Calibration pack cases must be a list.")
+
+    seen_ids: set[str] = set()
+    normalized: list[dict[str, str]] = []
+    for index, case in enumerate(cases):
+        if not isinstance(case, dict):
+            raise ValueError("cases[%s] must be an object." % index)
+        case_id = str(case.get("id") or "").strip()
+        if not case_id or case_id in seen_ids:
+            raise ValueError("Every candidate test case must have a unique non-empty id.")
+        seen_ids.add(case_id)
+        record = {"id": case_id}
+        for field in REQUIRED_SEPARATION_FIELDS:
+            value = str(case.get(field) or (case.get("metadata") or {}).get(field) or "").strip()
+            if not value:
+                raise ValueError("Case %s is missing required separation field %s." % (case_id, field))
+            record[field] = value
+        normalized.append(record)
+
+    calibration_values = {
+        field: {
+            str(case.get(field) or (case.get("metadata") or {}).get(field) or "").strip()
+            for case in calibration_cases
+            if isinstance(case, dict)
+        }
+        for field in DISJOINT_FIELDS
+    }
+    overlaps: dict[str, list[str]] = {}
+    for field in DISJOINT_FIELDS:
+        values = sorted({record[field] for record in normalized} & calibration_values[field])
+        if values:
+            overlaps[field] = values
+    if overlaps:
+        raise ValueError("Candidate split overlaps calibration data: %s" % json.dumps(overlaps, sort_keys=True))
+
+    normalized.sort(key=lambda item: item["id"])
+    canonical = json.dumps(normalized, sort_keys=True, separators=(",", ":"))
+    return {
+        "schema_version": 1,
+        "status": "frozen_unscored",
+        "frozen_at": datetime.now(timezone.utc).isoformat(),
+        "case_count": len(normalized),
+        "separation_fields": list(REQUIRED_SEPARATION_FIELDS),
+        "cases": normalized,
+        "manifest_sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(),
+        "policy": "Publish this manifest before successor-verifier implementation; score once after lock.",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Freeze and hash a source-family/domain/publication-window-disjoint test split."
+    )
+    parser.add_argument("--case-pack", required=True)
+    parser.add_argument("--calibration-pack", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    candidate = json.loads(Path(args.case_pack).read_text(encoding="utf-8"))
+    calibration = json.loads(Path(args.calibration_pack).read_text(encoding="utf-8"))
+    manifest = freeze_split(candidate, calibration)
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print("Frozen %s cases: %s" % (manifest["case_count"], manifest["manifest_sha256"]))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())