LLM-backed CMVK verifiers for production

.env.example

@@ -95,6 +95,15 @@ GRAFANA_PASSWORD=cortex_local
 # Host port 5001 avoids macOS AirPlay Receiver conflict on 5000.
 MLFLOW_TRACKING_URI=http://localhost:5001
 
+# ── CMVK (high-stakes write verification, importance > 0.8) ─────────────────
+# Backend: heuristic (default, zero API cost) | openai | ollama
+CORTEX_CMVK_ENABLED=true
+CORTEX_CMVK_BACKEND=heuristic
+# Production OpenAI verifier model (3 independent prompts, same model)
+CORTEX_CMVK_OPENAI_MODEL=gpt-4o-mini
+# Ollama CMVK model (falls back to OLLAMA_MODEL)
+# CORTEX_CMVK_OLLAMA_MODEL=llama3.1:8b
+
 # ── Feature flags (intelligence + semantic) ─────────────────────────────────
 CORTEX_CONTRADICTION_ENABLED=true
 CORTEX_CONTRADICTION_KAFKA=true

DECISIONS.md

@@ -23,6 +23,14 @@ Agent picks up OPEN instructions at session start, executes, marks DONE.
 
 ## ACTIVE INSTRUCTIONS
 
+### 2026-06-10 — LLM-backed CMVK verifiers (production)
+Priority: HIGH
+Status: DONE — `CORTEX_CMVK_BACKEND=openai|ollama` (2026-06-10)
+Detail:
+- `scoring/cmvk_llm.py` — three independent LLM verifier personas (structure, provenance, stakes)
+- `build_default_verifiers()` selects heuristic (default) or LLM backend via env
+- Tests: `tests/scoring/test_cmvk_llm.py`
+
 ### 2026-06-10 — GDPR erasure API route (admin-only)
 Priority: HIGH
 Status: DONE — `POST /gdpr/erase` (2026-06-10)

SESSIONS.md

@@ -507,3 +507,24 @@
 1. Merge PR #14 after CI green
 2. LLM-backed CMVK verifiers (production)
 3. Phase 6: dashboard polish + demo video
+
+---
+
+## Session — 2026-06-10 — LLM-backed CMVK verifiers
+**Duration:** ~25m
+**Phase:** Phase 4 completion — production CMVK
+
+### Built
+- **`scoring/cmvk_llm.py`** — `LLMDecisionVerifier` with OpenAI function calling + Ollama JSON mode; three personas (structure, provenance, stakes)
+- **`scoring/cmvk.py`** — `build_default_verifiers()` via `CORTEX_CMVK_BACKEND`; CMVK v0.2.0
+- **`tests/scoring/test_cmvk_llm.py`** — factory selection, OpenAI mock, kernel majority with LLM verifiers
+- **`.env.example`** — CMVK backend and model env vars documented
+
+### State at end
+- Branch **`feature/cmvk-llm-verifiers`** — committed `a73a853`, **PR #15** open
+- **350 passed** (full suite, `--no-cov`)
+
+### Next session starts with
+1. Merge PR #15 after CI green
+2. Phase 6: dashboard polish + demo video
+3. Staging smoke with `CORTEX_CMVK_BACKEND=openai`

scoring/cmvk.py

@@ -6,8 +6,9 @@
 
 Decision: D-006 — Bayesian trust scoring with CMVK majority voting.
 
-Production verifiers will call independent LLMs; v0.1 uses deterministic
-heuristic checks so dev/test paths incur zero API cost.
+Production: set ``CORTEX_CMVK_BACKEND=openai`` or ``ollama`` for three LLM
+verifiers (see ``scoring/cmvk_llm.py``). Default ``heuristic`` keeps dev/test
+paths at zero API cost.
 """
 
 from __future__ import annotations
@@ -21,7 +22,7 @@
 
 log = structlog.get_logger(__name__)
 
-CMVK_VERSION = "0.1.0"
+CMVK_VERSION = "0.2.0"
 CMVK_VERIFIER_COUNT = 3
 CMVK_MAJORITY = 2
 
@@ -96,6 +97,19 @@ def default_heuristic_verifiers() -> list[HeuristicDecisionVerifier]:
     ]
 
 
+def build_default_verifiers() -> list[DecisionVerifier]:
+    """Select verifier set from ``CORTEX_CMVK_BACKEND`` (default: heuristic)."""
+    backend = os.environ.get("CORTEX_CMVK_BACKEND", "heuristic").lower()
+    if backend == "heuristic":
+        return default_heuristic_verifiers()
+    if backend in {"openai", "ollama"}:
+        from scoring.cmvk_llm import build_llm_verifiers
+
+        return build_llm_verifiers(backend)
+    log.warning("cmvk.unknown_backend", backend=backend, fallback="heuristic")
+    return default_heuristic_verifiers()
+
+
 class CrossModelVerificationKernel:
     """Runs CMVK majority voting when importance exceeds the high-stakes threshold."""
 
@@ -105,7 +119,7 @@ def __init__(
         *,
         enabled: bool | None = None,
     ) -> None:
-        self._verifiers = verifiers or default_heuristic_verifiers()
+        self._verifiers = verifiers or build_default_verifiers()
         if enabled is None:
             enabled = os.environ.get("CORTEX_CMVK_ENABLED", "true").lower() in {
                 "1",

scoring/cmvk_llm.py

@@ -0,0 +1,269 @@
+"""LLM-backed CMVK verifiers for production high-stakes writes.
+
+Three independent verifier personas call the same OpenAI or Ollama backend with
+different review lenses. Majority vote (2/3) is applied by CrossModelVerificationKernel.
+
+Decision: D-006 — CMVK majority voting for importance > 0.8.
+Backend: CORTEX_CMVK_BACKEND=openai|ollama (heuristic is default in cmvk.py).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+import structlog
+from openai import OpenAI
+
+from scoring.cmvk import DecisionVerifier, VerifierVote
+from shared.models import DecisionEvent
+
+log = structlog.get_logger(__name__)
+
+_VERIFICATION_FUNCTION = {
+    "name": "verify_decision_event",
+    "description": (
+        "Approve or reject a high-stakes extracted decision before it is stored "
+        "in the organizational memory graph."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "approved": {
+                "type": "boolean",
+                "description": "True only when the decision passes this verifier's checks.",
+            },
+            "rationale": {
+                "type": "string",
+                "description": "One sentence explaining the approve/reject outcome.",
+            },
+        },
+        "required": ["approved", "rationale"],
+    },
+}
+
+_VERIFIER_PROMPTS: dict[str, str] = {
+    "cmvk-llm-structure": """You are CMVK verifier A — structural completeness.
+
+Review the extracted DecisionEvent JSON. Approve only when:
+- content is a concrete organizational decision/exception/rationale/update (not noise)
+- made_by is present when the text implies authors or approvers
+- affects or rationale is populated when the decision references systems or reasons
+
+Reject vague summaries, questions, or incomplete extractions.""",
+    "cmvk-llm-provenance": """You are CMVK verifier B — extraction quality and provenance.
+
+Review the extracted DecisionEvent JSON. Approve only when:
+- extraction_confidence is plausible for the content (not inflated)
+- fields appear grounded in the message (no obvious hallucinated people/systems)
+- source/channel metadata is consistent with a real connector event
+
+Reject when the extraction looks over-confident or fabricated.""",
+    "cmvk-llm-stakes": """You are CMVK verifier C — high-stakes justification.
+
+This decision has importance_score > 0.8. Approve only when:
+- the event materially affects architecture, policy, security, or operations
+- it is actionable organizational memory (not casual chat or weak status noise)
+- storing it permanently in a knowledge graph is justified
+
+Reject low-stakes chatter incorrectly marked as high importance.""",
+}
+
+_OLLAMA_EXAMPLE = '{"approved": true, "rationale": "Concrete decision with authors and affected systems."}'
+
+
+def decision_payload(decision: DecisionEvent) -> dict[str, Any]:
+    """Serialize fields relevant to CMVK review (no scores that bias verifiers)."""
+    return {
+        "event_type": decision.event_type,
+        "content": decision.content,
+        "made_by": decision.made_by,
+        "affects": decision.affects,
+        "rationale": decision.rationale,
+        "replaces": decision.replaces,
+        "triggered_by": decision.triggered_by,
+        "extraction_confidence": decision.extraction_confidence,
+        "importance_score": decision.importance_score,
+        "status": decision.status,
+        "provenance": {
+            "source": decision.provenance.source,
+            "channel": decision.provenance.channel,
+            "extractor_model": decision.provenance.extractor_model,
+            "extractor_version": decision.provenance.extractor_version,
+        },
+    }
+
+
+def _parse_verdict(raw: str) -> dict[str, Any] | None:
+    """Parse LLM JSON verdict; tolerate fenced code blocks."""
+    text = raw.strip()
+    if text.startswith("```"):
+        lines = text.split("\n")
+        text = "\n".join(lines[1:-1]) if len(lines) > 2 else text
+    try:
+        parsed: dict[str, Any] = json.loads(text)
+    except json.JSONDecodeError:
+        return None
+    if "approved" not in parsed:
+        return None
+    return parsed
+
+
+def _call_openai_verify(
+    client: OpenAI,
+    model: str,
+    system_prompt: str,
+    payload: dict[str, Any],
+) -> dict[str, Any] | None:
+    """Call OpenAI with function calling for a structured verdict."""
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {
+                "role": "user",
+                "content": (
+                    "Review this extracted DecisionEvent and return your verdict.\n\n"
+                    f"{json.dumps(payload, indent=2)}"
+                ),
+            },
+        ],
+        tools=[{"type": "function", "function": _VERIFICATION_FUNCTION}],
+        tool_choice={"type": "function", "function": {"name": "verify_decision_event"}},
+        temperature=0.0,
+    )
+    message = response.choices[0].message
+    if not message.tool_calls:
+        return None
+    arguments = message.tool_calls[0].function.arguments
+    try:
+        return json.loads(arguments)
+    except json.JSONDecodeError:
+        return None
+
+
+def _call_ollama_verify(
+    base_url: str,
+    model: str,
+    system_prompt: str,
+    payload: dict[str, Any],
+) -> dict[str, Any] | None:
+    """Call Ollama OpenAI-compatible API with JSON verdict output."""
+    client = OpenAI(api_key="ollama", base_url=f"{base_url.rstrip('/')}/v1")
+    prompt = (
+        f"{system_prompt}\n\n"
+        "Output ONLY a JSON object with fields: approved (boolean), rationale (string).\n"
+        f"Example:\n{_OLLAMA_EXAMPLE}\n\n"
+        f"DecisionEvent:\n{json.dumps(payload, indent=2)}"
+    )
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+        temperature=0.0,
+        max_tokens=256,
+        response_format={"type": "json_object"},
+    )
+    raw = response.choices[0].message.content or "{}"
+    return _parse_verdict(raw)
+
+
+class LLMDecisionVerifier:
+    """Single LLM verifier with a fixed review lens."""
+
+    def __init__(
+        self,
+        verifier_id: str,
+        system_prompt: str,
+        *,
+        backend: str,
+        model: str,
+        ollama_base_url: str = "http://localhost:11434",
+        client: OpenAI | None = None,
+    ) -> None:
+        self.verifier_id = verifier_id
+        self._system_prompt = system_prompt
+        self._backend = backend
+        self._model = model
+        self._ollama_base_url = ollama_base_url
+        self._client = client
+
+    def _openai_client(self) -> OpenAI:
+        if self._client is not None:
+            return self._client
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError("OPENAI_API_KEY must be set for openai CMVK backend")
+        return OpenAI(api_key=api_key)
+
+    def verify(self, decision: DecisionEvent) -> VerifierVote:
+        """Return approve/reject verdict from the configured LLM backend."""
+        payload = decision_payload(decision)
+        try:
+            if self._backend == "openai":
+                result = _call_openai_verify(
+                    self._openai_client(),
+                    self._model,
+                    self._system_prompt,
+                    payload,
+                )
+            elif self._backend == "ollama":
+                result = _call_ollama_verify(
+                    self._ollama_base_url,
+                    self._model,
+                    self._system_prompt,
+                    payload,
+                )
+            else:
+                return VerifierVote(
+                    self.verifier_id,
+                    False,
+                    f"unsupported backend: {self._backend}",
+                )
+        except Exception as exc:
+            log.warning(
+                "cmvk.llm.verifier_failed",
+                verifier_id=self.verifier_id,
+                error=str(exc),
+            )
+            return VerifierVote(self.verifier_id, False, f"llm error: {exc}")
+
+        if result is None:
+            return VerifierVote(self.verifier_id, False, "verifier returned no result")
+
+        approved = bool(result.get("approved"))
+        rationale = str(result.get("rationale", "")).strip()
+        if not rationale:
+            rationale = "approved" if approved else "rejected"
+        return VerifierVote(self.verifier_id, approved, rationale[:500])
+
+
+def build_llm_verifiers(backend: str) -> list[DecisionVerifier]:
+    """Construct three independent LLM verifiers for the given backend."""
+    if backend == "openai":
+        model = os.environ.get("CORTEX_CMVK_OPENAI_MODEL", "gpt-4o-mini")
+        ollama_base_url = ""
+    else:
+        model = os.environ.get("CORTEX_CMVK_OLLAMA_MODEL") or os.environ.get(
+            "OLLAMA_MODEL", "llama3.1:8b"
+        )
+        ollama_base_url = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
+
+    verifiers: list[DecisionVerifier] = []
+    for verifier_id, prompt in _VERIFIER_PROMPTS.items():
+        verifiers.append(
+            LLMDecisionVerifier(
+                verifier_id,
+                prompt,
+                backend=backend,
+                model=model,
+                ollama_base_url=ollama_base_url,
+            )
+        )
+    log.info(
+        "cmvk.llm_verifiers_built",
+        backend=backend,
+        model=model,
+        count=len(verifiers),
+    )
+    return verifiers