From a73a853d2a5629318ea474a3140cc7c1703075a8 Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni Date: Wed, 10 Jun 2026 22:04:22 -0400 Subject: [PATCH 1/2] Add LLM-backed CMVK verifiers for production high-stakes writes. Three independent verifier personas (structure, provenance, stakes) activate via CORTEX_CMVK_BACKEND=openai|ollama while heuristic remains the default for zero-cost dev and CI. Co-authored-by: Cursor --- .env.example | 9 ++ DECISIONS.md | 8 + SESSIONS.md | 22 +++ scoring/cmvk.py | 22 ++- scoring/cmvk_llm.py | 269 +++++++++++++++++++++++++++++++++ tests/scoring/test_cmvk_llm.py | 142 +++++++++++++++++ 6 files changed, 468 insertions(+), 4 deletions(-) create mode 100644 scoring/cmvk_llm.py create mode 100644 tests/scoring/test_cmvk_llm.py diff --git a/.env.example b/.env.example index e8764fb..91fba0f 100644 --- a/.env.example +++ b/.env.example @@ -95,6 +95,15 @@ GRAFANA_PASSWORD=cortex_local # Host port 5001 avoids macOS AirPlay Receiver conflict on 5000. MLFLOW_TRACKING_URI=http://localhost:5001 +# ── CMVK (high-stakes write verification, importance > 0.8) ───────────────── +# Backend: heuristic (default, zero API cost) | openai | ollama +CORTEX_CMVK_ENABLED=true +CORTEX_CMVK_BACKEND=heuristic +# Production OpenAI verifier model (3 independent prompts, same model) +CORTEX_CMVK_OPENAI_MODEL=gpt-4o-mini +# Ollama CMVK model (falls back to OLLAMA_MODEL) +# CORTEX_CMVK_OLLAMA_MODEL=llama3.1:8b + # ── Feature flags (intelligence + semantic) ───────────────────────────────── CORTEX_CONTRADICTION_ENABLED=true CORTEX_CONTRADICTION_KAFKA=true diff --git a/DECISIONS.md b/DECISIONS.md index 56560fe..7f032b7 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -23,6 +23,14 @@ Agent picks up OPEN instructions at session start, executes, marks DONE. ## ACTIVE INSTRUCTIONS +### 2026-06-10 — LLM-backed CMVK verifiers (production) +Priority: HIGH +Status: DONE — `CORTEX_CMVK_BACKEND=openai|ollama` (2026-06-10) +Detail: +- `scoring/cmvk_llm.py` — three independent LLM verifier personas (structure, provenance, stakes) +- `build_default_verifiers()` selects heuristic (default) or LLM backend via env +- Tests: `tests/scoring/test_cmvk_llm.py` + ### 2026-06-10 — GDPR erasure API route (admin-only) Priority: HIGH Status: DONE — `POST /gdpr/erase` (2026-06-10) diff --git a/SESSIONS.md b/SESSIONS.md index f3eb7d6..d225cc7 100644 --- a/SESSIONS.md +++ b/SESSIONS.md @@ -507,3 +507,25 @@ 1. Merge PR #14 after CI green 2. LLM-backed CMVK verifiers (production) 3. Phase 6: dashboard polish + demo video + +--- + +## Session — 2026-06-10 — LLM-backed CMVK verifiers +**Duration:** ~25m +**Phase:** Phase 4 completion — production CMVK + +### Built +- **`scoring/cmvk_llm.py`** — `LLMDecisionVerifier` with OpenAI function calling + Ollama JSON mode; three personas (structure, provenance, stakes) +- **`scoring/cmvk.py`** — `build_default_verifiers()` via `CORTEX_CMVK_BACKEND`; CMVK v0.2.0 +- **`tests/scoring/test_cmvk_llm.py`** — factory selection, OpenAI mock, kernel majority with LLM verifiers +- **`.env.example`** — CMVK backend and model env vars documented + +### State at end +- Branch **`feature/cmvk-llm-verifiers`** from `main` (PR #14 merged) +- **350 passed** (full suite, `--no-cov`) +- Changes uncommitted + +### Next session starts with +1. Commit + PR for CMVK LLM verifiers +2. Phase 6: dashboard polish + demo video +3. Staging smoke with `CORTEX_CMVK_BACKEND=openai` diff --git a/scoring/cmvk.py b/scoring/cmvk.py index e494afa..85abafe 100644 --- a/scoring/cmvk.py +++ b/scoring/cmvk.py @@ -6,8 +6,9 @@ Decision: D-006 — Bayesian trust scoring with CMVK majority voting. -Production verifiers will call independent LLMs; v0.1 uses deterministic -heuristic checks so dev/test paths incur zero API cost. +Production: set ``CORTEX_CMVK_BACKEND=openai`` or ``ollama`` for three LLM +verifiers (see ``scoring/cmvk_llm.py``). Default ``heuristic`` keeps dev/test +paths at zero API cost. """ from __future__ import annotations @@ -21,7 +22,7 @@ log = structlog.get_logger(__name__) -CMVK_VERSION = "0.1.0" +CMVK_VERSION = "0.2.0" CMVK_VERIFIER_COUNT = 3 CMVK_MAJORITY = 2 @@ -96,6 +97,19 @@ def default_heuristic_verifiers() -> list[HeuristicDecisionVerifier]: ] +def build_default_verifiers() -> list[DecisionVerifier]: + """Select verifier set from ``CORTEX_CMVK_BACKEND`` (default: heuristic).""" + backend = os.environ.get("CORTEX_CMVK_BACKEND", "heuristic").lower() + if backend == "heuristic": + return default_heuristic_verifiers() + if backend in {"openai", "ollama"}: + from scoring.cmvk_llm import build_llm_verifiers + + return build_llm_verifiers(backend) + log.warning("cmvk.unknown_backend", backend=backend, fallback="heuristic") + return default_heuristic_verifiers() + + class CrossModelVerificationKernel: """Runs CMVK majority voting when importance exceeds the high-stakes threshold.""" @@ -105,7 +119,7 @@ def __init__( *, enabled: bool | None = None, ) -> None: - self._verifiers = verifiers or default_heuristic_verifiers() + self._verifiers = verifiers or build_default_verifiers() if enabled is None: enabled = os.environ.get("CORTEX_CMVK_ENABLED", "true").lower() in { "1", diff --git a/scoring/cmvk_llm.py b/scoring/cmvk_llm.py new file mode 100644 index 0000000..fad7ac0 --- /dev/null +++ b/scoring/cmvk_llm.py @@ -0,0 +1,269 @@ +"""LLM-backed CMVK verifiers for production high-stakes writes. + +Three independent verifier personas call the same OpenAI or Ollama backend with +different review lenses. Majority vote (2/3) is applied by CrossModelVerificationKernel. + +Decision: D-006 — CMVK majority voting for importance > 0.8. +Backend: CORTEX_CMVK_BACKEND=openai|ollama (heuristic is default in cmvk.py). +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +import structlog +from openai import OpenAI + +from scoring.cmvk import DecisionVerifier, VerifierVote +from shared.models import DecisionEvent + +log = structlog.get_logger(__name__) + +_VERIFICATION_FUNCTION = { + "name": "verify_decision_event", + "description": ( + "Approve or reject a high-stakes extracted decision before it is stored " + "in the organizational memory graph." + ), + "parameters": { + "type": "object", + "properties": { + "approved": { + "type": "boolean", + "description": "True only when the decision passes this verifier's checks.", + }, + "rationale": { + "type": "string", + "description": "One sentence explaining the approve/reject outcome.", + }, + }, + "required": ["approved", "rationale"], + }, +} + +_VERIFIER_PROMPTS: dict[str, str] = { + "cmvk-llm-structure": """You are CMVK verifier A — structural completeness. + +Review the extracted DecisionEvent JSON. Approve only when: +- content is a concrete organizational decision/exception/rationale/update (not noise) +- made_by is present when the text implies authors or approvers +- affects or rationale is populated when the decision references systems or reasons + +Reject vague summaries, questions, or incomplete extractions.""", + "cmvk-llm-provenance": """You are CMVK verifier B — extraction quality and provenance. + +Review the extracted DecisionEvent JSON. Approve only when: +- extraction_confidence is plausible for the content (not inflated) +- fields appear grounded in the message (no obvious hallucinated people/systems) +- source/channel metadata is consistent with a real connector event + +Reject when the extraction looks over-confident or fabricated.""", + "cmvk-llm-stakes": """You are CMVK verifier C — high-stakes justification. + +This decision has importance_score > 0.8. Approve only when: +- the event materially affects architecture, policy, security, or operations +- it is actionable organizational memory (not casual chat or weak status noise) +- storing it permanently in a knowledge graph is justified + +Reject low-stakes chatter incorrectly marked as high importance.""", +} + +_OLLAMA_EXAMPLE = '{"approved": true, "rationale": "Concrete decision with authors and affected systems."}' + + +def decision_payload(decision: DecisionEvent) -> dict[str, Any]: + """Serialize fields relevant to CMVK review (no scores that bias verifiers).""" + return { + "event_type": decision.event_type, + "content": decision.content, + "made_by": decision.made_by, + "affects": decision.affects, + "rationale": decision.rationale, + "replaces": decision.replaces, + "triggered_by": decision.triggered_by, + "extraction_confidence": decision.extraction_confidence, + "importance_score": decision.importance_score, + "status": decision.status, + "provenance": { + "source": decision.provenance.source, + "channel": decision.provenance.channel, + "extractor_model": decision.provenance.extractor_model, + "extractor_version": decision.provenance.extractor_version, + }, + } + + +def _parse_verdict(raw: str) -> dict[str, Any] | None: + """Parse LLM JSON verdict; tolerate fenced code blocks.""" + text = raw.strip() + if text.startswith("```"): + lines = text.split("\n") + text = "\n".join(lines[1:-1]) if len(lines) > 2 else text + try: + parsed: dict[str, Any] = json.loads(text) + except json.JSONDecodeError: + return None + if "approved" not in parsed: + return None + return parsed + + +def _call_openai_verify( + client: OpenAI, + model: str, + system_prompt: str, + payload: dict[str, Any], +) -> dict[str, Any] | None: + """Call OpenAI with function calling for a structured verdict.""" + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": ( + "Review this extracted DecisionEvent and return your verdict.\n\n" + f"{json.dumps(payload, indent=2)}" + ), + }, + ], + tools=[{"type": "function", "function": _VERIFICATION_FUNCTION}], + tool_choice={"type": "function", "function": {"name": "verify_decision_event"}}, + temperature=0.0, + ) + message = response.choices[0].message + if not message.tool_calls: + return None + arguments = message.tool_calls[0].function.arguments + try: + return json.loads(arguments) + except json.JSONDecodeError: + return None + + +def _call_ollama_verify( + base_url: str, + model: str, + system_prompt: str, + payload: dict[str, Any], +) -> dict[str, Any] | None: + """Call Ollama OpenAI-compatible API with JSON verdict output.""" + client = OpenAI(api_key="ollama", base_url=f"{base_url.rstrip('/')}/v1") + prompt = ( + f"{system_prompt}\n\n" + "Output ONLY a JSON object with fields: approved (boolean), rationale (string).\n" + f"Example:\n{_OLLAMA_EXAMPLE}\n\n" + f"DecisionEvent:\n{json.dumps(payload, indent=2)}" + ) + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=0.0, + max_tokens=256, + response_format={"type": "json_object"}, + ) + raw = response.choices[0].message.content or "{}" + return _parse_verdict(raw) + + +class LLMDecisionVerifier: + """Single LLM verifier with a fixed review lens.""" + + def __init__( + self, + verifier_id: str, + system_prompt: str, + *, + backend: str, + model: str, + ollama_base_url: str = "http://localhost:11434", + client: OpenAI | None = None, + ) -> None: + self.verifier_id = verifier_id + self._system_prompt = system_prompt + self._backend = backend + self._model = model + self._ollama_base_url = ollama_base_url + self._client = client + + def _openai_client(self) -> OpenAI: + if self._client is not None: + return self._client + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise ValueError("OPENAI_API_KEY must be set for openai CMVK backend") + return OpenAI(api_key=api_key) + + def verify(self, decision: DecisionEvent) -> VerifierVote: + """Return approve/reject verdict from the configured LLM backend.""" + payload = decision_payload(decision) + try: + if self._backend == "openai": + result = _call_openai_verify( + self._openai_client(), + self._model, + self._system_prompt, + payload, + ) + elif self._backend == "ollama": + result = _call_ollama_verify( + self._ollama_base_url, + self._model, + self._system_prompt, + payload, + ) + else: + return VerifierVote( + self.verifier_id, + False, + f"unsupported backend: {self._backend}", + ) + except Exception as exc: + log.warning( + "cmvk.llm.verifier_failed", + verifier_id=self.verifier_id, + error=str(exc), + ) + return VerifierVote(self.verifier_id, False, f"llm error: {exc}") + + if result is None: + return VerifierVote(self.verifier_id, False, "verifier returned no result") + + approved = bool(result.get("approved")) + rationale = str(result.get("rationale", "")).strip() + if not rationale: + rationale = "approved" if approved else "rejected" + return VerifierVote(self.verifier_id, approved, rationale[:500]) + + +def build_llm_verifiers(backend: str) -> list[DecisionVerifier]: + """Construct three independent LLM verifiers for the given backend.""" + if backend == "openai": + model = os.environ.get("CORTEX_CMVK_OPENAI_MODEL", "gpt-4o-mini") + ollama_base_url = "" + else: + model = os.environ.get("CORTEX_CMVK_OLLAMA_MODEL") or os.environ.get( + "OLLAMA_MODEL", "llama3.1:8b" + ) + ollama_base_url = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434") + + verifiers: list[DecisionVerifier] = [] + for verifier_id, prompt in _VERIFIER_PROMPTS.items(): + verifiers.append( + LLMDecisionVerifier( + verifier_id, + prompt, + backend=backend, + model=model, + ollama_base_url=ollama_base_url, + ) + ) + log.info( + "cmvk.llm_verifiers_built", + backend=backend, + model=model, + count=len(verifiers), + ) + return verifiers diff --git a/tests/scoring/test_cmvk_llm.py b/tests/scoring/test_cmvk_llm.py new file mode 100644 index 0000000..422c0c4 --- /dev/null +++ b/tests/scoring/test_cmvk_llm.py @@ -0,0 +1,142 @@ +"""Tests for scoring/cmvk_llm.py — LLM-backed CMVK verifiers.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from unittest.mock import MagicMock, patch + +import pytest + +from scoring.cmvk import CrossModelVerificationKernel, build_default_verifiers +from scoring.cmvk_llm import ( + LLMDecisionVerifier, + _call_openai_verify, + build_llm_verifiers, + decision_payload, +) +from shared.models import IMPORTANCE_FULL, DecisionEvent, Provenance + +NOW = datetime(2026, 5, 11, 12, 0, 0, tzinfo=UTC) + + +def _decision(**overrides: object) -> DecisionEvent: + base = DecisionEvent( + source_raw_event_id="raw-1", + workspace_id="ws-1", + event_type="decision", + content="We decided to migrate payments to CockroachDB for scale.", + made_by=["alice@company.com"], + affects=["payments-service"], + rationale=["Scale ceiling at 10M txn/day"], + extraction_confidence=0.9, + importance_score=IMPORTANCE_FULL + 0.05, + provenance=Provenance( + source="github", + channel="payments", + original_timestamp=NOW, + extractor_version="0.1.0", + extractor_model="gpt-4o", + verified_by=[], + raw_event_id="raw-1", + ), + extracted_at=NOW, + ) + for key, value in overrides.items(): + setattr(base, key, value) + return base + + +def test_decision_payload_includes_review_fields() -> None: + payload = decision_payload(_decision()) + assert payload["event_type"] == "decision" + assert payload["provenance"]["source"] == "github" + assert payload["importance_score"] > IMPORTANCE_FULL + + +def test_build_default_verifiers_heuristic_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("CORTEX_CMVK_BACKEND", raising=False) + verifiers = build_default_verifiers() + assert len(verifiers) == 3 + assert verifiers[0].verifier_id.startswith("cmvk-heuristic") + + +def test_build_default_verifiers_openai_backend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CORTEX_CMVK_BACKEND", "openai") + verifiers = build_default_verifiers() + assert [v.verifier_id for v in verifiers] == [ + "cmvk-llm-structure", + "cmvk-llm-provenance", + "cmvk-llm-stakes", + ] + + +def test_llm_verifier_approves_openai_response() -> None: + mock_client = MagicMock() + tool_call = MagicMock() + tool_call.function.arguments = json.dumps( + {"approved": True, "rationale": "well-formed decision"} + ) + mock_client.chat.completions.create.return_value.choices = [ + MagicMock(message=MagicMock(tool_calls=[tool_call])) + ] + verifier = LLMDecisionVerifier( + "cmvk-llm-structure", + "prompt", + backend="openai", + model="gpt-4o-mini", + client=mock_client, + ) + vote = verifier.verify(_decision()) + assert vote.approved + assert vote.rationale == "well-formed decision" + + +def test_llm_verifier_rejects_on_api_error() -> None: + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = RuntimeError("rate limited") + verifier = LLMDecisionVerifier( + "cmvk-llm-structure", + "prompt", + backend="openai", + model="gpt-4o-mini", + client=mock_client, + ) + vote = verifier.verify(_decision()) + assert not vote.approved + assert "llm error" in vote.rationale + + +def test_call_openai_verify_parses_tool_arguments() -> None: + mock_client = MagicMock() + tool_call = MagicMock() + tool_call.function.arguments = json.dumps( + {"approved": False, "rationale": "content too vague"} + ) + mock_client.chat.completions.create.return_value.choices = [ + MagicMock(message=MagicMock(tool_calls=[tool_call])) + ] + result = _call_openai_verify(mock_client, "gpt-4o-mini", "prompt", {"content": "x"}) + assert result is not None + assert result["approved"] is False + + +@patch("scoring.cmvk_llm._call_openai_verify") +def test_kernel_majority_with_llm_verifiers( + mock_verify: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + mock_verify.side_effect = [ + {"approved": True, "rationale": "ok"}, + {"approved": True, "rationale": "ok"}, + {"approved": False, "rationale": "stakes too low"}, + ] + kernel = CrossModelVerificationKernel(verifiers=build_llm_verifiers("openai")) + result = kernel.verify(_decision()) + assert result.approved + assert len(result.approved_verifier_ids) == 2 From 2dafd629979f34bb5b8f37c6d915c9d845f9082c Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni Date: Wed, 10 Jun 2026 22:04:49 -0400 Subject: [PATCH 2/2] Update session log with PR #15 reference. Co-authored-by: Cursor --- SESSIONS.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/SESSIONS.md b/SESSIONS.md index d225cc7..bee5e17 100644 --- a/SESSIONS.md +++ b/SESSIONS.md @@ -521,11 +521,10 @@ - **`.env.example`** — CMVK backend and model env vars documented ### State at end -- Branch **`feature/cmvk-llm-verifiers`** from `main` (PR #14 merged) +- Branch **`feature/cmvk-llm-verifiers`** — committed `a73a853`, **PR #15** open - **350 passed** (full suite, `--no-cov`) -- Changes uncommitted ### Next session starts with -1. Commit + PR for CMVK LLM verifiers +1. Merge PR #15 after CI green 2. Phase 6: dashboard polish + demo video 3. Staging smoke with `CORTEX_CMVK_BACKEND=openai`