Skills evaluation

.gitignore

@@ -6,3 +6,20 @@ debug.jsonl
 metadata.json
 spans.jsonl
 designs/
+
+# Eval run artifacts (transcripts, per-eval results, summaries).
+# Keep evals/baselines/ tracked -- that's the canonical pass/fail reference.
+evals/results/
+
+# Python
+*.pyc
+.pytest_cache/
+*.egg-info/
+
+# Virtualenvs
+.venv/
+venv/
+
+# Local secrets / env overrides
+.env
+.envrc

evals/README.md

@@ -0,0 +1,78 @@
+# Skill Evals
+
+Evaluation of skills under `skills/<name>/`. For each eval, the runner spawns a headless `claude --print` agent, captures its transcript, then asks a separate LLM-as-judge call to score each expectation pass/fail.
+
+## Running
+
+```bash
+# Single skill
+python evals/run.py --skill rai-predictive-training
+
+# Single eval id (faster iteration)
+python evals/run.py --skill rai-predictive-training --eval-id 1
+
+# Multiple skills in one run
+python evals/run.py --skill rai-ontology-design --skill rai-predictive-modeling
+
+# Run from a different working directory (where `raiconfig.yaml` lives, etc.)
+python evals/run.py --skill <name> --cwd /path/to/project
+
+# Freeze current pass/fail map as the baseline
+python evals/run.py --skill <name> --update-baseline
+
+# Verbose: per-expectation judge progress and banners around each eval
+python evals/run.py --skill <name> --eval-id 1 -v
+```
+
+`run.py` exits non-zero if any expectation regressed against the stored baseline (passing in baseline, now failing).
+
+**About `-v`.** It prints banners between phases (`running agent...`, `agent done`, `judging i/N: ...`, `PASS/FAIL -- <justification>`) so you can see judge-by-judge progress live. It does **not** stream the agent's stdout in real time — `claude --print` with text output buffers the entire reply and emits it at the end of the run, so the agent phase is silent regardless of `-v`. Use `-v` mainly to watch the judge phase or to confirm the runner is alive between evals.
+
+## Adding evals
+
+Edit `skills/<skill>/evals/evals.json`:
+
+```json
+{
+  "id": 1,
+  "prompt": "<user prompt>",
+  "expected_output": "<plain-language description of the right answer>",
+  "expectations": [
+    "<one atomic, judgeable claim>",
+    "<another>"
+  ]
+}
+```
+
+- **Each `expectation` is judged in isolation** — keep them atomic. Vague or compound claims produce inconsistent verdicts.
+- **`expected_output` is documentation only**; the judge never sees it. It's a note for whoever maintains the eval.
+- **Anchor on artifacts**, not process: specific code patterns, argument names, design choices — not "explained well."
+
+After adding, run once, spot-check verdicts in `results/<timestamp>/<skill>/eval_<id>.json`, then re-run with `--update-baseline` to freeze the reference.
+
+## Layout
+
+```
+evals/
+  run.py              # CLI orchestrator
+  runner.py           # claude --print subprocess wrapper (one per eval)
+  judge.py            # LLM-as-judge (one call per expectation)
+  diff.py             # baseline comparison
+  baselines/<skill>.json
+  results/<YYYYMMDD_HHMMSS>/<skill>/
+    eval_<id>_transcript.txt    # raw agent transcript
+    eval_<id>.json              # per-eval result + per-expectation judgments
+    skill_summary.json          # aggregate + diff vs baseline
+```
+
+Eval prompts live next to each skill: `skills/<skill>/evals/evals.json`.
+
+## Prerequisites
+
+- `claude` CLI authenticated (`claude --version` works).
+- The skill must resolve from the agent's `--cwd` (default: `Path.cwd()`).
+- Evals that touch Snowflake need a working `raiconfig.yaml` and the `relationalai` SDK in that cwd.
+
+## Permissions
+
+The runner passes `--dangerously-skip-permissions` so the agent can read files, run Bash, and introspect Snowflake without per-call approval. Same blast radius as an unsupervised agent — review prompts and `cwd` before running.

evals/baselines/rai-predictive-training.json

@@ -0,0 +1,28 @@
+{
+  "1": [
+    {
+      "expectation": "The produced script contains all phases in order: Concept declarations (Customer, Article, Transaction, task table concepts without identify_by), Table-backed define() calls, Train/Val/Test Relationships with 'at {Any:timestamp}' (Train/Val also 'has {Any:label}', Test omits the label), Graph with Edges, PropertyTransformer, GNN constructor, gnn.fit(), gnn.predictions(domain=Test) bound to Customer.predictions, and a final select(...).where(Customer.predictions).inspect()",
+      "passed": true
+    },
+    {
+      "expectation": "Script executes without raising any Python exception or PyRel compile error end-to-end",
+      "passed": true
+    },
+    {
+      "expectation": "Training job reaches JOB_COMPLETED status (visible in streamed logs); the agent does not declare success on JOB_START or partial logs",
+      "passed": true
+    },
+    {
+      "expectation": "Prediction job reaches JOB_COMPLETED status and predictions are loaded back into the logic engine",
+      "passed": true
+    },
+    {
+      "expectation": "Final select(...).inspect() (or .to_df()) returns a non-empty result with columns (c_customer_id, probs, predicted_labels), probs in [0, 1], predicted_labels in {0, 1}",
+      "passed": true
+    },
+    {
+      "expectation": "The script does not introduce manual job polling, sleep loops, or status-check API calls around fit() or predictions() -- both are blocking and stream logs by default",
+      "passed": true
+    }
+  ]
+}

evals/diff.py

@@ -0,0 +1,107 @@
+"""Compare a current run's pass/fail map against a stored baseline.
+
+Baseline format (per skill):
+    {
+      "<eval_id>": [
+        {"expectation": "...", "passed": true|false},
+        ...
+      ],
+      ...
+    }
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def to_passmap(results: dict[str, list[dict]]) -> dict[str, list[dict[str, Any]]]:
+    """Project a per-skill results dict to {eval_id: [{expectation, passed}, ...]}."""
+    out: dict[str, list[dict[str, Any]]] = {}
+    for eid, exp_list in results.items():
+        out[str(eid)] = [
+            {"expectation": e["expectation"], "passed": bool(e["passed"])}
+            for e in exp_list
+        ]
+    return out
+
+
+def compare(
+    current: dict[str, list[dict]],
+    baseline: dict[str, list[dict]] | None,
+) -> dict:
+    if baseline is None:
+        return {
+            "first_run": True,
+            "regressions": [],
+            "improvements": [],
+            "unchanged": [],
+            "added": [],
+            "removed": [],
+        }
+
+    regressions: list[dict] = []
+    improvements: list[dict] = []
+    unchanged: list[dict] = []
+    added: list[dict] = []
+    removed: list[dict] = []
+
+    cur = to_passmap(current)
+
+    for eid, exp_list in cur.items():
+        base_list = baseline.get(eid, [])
+        # Index baseline by expectation text (resilient to reordering).
+        base_by_text = {e["expectation"]: bool(e["passed"]) for e in base_list}
+        for exp in exp_list:
+            text = exp["expectation"]
+            now_pass = bool(exp["passed"])
+            if text not in base_by_text:
+                added.append({
+                    "eval_id": eid,
+                    "expectation": text,
+                    "passed": now_pass,
+                })
+                continue
+            was_pass = base_by_text[text]
+            row = {"eval_id": eid, "expectation": text}
+            if was_pass and not now_pass:
+                regressions.append(row)
+            elif not was_pass and now_pass:
+                improvements.append(row)
+            else:
+                unchanged.append({**row, "passed": now_pass})
+
+    # Detect removed expectations / removed evals.
+    for eid, base_list in baseline.items():
+        cur_texts = {e["expectation"] for e in cur.get(eid, [])}
+        for be in base_list:
+            if be["expectation"] not in cur_texts:
+                removed.append({
+                    "eval_id": eid,
+                    "expectation": be["expectation"],
+                    "was_passed": bool(be["passed"]),
+                })
+
+    return {
+        "first_run": False,
+        "regressions": regressions,
+        "improvements": improvements,
+        "unchanged": unchanged,
+        "added": added,
+        "removed": removed,
+    }
+
+
+def load_baseline(path: Path) -> dict[str, list[dict]] | None:
+    if not path.exists():
+        return None
+    with open(path) as f:
+        return json.load(f)
+
+
+def write_baseline(path: Path, current: dict[str, list[dict]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(to_passmap(current), f, indent=2, sort_keys=True)

evals/judge.py

@@ -0,0 +1,102 @@
+"""LLM-as-judge for skill evals.
+
+One `claude --print` call per (transcript, expectation) pair. The judge
+prompt forces a single-line JSON response: {"passed": bool, "justification": str}.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shlex
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_TIMEOUT = 180
+DEFAULT_CLAUDE_BIN = "claude"
+
+JUDGE_PROMPT = """\
+You are evaluating whether an AI agent's response satisfies a specific expectation.
+Respond with only a single JSON object on one line. No markdown. No prose.
+
+EXPECTATION:
+{expectation}
+
+AGENT_RESPONSE:
+<<<BEGIN>>>
+{transcript}
+<<<END>>>
+
+Output schema (single line, no code fence):
+{{"passed": true|false, "justification": "<1-2 sentences citing specific evidence from AGENT_RESPONSE>"}}
+"""
+
+
+@dataclass
+class JudgeResult:
+    passed: bool
+    justification: str
+    raw: str
+
+    def to_dict(self) -> dict:
+        return {
+            "passed": self.passed,
+            "justification": self.justification,
+            "raw": self.raw,
+        }
+
+
+def _extract_json(s: str) -> dict | None:
+    s = s.strip()
+    # Strip code fences if present.
+    s = re.sub(r"^```(?:json)?\s*", "", s)
+    s = re.sub(r"\s*```$", "", s)
+    # Find the first {...} block.
+    match = re.search(r"\{.*\}", s, re.DOTALL)
+    if not match:
+        return None
+    try:
+        return json.loads(match.group(0))
+    except json.JSONDecodeError:
+        return None
+
+
+def judge(
+    transcript: str,
+    expectation: str,
+    timeout: int = DEFAULT_TIMEOUT,
+    claude_bin: str = DEFAULT_CLAUDE_BIN,
+    cwd: Path | None = None,
+) -> JudgeResult:
+    prompt = JUDGE_PROMPT.format(expectation=expectation, transcript=transcript)
+    cmd = [claude_bin, "--print", prompt]
+    try:
+        proc = subprocess.run(
+            cmd,
+            cwd=str(cwd) if cwd else None,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return JudgeResult(
+            passed=False,
+            justification="Judge call timed out.",
+            raw="",
+        )
+
+    raw = proc.stdout.strip()
+    parsed = _extract_json(raw)
+    if parsed is None:
+        return JudgeResult(
+            passed=False,
+            justification=f"Could not parse JSON from judge: {raw[:300]!r}",
+            raw=raw,
+        )
+
+    return JudgeResult(
+        passed=bool(parsed.get("passed", False)),
+        justification=str(parsed.get("justification", "")),
+        raw=raw,
+    )