feat(eval): add CompileAndRun scorer with sandbox + YAGNI-for-tests

AGENTS.md

@@ -743,6 +743,9 @@ and inspect trajectories visually (Langfuse / Jaeger / Arize Phoenix).
 | `cmd/sin-code/internal/evalharness/prices.go` | v3.18.0 self-pricing price book (USD/1k tokens per model) |
 | `cmd/sin-code/internal/evalharness/snapshot.go` | v3.18.0 deterministic snapshot round-trip (caveman evals/README.md §3) |
 | `cmd/sin-code/eval_cmd.go` | `sin-code eval run` + `eval compare` + `eval snapshot` + `eval diff` (issue #171) |
+| `cmd/sin-code/internal/evalharness/scorer.go` | Scorers: exact, contains, success, LLM-judge, composite, **CompileAndRun** |
+| `cmd/sin-code/internal/evalharness/runner_extras.go` | Compile-and-run helpers (code extraction, sandboxed compile/run) |
+| `cmd/sin-code/eval_cmd.go` | `sin-code eval run` + `sin-code eval list` |
 | `cmd/sin-code/trace_cmd.go` | `sin-code trace doctor` — exporter-only sanity check |
 | `evals/critical.json` | Example Golden Dataset (3 cases, no LLM needed) |
 | `evals/three-arm-example.json` | v3.18.0 four-arm bench, 3 cases (issue #171) |
@@ -769,6 +772,14 @@ sin-code eval run --dataset evals/three-arm-example.json \
 sin-code eval compare --dataset evals/three-arm-example.json
 sin-code eval snapshot --dataset evals/three-arm-example.json --out /tmp/snap.json
 sin-code eval diff --snapshot /tmp/snap-base.json --snapshot-b /tmp/snap-head.json
+# Compile-and-run scorer (ponytail correctness.js analog)
+sin-code eval run --dataset evals/coding.json \
+    --scorer compile-and-run --language python \
+    --self-check "assert fizzbuzz(15) == 'FizzBuzz'"
+
+# YAGNI mode: trivial one-liners accepted after compile-only
+sin-code eval run --dataset evals/trivial.json \
+    --scorer compile-and-run --language python --skip-test
 
 # Sanity-check the OTel exporter setup without a full eval
 sin-code trace doctor --exporter stdout --emit-sample-span
@@ -841,4 +852,8 @@ same input on every CI run (caveman evals/README.md §3 promise:
   `Compare` runner, arm constructors, snapshot round-trip (issue #171).
 - `cmd/sin-code/internal/evalharness/snapshot.doc.md` — deterministic
   matrix + diff (issue #171).
+- `cmd/sin-code/internal/evalharness/scorer.doc.md` — scorer interface
+  and built-in scorers (including `CompileAndRun`).
+- `cmd/sin-code/internal/evalharness/runner_extras.doc.md` — code-block
+  extraction, per-language compile, and sandboxed self-check execution.
 

CHANGELOG.md

@@ -528,9 +528,17 @@ reports them; `debt check` gates them.
   `sin assets list|validate|show|import`.
 - **`internal/evalharness/`** — eval-driven development. `EvalSet` /
   `Run` / `Result` types, pluggable `Scorer`s (exact, contains-all,
-  success-flag, LLM-judge, composite), per-case timeout, JSONL run
-  history, and `Compare` for case-by-case regression detection with
-  `--fail-on-regress` as a CI gate. CLI: `sin eval run|list|compare`.
+  success-flag, LLM-judge, composite, **CompileAndRun**), per-case
+  timeout, JSONL run history, and `Compare` for case-by-case regression
+  detection with `--fail-on-regress` as a CI gate. CLI:
+  `sin eval run|list|compare`.
+- **`CompileAndRun` scorer** (issue #181) — ponytail `correctness.js`
+  analog for SIN-Code. Extracts fenced code from model output, compiles
+  it (`go`/`python`/`javascript`/`bash`), and runs a sandboxed self-check.
+  Returns 1.0 only when compile + run pass; `skip_test` mode accepts
+  trivial one-liners after compile-only (YAGNI for tests). Wired into
+  `sin-code eval run` via `--scorer compile-and-run --language <lang>`
+  and into Golden Datasets via `test_cases[].scorer`.
 - **`internal/dispatch/`** — turns loaded command and agent assets
   into executable actions. ECC-style placeholder substitution
   (`$ARGUMENTS`, `$1..$9`, `$@`, `${flag}`), `Dispatcher` routes

cmd/sin-code/eval_cmd.go

@@ -97,6 +97,11 @@
 		armsFlag      string
 		userSkill     string
 		modelPricing  string
+		scorerType    string
+		scorerLang    string
+		scorerSelfChk string
+		scorerSkip    bool
+		scorerBinary  string
 	)
 
 	cmd := &cobra.Command{
@@ -160,6 +165,15 @@
 				RunOverride: stubRunOverride,
 			}
 
+			var overrideScorer evalharness.Scorer
+			if scorerType != "" {
+				scorer, err := buildScorer(scorerType, scorerLang, scorerSelfChk, scorerSkip, scorerBinary)
+				if err != nil {
+					return fmt.Errorf("eval run: scorer: %w", err)
+				}
+				overrideScorer = scorer
+			}
+
 			runner, err := dataset.NewRunner(dataset.RunnerConfig{
 				ProfileName:    profile,
 				HeadlessMode:   true,
@@ -170,6 +184,7 @@
 			if err != nil {
 				return fmt.Errorf("eval run: new runner: %w", err)
 			}
+			runner.Scorer = overrideScorer
 
 			results, err := runner.RunDataset(ctx, ds)
 			if err != nil {
@@ -215,6 +230,11 @@
 	cmd.Flags().StringVar(&armsFlag, "arm", "", "Comma-separated arm list (issue #171 comparator). Reserved tokens: baseline, terse, lazy_skill. Anything else is treated as a user-skill name. Empty = single-arm (legacy behavior).")
 	cmd.Flags().StringVar(&userSkill, "skill", "skill-code-create", "User-skill arm name used when --arm contains a non-reserved token (issue #171).")
 	cmd.Flags().StringVar(&modelPricing, "model-pricing", "stub", "Model-price entry from the comparator price book (issue #171). Default 'stub' = USD 0 in CI.")
+	cmd.Flags().StringVar(&scorerType, "scorer", "", "Override scorer type (compile_and_run|exact|contains)")
+	cmd.Flags().StringVar(&scorerLang, "language", "", "Language for --scorer compile-and-run (go|python|javascript|bash)")
+	cmd.Flags().StringVar(&scorerSelfChk, "self-check", "", "Self-check code for --scorer compile-and-run")
+	cmd.Flags().BoolVar(&scorerSkip, "skip-test", false, "YAGNI mode: accept compile-only for trivial one-liners")
+	cmd.Flags().StringVar(&scorerBinary, "scorer-binary", "", "Explicit compiler/interpreter for --scorer compile-and-run")
 
 	cmd.MarkFlagRequired("dataset")
 	return cmd
@@ -671,4 +691,28 @@
 	_ = cmd.MarkFlagRequired("snapshot")
 	_ = cmd.MarkFlagRequired("snapshot-b")
 	return cmd
+// buildScorer constructs an evalharness.Scorer from the CLI flags.
+// Supported types: compile_and_run, exact, contains.
+func buildScorer(typ, lang, selfCheck string, skipTest bool, binary string) (evalharness.Scorer, error) {
+	switch typ {
+	case "compile_and_run":
+		if lang == "" {
+			return nil, errors.New("--language is required for compile_and_run scorer")
+		}
+		if !evalharness.IsCompileAndRunLanguage(lang) {
+			return nil, fmt.Errorf("unsupported language %q", lang)
+		}
+		return evalharness.CompileAndRun{
+			Language:  lang,
+			SelfCheck: selfCheck,
+			SkipTest:  skipTest,
+			Binary:    binary,
+		}, nil
+	case "exact":
+		return evalharness.ExactMatch{}, nil
+	case "contains":
+		return evalharness.ContainsAll{}, nil
+	default:
+		return nil, fmt.Errorf("unsupported scorer %q", typ)
+	}
 }

cmd/sin-code/internal/dataset/dataset.doc.md

@@ -15,7 +15,8 @@ A Golden Dataset is one JSON file under `evals/*.json`:
       "id": "auth_refactor",
       "prompt": "Refactor the auth system…",
       "constraints": {"must_use_tools": ["sin_edit"], "max_turns": 10},
-      "expected": {"contains_keywords": ["argon2"], "min_quality": 0.8}
+      "expected": {"contains_keywords": ["argon2"], "min_quality": 0.8},
+      "scorer": {"type": "compile_and_run", "language": "go", "self_check": "assert foo() == 1"}
     }
   ]
 }
@@ -39,6 +40,9 @@ A Golden Dataset is one JSON file under `evals/*.json`:
 - `require_verify=true` requires a non-empty `verify_cmd`.
 - `min_quality` and `max_tokens` are bounded; out-of-range values
   fail validation at load time.
+- `scorer` selects a post-hoc output scorer. `type: compile_and_run`
+  requires a supported `language` (`go`, `python`, `javascript`, `bash`)
+  and optionally `self_check`, `skip_test`, `timeout`, and `binary`.
 
 ## Without the issue's reference library
 

cmd/sin-code/internal/dataset/dataset.go

@@ -56,6 +56,7 @@ type TestCase struct {
 	Expected    Expected          `json:"expected,omitempty"`
 	VerifyCmd   string            `json:"verify_cmd,omitempty"`
 	Metadata    map[string]string `json:"metadata,omitempty"`
+	Scorer      ScorerConfig      `json:"scorer,omitempty"`
 }
 
 // Constraints holds hard rules. All fields are optional; Validate
@@ -82,6 +83,19 @@ type Expected struct {
 	CustomCriteria   string   `json:"custom_criteria,omitempty"`
 }
 
+// ScorerConfig selects a scorer and its parameters. It is the dataset
+// representation of an evalharness.Scorer; the runner resolves it when
+// applying post-hoc output gates.
+type ScorerConfig struct {
+	Type          string  `json:"type,omitempty"`           // "compile_and_run" | "exact" | "contains"
+	Language      string  `json:"language,omitempty"`       // language for compile_and_run
+	SelfCheck     string  `json:"self_check,omitempty"`     // self-check code
+	SkipTest      bool    `json:"skip_test,omitempty"`      // YAGNI mode for trivial one-liners
+	Timeout       string  `json:"timeout,omitempty"`        // e.g. "30s"
+	Binary        string  `json:"binary,omitempty"`         // explicit compiler/interpreter
+	PassThreshold float64 `json:"pass_threshold,omitempty"` // threshold for contains scorer
+}
+
 // LoadDataset reads + validates a single JSON file. Returns the
 // Dataset or a wrapped error including the absolute path so users
 // see which file failed without running the command twice.
@@ -150,10 +164,76 @@ func (ds *Dataset) Validate() error {
 				return fmt.Errorf("test_cases[%s]: invalid timeout %q: %v", tc.ID, tc.Constraints.Timeout, err)
 			}
 		}
+		if err := tc.Scorer.Validate(); err != nil {
+			return fmt.Errorf("test_cases[%s]: scorer: %w", tc.ID, err)
+		}
 	}
 	return nil
 }
 
+// Validate checks a ScorerConfig for consistency.
+func (sc ScorerConfig) Validate() error {
+	if sc.Type == "" {
+		return nil
+	}
+	switch sc.Type {
+	case "compile_and_run":
+		if !isCompileAndRunLanguage(sc.Language) {
+			return fmt.Errorf("unsupported compile_and_run language %q", sc.Language)
+		}
+	case "exact", "contains":
+		// no extra fields required
+	default:
+		return fmt.Errorf("unknown scorer type %q", sc.Type)
+	}
+	if sc.Timeout != "" {
+		if _, err := time.ParseDuration(sc.Timeout); err != nil {
+			return fmt.Errorf("invalid timeout %q: %v", sc.Timeout, err)
+		}
+	}
+	return nil
+}
+
+// isCompileAndRunLanguage is a local copy of the evalharness allow-list
+// to avoid importing the package here and creating a cycle.
+func isCompileAndRunLanguage(lang string) bool {
+	switch lang {
+	case "go", "python", "javascript", "bash":
+		return true
+	}
+	return false
+}
+
+// ToEvalharnessConfig converts ScorerConfig to the map[string]any format
+// consumed by evalharness.ScorerFromConfig.
+func (sc ScorerConfig) ToEvalharnessConfig() map[string]any {
+	if sc.Type == "" {
+		return nil
+	}
+	cfg := map[string]any{
+		"type": sc.Type,
+	}
+	if sc.Language != "" {
+		cfg["language"] = sc.Language
+	}
+	if sc.SelfCheck != "" {
+		cfg["self_check"] = sc.SelfCheck
+	}
+	if sc.SkipTest {
+		cfg["skip_test"] = true
+	}
+	if sc.Timeout != "" {
+		cfg["timeout"] = sc.Timeout
+	}
+	if sc.Binary != "" {
+		cfg["binary"] = sc.Binary
+	}
+	if sc.PassThreshold != 0 {
+		cfg["pass_threshold"] = sc.PassThreshold
+	}
+	return cfg
+}
+
 // FilterByTag returns a copy of ds that contains only TestCases whose
 // Tags include tag (case-insensitive). Empty tag returns a shallow
 // copy with all cases — useful for "no filter" default UX.

cmd/sin-code/internal/dataset/runner.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/agentloop"
+	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/evalharness"
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/session"
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/verify"
 )
@@ -58,9 +59,10 @@ type RunnerConfig struct {
 // are created in the supplied store; if store is nil the runner
 // panics — that is a user bug, not a graceful degradation case.
 type Runner struct {
-	cfg   RunnerConfig
-	loop  *agentloop.Loop
-	store *session.Store
+	cfg    RunnerConfig
+	loop   *agentloop.Loop
+	store  *session.Store
+	Scorer evalharness.Scorer // optional override scorer applied to every case
 }
 
 // NewRunner constructs a Runner. Zero-value fields get sensible
@@ -155,7 +157,9 @@ func (r *Runner) RunCase(ctx context.Context, tc *TestCase) RunResult {
 	res.VerifyPassed = loopRes.Verified
 	res.FinalOutput = loopRes.Summary
 	res.Success = loopRes.Verified
-	return r.applyRules(tc, &res)
+	r.applyRules(tc, &res)
+	r.applyScorer(tc, &res)
+	return res
 }
 
 // applyRules evaluates Constraints + Expected; if any rule fails the
@@ -214,6 +218,39 @@ func (r *Runner) applyRules(tc *TestCase, res *RunResult) RunResult {
 	return *res
 }
 
+// applyScorer evaluates a configured or override scorer against the
+// model's final output. If the scorer fails, res.Success flips to false.
+func (r *Runner) applyScorer(tc *TestCase, res *RunResult) {
+	var scorer evalharness.Scorer
+	var cfg map[string]any
+	if r.Scorer != nil {
+		scorer = r.Scorer
+	} else if tc.Scorer.Type != "" {
+		cfg = tc.Scorer.ToEvalharnessConfig()
+		var err error
+		scorer, err = evalharness.ScorerFromConfig(cfg)
+		if err != nil {
+			res.Success = false
+			if res.Error == "" {
+				res.Error = "scorer: " + err.Error()
+			}
+			return
+		}
+	}
+	if scorer == nil {
+		return
+	}
+	score, passed, detail := scorer.Score(evalharness.EvalCase{
+		ID:     tc.ID,
+		Prompt: tc.Prompt,
+		Scorer: cfg,
+	}, evalharness.Output{Text: res.FinalOutput, Success: res.VerifyPassed})
+	res.Success = res.Success && passed
+	if res.Error == "" && !passed {
+		res.Error = fmt.Sprintf("scorer: score=%.2f detail=%s", score, detail)
+	}
+}
+
 // contains is a small linear scan; test cases are tiny (≤ 100 tools).
 func contains(list []string, want string) bool {
 	for _, s := range list {

cmd/sin-code/internal/evalharness/runner.go

@@ -25,9 +25,6 @@ func (r Runner) Execute(ctx context.Context, set EvalSet) (Run, error) {
 	if r.Subject == nil {
 		return Run{}, fmt.Errorf("evalharness: nil Subject")
 	}
-	if r.Scorer == nil {
-		r.Scorer = SuccessFlag{}
-	}
 	run := Run{
 		ID:        fmt.Sprintf("%s-%d", set.Name, time.Now().Unix()),
 		SetName:   set.Name,
@@ -62,7 +59,20 @@ func (r Runner) runCase(ctx context.Context, c EvalCase) Result {
 	if err != nil {
 		return Result{CaseID: c.ID, Score: 0, Weight: w, Passed: false, Duration: dur, Err: err.Error()}
 	}
-	score, passed, detail := r.Scorer.Score(c, out)
+	scorer := r.Scorer
+	if c.Scorer != nil {
+		caseScorer, serr := ScorerFromConfig(c.Scorer)
+		if serr != nil {
+			return Result{CaseID: c.ID, Score: 0, Weight: w, Passed: false, Duration: dur, Err: serr.Error()}
+		}
+		if caseScorer != nil {
+			scorer = caseScorer
+		}
+	}
+	if scorer == nil {
+		scorer = SuccessFlag{}
+	}
+	score, passed, detail := scorer.Score(c, out)
 	return Result{
 		CaseID: c.ID, Score: score, Weight: w, Passed: passed,
 		Output: truncate(out.Text, 500), Detail: detail, Duration: dur,