diff --git a/AGENTS.md b/AGENTS.md index 18925680..29ef2672 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -743,6 +743,9 @@ and inspect trajectories visually (Langfuse / Jaeger / Arize Phoenix). | `cmd/sin-code/internal/evalharness/prices.go` | v3.18.0 self-pricing price book (USD/1k tokens per model) | | `cmd/sin-code/internal/evalharness/snapshot.go` | v3.18.0 deterministic snapshot round-trip (caveman evals/README.md §3) | | `cmd/sin-code/eval_cmd.go` | `sin-code eval run` + `eval compare` + `eval snapshot` + `eval diff` (issue #171) | +| `cmd/sin-code/internal/evalharness/scorer.go` | Scorers: exact, contains, success, LLM-judge, composite, **CompileAndRun** | +| `cmd/sin-code/internal/evalharness/runner_extras.go` | Compile-and-run helpers (code extraction, sandboxed compile/run) | +| `cmd/sin-code/eval_cmd.go` | `sin-code eval run` + `sin-code eval list` | | `cmd/sin-code/trace_cmd.go` | `sin-code trace doctor` — exporter-only sanity check | | `evals/critical.json` | Example Golden Dataset (3 cases, no LLM needed) | | `evals/three-arm-example.json` | v3.18.0 four-arm bench, 3 cases (issue #171) | @@ -769,6 +772,14 @@ sin-code eval run --dataset evals/three-arm-example.json \ sin-code eval compare --dataset evals/three-arm-example.json sin-code eval snapshot --dataset evals/three-arm-example.json --out /tmp/snap.json sin-code eval diff --snapshot /tmp/snap-base.json --snapshot-b /tmp/snap-head.json +# Compile-and-run scorer (ponytail correctness.js analog) +sin-code eval run --dataset evals/coding.json \ + --scorer compile-and-run --language python \ + --self-check "assert fizzbuzz(15) == 'FizzBuzz'" + +# YAGNI mode: trivial one-liners accepted after compile-only +sin-code eval run --dataset evals/trivial.json \ + --scorer compile-and-run --language python --skip-test # Sanity-check the OTel exporter setup without a full eval sin-code trace doctor --exporter stdout --emit-sample-span @@ -841,4 +852,8 @@ same input on every CI run (caveman evals/README.md §3 promise: `Compare` runner, arm constructors, snapshot round-trip (issue #171). - `cmd/sin-code/internal/evalharness/snapshot.doc.md` — deterministic matrix + diff (issue #171). +- `cmd/sin-code/internal/evalharness/scorer.doc.md` — scorer interface + and built-in scorers (including `CompileAndRun`). +- `cmd/sin-code/internal/evalharness/runner_extras.doc.md` — code-block + extraction, per-language compile, and sandboxed self-check execution. diff --git a/CHANGELOG.md b/CHANGELOG.md index 674a0088..a95562d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -528,9 +528,17 @@ reports them; `debt check` gates them. `sin assets list|validate|show|import`. - **`internal/evalharness/`** — eval-driven development. `EvalSet` / `Run` / `Result` types, pluggable `Scorer`s (exact, contains-all, - success-flag, LLM-judge, composite), per-case timeout, JSONL run - history, and `Compare` for case-by-case regression detection with - `--fail-on-regress` as a CI gate. CLI: `sin eval run|list|compare`. + success-flag, LLM-judge, composite, **CompileAndRun**), per-case + timeout, JSONL run history, and `Compare` for case-by-case regression + detection with `--fail-on-regress` as a CI gate. CLI: + `sin eval run|list|compare`. +- **`CompileAndRun` scorer** (issue #181) — ponytail `correctness.js` + analog for SIN-Code. Extracts fenced code from model output, compiles + it (`go`/`python`/`javascript`/`bash`), and runs a sandboxed self-check. + Returns 1.0 only when compile + run pass; `skip_test` mode accepts + trivial one-liners after compile-only (YAGNI for tests). Wired into + `sin-code eval run` via `--scorer compile-and-run --language ` + and into Golden Datasets via `test_cases[].scorer`. - **`internal/dispatch/`** — turns loaded command and agent assets into executable actions. ECC-style placeholder substitution (`$ARGUMENTS`, `$1..$9`, `$@`, `${flag}`), `Dispatcher` routes diff --git a/cmd/sin-code/eval_cmd.go b/cmd/sin-code/eval_cmd.go index 2741bfaa..63d27246 100644 --- a/cmd/sin-code/eval_cmd.go +++ b/cmd/sin-code/eval_cmd.go @@ -97,6 +97,11 @@ func newEvalRunCmd() *cobra.Command { armsFlag string userSkill string modelPricing string + scorerType string + scorerLang string + scorerSelfChk string + scorerSkip bool + scorerBinary string ) cmd := &cobra.Command{ @@ -160,6 +165,15 @@ func newEvalRunCmd() *cobra.Command { RunOverride: stubRunOverride, } + var overrideScorer evalharness.Scorer + if scorerType != "" { + scorer, err := buildScorer(scorerType, scorerLang, scorerSelfChk, scorerSkip, scorerBinary) + if err != nil { + return fmt.Errorf("eval run: scorer: %w", err) + } + overrideScorer = scorer + } + runner, err := dataset.NewRunner(dataset.RunnerConfig{ ProfileName: profile, HeadlessMode: true, @@ -170,6 +184,7 @@ func newEvalRunCmd() *cobra.Command { if err != nil { return fmt.Errorf("eval run: new runner: %w", err) } + runner.Scorer = overrideScorer results, err := runner.RunDataset(ctx, ds) if err != nil { @@ -215,6 +230,11 @@ func newEvalRunCmd() *cobra.Command { cmd.Flags().StringVar(&armsFlag, "arm", "", "Comma-separated arm list (issue #171 comparator). Reserved tokens: baseline, terse, lazy_skill. Anything else is treated as a user-skill name. Empty = single-arm (legacy behavior).") cmd.Flags().StringVar(&userSkill, "skill", "skill-code-create", "User-skill arm name used when --arm contains a non-reserved token (issue #171).") cmd.Flags().StringVar(&modelPricing, "model-pricing", "stub", "Model-price entry from the comparator price book (issue #171). Default 'stub' = USD 0 in CI.") + cmd.Flags().StringVar(&scorerType, "scorer", "", "Override scorer type (compile_and_run|exact|contains)") + cmd.Flags().StringVar(&scorerLang, "language", "", "Language for --scorer compile-and-run (go|python|javascript|bash)") + cmd.Flags().StringVar(&scorerSelfChk, "self-check", "", "Self-check code for --scorer compile-and-run") + cmd.Flags().BoolVar(&scorerSkip, "skip-test", false, "YAGNI mode: accept compile-only for trivial one-liners") + cmd.Flags().StringVar(&scorerBinary, "scorer-binary", "", "Explicit compiler/interpreter for --scorer compile-and-run") cmd.MarkFlagRequired("dataset") return cmd @@ -671,4 +691,28 @@ func newEvalDiffCmd() *cobra.Command { _ = cmd.MarkFlagRequired("snapshot") _ = cmd.MarkFlagRequired("snapshot-b") return cmd +// buildScorer constructs an evalharness.Scorer from the CLI flags. +// Supported types: compile_and_run, exact, contains. +func buildScorer(typ, lang, selfCheck string, skipTest bool, binary string) (evalharness.Scorer, error) { + switch typ { + case "compile_and_run": + if lang == "" { + return nil, errors.New("--language is required for compile_and_run scorer") + } + if !evalharness.IsCompileAndRunLanguage(lang) { + return nil, fmt.Errorf("unsupported language %q", lang) + } + return evalharness.CompileAndRun{ + Language: lang, + SelfCheck: selfCheck, + SkipTest: skipTest, + Binary: binary, + }, nil + case "exact": + return evalharness.ExactMatch{}, nil + case "contains": + return evalharness.ContainsAll{}, nil + default: + return nil, fmt.Errorf("unsupported scorer %q", typ) + } } diff --git a/cmd/sin-code/internal/dataset/dataset.doc.md b/cmd/sin-code/internal/dataset/dataset.doc.md index c27f84de..b22781d4 100644 --- a/cmd/sin-code/internal/dataset/dataset.doc.md +++ b/cmd/sin-code/internal/dataset/dataset.doc.md @@ -15,7 +15,8 @@ A Golden Dataset is one JSON file under `evals/*.json`: "id": "auth_refactor", "prompt": "Refactor the auth system…", "constraints": {"must_use_tools": ["sin_edit"], "max_turns": 10}, - "expected": {"contains_keywords": ["argon2"], "min_quality": 0.8} + "expected": {"contains_keywords": ["argon2"], "min_quality": 0.8}, + "scorer": {"type": "compile_and_run", "language": "go", "self_check": "assert foo() == 1"} } ] } @@ -39,6 +40,9 @@ A Golden Dataset is one JSON file under `evals/*.json`: - `require_verify=true` requires a non-empty `verify_cmd`. - `min_quality` and `max_tokens` are bounded; out-of-range values fail validation at load time. +- `scorer` selects a post-hoc output scorer. `type: compile_and_run` + requires a supported `language` (`go`, `python`, `javascript`, `bash`) + and optionally `self_check`, `skip_test`, `timeout`, and `binary`. ## Without the issue's reference library diff --git a/cmd/sin-code/internal/dataset/dataset.go b/cmd/sin-code/internal/dataset/dataset.go index 27c120fe..7cf20f2e 100644 --- a/cmd/sin-code/internal/dataset/dataset.go +++ b/cmd/sin-code/internal/dataset/dataset.go @@ -56,6 +56,7 @@ type TestCase struct { Expected Expected `json:"expected,omitempty"` VerifyCmd string `json:"verify_cmd,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` + Scorer ScorerConfig `json:"scorer,omitempty"` } // Constraints holds hard rules. All fields are optional; Validate @@ -82,6 +83,19 @@ type Expected struct { CustomCriteria string `json:"custom_criteria,omitempty"` } +// ScorerConfig selects a scorer and its parameters. It is the dataset +// representation of an evalharness.Scorer; the runner resolves it when +// applying post-hoc output gates. +type ScorerConfig struct { + Type string `json:"type,omitempty"` // "compile_and_run" | "exact" | "contains" + Language string `json:"language,omitempty"` // language for compile_and_run + SelfCheck string `json:"self_check,omitempty"` // self-check code + SkipTest bool `json:"skip_test,omitempty"` // YAGNI mode for trivial one-liners + Timeout string `json:"timeout,omitempty"` // e.g. "30s" + Binary string `json:"binary,omitempty"` // explicit compiler/interpreter + PassThreshold float64 `json:"pass_threshold,omitempty"` // threshold for contains scorer +} + // LoadDataset reads + validates a single JSON file. Returns the // Dataset or a wrapped error including the absolute path so users // see which file failed without running the command twice. @@ -150,10 +164,76 @@ func (ds *Dataset) Validate() error { return fmt.Errorf("test_cases[%s]: invalid timeout %q: %v", tc.ID, tc.Constraints.Timeout, err) } } + if err := tc.Scorer.Validate(); err != nil { + return fmt.Errorf("test_cases[%s]: scorer: %w", tc.ID, err) + } } return nil } +// Validate checks a ScorerConfig for consistency. +func (sc ScorerConfig) Validate() error { + if sc.Type == "" { + return nil + } + switch sc.Type { + case "compile_and_run": + if !isCompileAndRunLanguage(sc.Language) { + return fmt.Errorf("unsupported compile_and_run language %q", sc.Language) + } + case "exact", "contains": + // no extra fields required + default: + return fmt.Errorf("unknown scorer type %q", sc.Type) + } + if sc.Timeout != "" { + if _, err := time.ParseDuration(sc.Timeout); err != nil { + return fmt.Errorf("invalid timeout %q: %v", sc.Timeout, err) + } + } + return nil +} + +// isCompileAndRunLanguage is a local copy of the evalharness allow-list +// to avoid importing the package here and creating a cycle. +func isCompileAndRunLanguage(lang string) bool { + switch lang { + case "go", "python", "javascript", "bash": + return true + } + return false +} + +// ToEvalharnessConfig converts ScorerConfig to the map[string]any format +// consumed by evalharness.ScorerFromConfig. +func (sc ScorerConfig) ToEvalharnessConfig() map[string]any { + if sc.Type == "" { + return nil + } + cfg := map[string]any{ + "type": sc.Type, + } + if sc.Language != "" { + cfg["language"] = sc.Language + } + if sc.SelfCheck != "" { + cfg["self_check"] = sc.SelfCheck + } + if sc.SkipTest { + cfg["skip_test"] = true + } + if sc.Timeout != "" { + cfg["timeout"] = sc.Timeout + } + if sc.Binary != "" { + cfg["binary"] = sc.Binary + } + if sc.PassThreshold != 0 { + cfg["pass_threshold"] = sc.PassThreshold + } + return cfg +} + // FilterByTag returns a copy of ds that contains only TestCases whose // Tags include tag (case-insensitive). Empty tag returns a shallow // copy with all cases — useful for "no filter" default UX. diff --git a/cmd/sin-code/internal/dataset/runner.go b/cmd/sin-code/internal/dataset/runner.go index 13969a1c..c3c24659 100644 --- a/cmd/sin-code/internal/dataset/runner.go +++ b/cmd/sin-code/internal/dataset/runner.go @@ -20,6 +20,7 @@ import ( "time" "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/agentloop" + "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/evalharness" "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/session" "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/verify" ) @@ -58,9 +59,10 @@ type RunnerConfig struct { // are created in the supplied store; if store is nil the runner // panics — that is a user bug, not a graceful degradation case. type Runner struct { - cfg RunnerConfig - loop *agentloop.Loop - store *session.Store + cfg RunnerConfig + loop *agentloop.Loop + store *session.Store + Scorer evalharness.Scorer // optional override scorer applied to every case } // NewRunner constructs a Runner. Zero-value fields get sensible @@ -155,7 +157,9 @@ func (r *Runner) RunCase(ctx context.Context, tc *TestCase) RunResult { res.VerifyPassed = loopRes.Verified res.FinalOutput = loopRes.Summary res.Success = loopRes.Verified - return r.applyRules(tc, &res) + r.applyRules(tc, &res) + r.applyScorer(tc, &res) + return res } // applyRules evaluates Constraints + Expected; if any rule fails the @@ -214,6 +218,39 @@ func (r *Runner) applyRules(tc *TestCase, res *RunResult) RunResult { return *res } +// applyScorer evaluates a configured or override scorer against the +// model's final output. If the scorer fails, res.Success flips to false. +func (r *Runner) applyScorer(tc *TestCase, res *RunResult) { + var scorer evalharness.Scorer + var cfg map[string]any + if r.Scorer != nil { + scorer = r.Scorer + } else if tc.Scorer.Type != "" { + cfg = tc.Scorer.ToEvalharnessConfig() + var err error + scorer, err = evalharness.ScorerFromConfig(cfg) + if err != nil { + res.Success = false + if res.Error == "" { + res.Error = "scorer: " + err.Error() + } + return + } + } + if scorer == nil { + return + } + score, passed, detail := scorer.Score(evalharness.EvalCase{ + ID: tc.ID, + Prompt: tc.Prompt, + Scorer: cfg, + }, evalharness.Output{Text: res.FinalOutput, Success: res.VerifyPassed}) + res.Success = res.Success && passed + if res.Error == "" && !passed { + res.Error = fmt.Sprintf("scorer: score=%.2f detail=%s", score, detail) + } +} + // contains is a small linear scan; test cases are tiny (≤ 100 tools). func contains(list []string, want string) bool { for _, s := range list { diff --git a/cmd/sin-code/internal/evalharness/runner.go b/cmd/sin-code/internal/evalharness/runner.go index 6e8f85c7..a05afb1c 100644 --- a/cmd/sin-code/internal/evalharness/runner.go +++ b/cmd/sin-code/internal/evalharness/runner.go @@ -25,9 +25,6 @@ func (r Runner) Execute(ctx context.Context, set EvalSet) (Run, error) { if r.Subject == nil { return Run{}, fmt.Errorf("evalharness: nil Subject") } - if r.Scorer == nil { - r.Scorer = SuccessFlag{} - } run := Run{ ID: fmt.Sprintf("%s-%d", set.Name, time.Now().Unix()), SetName: set.Name, @@ -62,7 +59,20 @@ func (r Runner) runCase(ctx context.Context, c EvalCase) Result { if err != nil { return Result{CaseID: c.ID, Score: 0, Weight: w, Passed: false, Duration: dur, Err: err.Error()} } - score, passed, detail := r.Scorer.Score(c, out) + scorer := r.Scorer + if c.Scorer != nil { + caseScorer, serr := ScorerFromConfig(c.Scorer) + if serr != nil { + return Result{CaseID: c.ID, Score: 0, Weight: w, Passed: false, Duration: dur, Err: serr.Error()} + } + if caseScorer != nil { + scorer = caseScorer + } + } + if scorer == nil { + scorer = SuccessFlag{} + } + score, passed, detail := scorer.Score(c, out) return Result{ CaseID: c.ID, Score: score, Weight: w, Passed: passed, Output: truncate(out.Text, 500), Detail: detail, Duration: dur, diff --git a/cmd/sin-code/internal/evalharness/runner_extras.doc.md b/cmd/sin-code/internal/evalharness/runner_extras.doc.md new file mode 100644 index 00000000..3e04d466 --- /dev/null +++ b/cmd/sin-code/internal/evalharness/runner_extras.doc.md @@ -0,0 +1,45 @@ +# evalharness/runner_extras.go — CompileAndRun helpers + +This file holds the implementation details for the `CompileAndRun` +scorer: fenced code-block extraction, per-language compile, and +sandboxed self-check execution. + +## `extractCodeBlock(s string) string` + +Returns the first Markdown fenced code block (` ```...``` `) in `s`, +stripping the info string (e.g. `python`) and surrounding whitespace. +If no block is found, returns `""`. + +## Compile step + +Each supported language has a dedicated `compile*` method that writes the +extracted code to a temporary file and runs the language's syntax checker: + +| Language | Compile command | +|---|---| +| `python` | `python3 -m py_compile ` | +| `go` | `go build -o solution .` in a temp module | +| `javascript` | `node --check ` | +| `bash` | `bash -n ` | + +## Run step + +The run methods append the `SelfCheck` code to the same temporary file +(or a sibling file for Go) and execute the language interpreter in a +sandboxed subprocess. The sandbox is the existing `internal/sandbox` +package: on Linux it uses Landlock, on other platforms it degrades to a +no-op with a warning while still enforcing timeouts. + +## Safety + +- A fresh temp directory is created for every compile and every run. +- `os.RemoveAll` cleans up after each step. +- The sandbox policy denies network access and restricts writes to the + temp directory. +- `context.WithTimeout` bounds both compile and run. + +## Related files + +- `scorer.go` — `CompileAndRun` type and `Score` entry point +- `types.go` — `EvalCase.Scorer` config field +- `internal/sandbox/` — OS-level isolation for subprocesses diff --git a/cmd/sin-code/internal/evalharness/runner_extras.go b/cmd/sin-code/internal/evalharness/runner_extras.go new file mode 100644 index 00000000..d78639e3 --- /dev/null +++ b/cmd/sin-code/internal/evalharness/runner_extras.go @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: MIT +// Purpose: helpers for CompileAndRun scorer — code-block extraction, +// per-language compile, and sandboxed self-check execution. +// Docs: runner_extras.doc.md +package evalharness + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/sandbox" +) + +// codeBlockRe matches the first fenced ```...``` block. It tolerates an +// info string (e.g. ```python) and captures the inner content. +var codeBlockRe = regexp.MustCompile("(?s)```(?:\\w+)?\\n?(.*?)\\n?```") + +// extractCodeBlock returns the first fenced code block in s, or "" if none. +func extractCodeBlock(s string) string { + m := codeBlockRe.FindStringSubmatch(s) + if len(m) < 2 { + return "" + } + return strings.TrimSpace(m[1]) +} + +// compile validates the code by spawning the language's syntax checker. +// It uses the sandbox package so untrusted code runs with limited access. +func (c CompileAndRun) compile(code string, timeout time.Duration) error { + tmpDir, err := os.MkdirTemp("", "sin-eval-compile-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + policy := sandbox.DefaultPolicy(tmpDir, tmpDir) + policy.Timeout = timeout + + switch c.Language { + case "python": + return c.compilePython(ctx, policy, tmpDir, code) + case "go": + return c.compileGo(ctx, policy, tmpDir, code) + case "javascript": + return c.compileJavaScript(ctx, policy, tmpDir, code) + case "bash": + return c.compileBash(ctx, policy, tmpDir, code) + default: + return fmt.Errorf("unsupported language: %q", c.Language) + } +} + +// run executes the code with selfCheck appended, using the sandbox. +// For Go the self-check is written into a separate file in the same package. +func (c CompileAndRun) run(code, selfCheck string, timeout time.Duration) error { + tmpDir, err := os.MkdirTemp("", "sin-eval-run-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + policy := sandbox.DefaultPolicy(tmpDir, tmpDir) + policy.Timeout = timeout + + switch c.Language { + case "python": + return c.runPython(ctx, policy, tmpDir, code, selfCheck) + case "go": + return c.runGo(ctx, policy, tmpDir, code, selfCheck) + case "javascript": + return c.runJavaScript(ctx, policy, tmpDir, code, selfCheck) + case "bash": + return c.runBash(ctx, policy, tmpDir, code, selfCheck) + default: + return fmt.Errorf("unsupported language: %q", c.Language) + } +} + +// compilePython runs python3 -m py_compile on the source file. +func (c CompileAndRun) compilePython(ctx context.Context, policy sandbox.Policy, tmpDir, code string) error { + src := filepath.Join(tmpDir, "solution.py") + if err := os.WriteFile(src, []byte(code), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("python3", "python"), "-m", "py_compile", src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// runPython appends the self-check and runs the combined file. +func (c CompileAndRun) runPython(ctx context.Context, policy sandbox.Policy, tmpDir, code, selfCheck string) error { + src := filepath.Join(tmpDir, "solution.py") + full := code + "\n" + selfCheck + "\n" + if err := os.WriteFile(src, []byte(full), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("python3", "python"), src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// compileGo creates a tiny module, writes the code, and builds it. +func (c CompileAndRun) compileGo(ctx context.Context, policy sandbox.Policy, tmpDir, code string) error { + if err := initGoModule(tmpDir); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(tmpDir, "solution.go"), []byte(code), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("go", "go"), "build", "-o", "solution", ".") + if err != nil { + return err + } + cmd.Dir = tmpDir + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// runGo appends the self-check as a separate main file and runs the module. +func (c CompileAndRun) runGo(ctx context.Context, policy sandbox.Policy, tmpDir, code, selfCheck string) error { + if err := initGoModule(tmpDir); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(tmpDir, "solution.go"), []byte(code), 0600); err != nil { + return err + } + mainFile := filepath.Join(tmpDir, "main.go") + mainBody := selfCheck + if !strings.HasPrefix(strings.TrimSpace(selfCheck), "package ") { + mainBody = "package main\n" + mainBody + } + if err := os.WriteFile(mainFile, []byte(mainBody), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("go", "go"), "run", ".") + if err != nil { + return err + } + cmd.Dir = tmpDir + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// compileJavaScript runs node --check on the source file. +func (c CompileAndRun) compileJavaScript(ctx context.Context, policy sandbox.Policy, tmpDir, code string) error { + src := filepath.Join(tmpDir, "solution.js") + if err := os.WriteFile(src, []byte(code), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("node", "node"), "--check", src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// runJavaScript appends the self-check and runs the combined file. +func (c CompileAndRun) runJavaScript(ctx context.Context, policy sandbox.Policy, tmpDir, code, selfCheck string) error { + src := filepath.Join(tmpDir, "solution.js") + full := code + "\n" + selfCheck + "\n" + if err := os.WriteFile(src, []byte(full), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("node", "node"), src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// compileBash runs bash -n for syntax checking. +func (c CompileAndRun) compileBash(ctx context.Context, policy sandbox.Policy, tmpDir, code string) error { + src := filepath.Join(tmpDir, "solution.sh") + if err := os.WriteFile(src, []byte(code), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("bash", "bash"), "-n", src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// runBash appends the self-check and runs the combined script. +func (c CompileAndRun) runBash(ctx context.Context, policy sandbox.Policy, tmpDir, code, selfCheck string) error { + src := filepath.Join(tmpDir, "solution.sh") + full := code + "\n" + selfCheck + "\n" + if err := os.WriteFile(src, []byte(full), 0600); err != nil { + return err + } + cmd, res, err := sandbox.Command(ctx, policy, c.interpreter("bash", "bash"), src) + if err != nil { + return err + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, sandboxOutput(res, out)) + } + return nil +} + +// interpreter returns the explicit Binary if set, otherwise tries the +// preferred names and falls back to the bare command name. +func (c CompileAndRun) interpreter(preferred, fallback string) string { + if c.Binary != "" { + return c.Binary + } + if _, err := exec.LookPath(preferred); err == nil { + return preferred + } + return fallback +} + +// initGoModule creates a minimal go.mod in dir so `go build` / `go run` +// work without an enclosing module. +func initGoModule(dir string) error { + mod := filepath.Join(dir, "go.mod") + return os.WriteFile(mod, []byte("module sin\n\ngo 1.23\n"), 0600) +} + +// sandboxOutput decorates command output with the sandbox mechanism. +func sandboxOutput(res sandbox.Result, out []byte) string { + s := strings.TrimSpace(string(out)) + if s == "" { + s = "" + } + if res.Warning != "" { + return fmt.Sprintf("%s (sandbox: %s)", s, res.Warning) + } + return fmt.Sprintf("%s (sandbox: %s)", s, res.Mechanism) +} diff --git a/cmd/sin-code/internal/evalharness/scorer.doc.md b/cmd/sin-code/internal/evalharness/scorer.doc.md index ded2e0c1..82069e70 100644 --- a/cmd/sin-code/internal/evalharness/scorer.doc.md +++ b/cmd/sin-code/internal/evalharness/scorer.doc.md @@ -13,6 +13,26 @@ implementation it uses. | `SuccessFlag` | use the subject's own pass/fail boolean (verify gate) | | `LLMJudge` | delegate to a model via the `Judge` function field | | `Composite` | weighted average of multiple scorers | +| `CompileAndRun` | extract, compile, and run a self-check on generated code | + +## `CompileAndRun` + +`CompileAndRun` is the SIN-Code equivalent of ponytail's `correctness.js` +gate. It extracts the first fenced code block from the model output, +compiles it, and runs a self-check in a sandboxed subprocess. + +Supported languages: `go`, `python`, `javascript`, `bash`. + +Score semantics: +- No code block → `0.0` / fail. +- Compile failure → `0.0` / fail. +- `SkipTest=true` and compile passes → `1.0` / pass (YAGNI for trivial + one-liners). +- Compile passes but no `SelfCheck` and `SkipTest=false` → `0.5` / fail. +- Compile + self-check pass → `1.0` / pass. + +The default timeout is 30 seconds; set `Timeout` to override. +`Binary` optionally pins the compiler/interpreter executable. ## `LLMJudge` and circularity diff --git a/cmd/sin-code/internal/evalharness/scorer.go b/cmd/sin-code/internal/evalharness/scorer.go index 660ab7cb..eaa960ea 100644 --- a/cmd/sin-code/internal/evalharness/scorer.go +++ b/cmd/sin-code/internal/evalharness/scorer.go @@ -1,16 +1,24 @@ // SPDX-License-Identifier: MIT // Purpose: Scorer implementations — exact, contains, success, LLM-judge, -// composite. Each Scorer turns a (case, output) into (score, passed, detail). +// composite, compile-and-run. Each Scorer turns a (case, output) into +// (score, passed, detail). // Docs: scorer.doc.md package evalharness -import "strings" +import ( + "fmt" + "strings" + "time" +) // Scorer turns a case + output into a 0..1 score and a pass/fail. type Scorer interface { Score(c EvalCase, out Output) (score float64, passed bool, detail string) } +// compileRunTimeout is the default wall-clock budget for compile + run. +const compileRunTimeout = 30 * time.Second + // ExactMatch scores 1.0 when output equals the expected string (trimmed). type ExactMatch struct{} @@ -124,3 +132,113 @@ func boolScore(b bool) float64 { } return 0 } + +// CompileAndRun extracts a fenced code block from the model output, +// compiles it, and runs a self-check. It returns 1.0 only when both +// compile and self-check pass. This is the SIN-Code equivalent of +// ponytail's correctness.js gate (ponytail/benchmarks/README.md:69-79). +// +// If SkipTest is true, a trivial one-liner is accepted after compile +// without a self-check (YAGNI for tests). If SelfCheck is empty and +// SkipTest is false, the score is 0.5 because compile alone is not +// enough to prove correctness. +type CompileAndRun struct { + Language string // "go" | "python" | "javascript" | "bash" + SelfCheck string // code appended to the extracted block before execution + Timeout time.Duration // default 30s + Binary string // optional explicit compiler/interpreter path + SkipTest bool // true for trivial one-liners that need no test +} + +// Score implements Scorer. +func (c CompileAndRun) Score(ca EvalCase, out Output) (float64, bool, string) { + code := extractCodeBlock(out.Text) + if code == "" { + return 0, false, "no code block in output" + } + + timeout := c.Timeout + if timeout <= 0 { + timeout = compileRunTimeout + } + + // Step 1: compile. + if err := c.compile(code, timeout); err != nil { + return 0, false, "compile failed: " + err.Error() + } + + // Step 2: run self-check (unless YAGNI applies). + if c.SkipTest { + return 1, true, "compile passed, trivial one-liner" + } + if c.SelfCheck == "" { + return 0.5, false, "compile passed but no self-check (set SkipTest for trivial one-liners)" + } + if err := c.run(code, c.SelfCheck, timeout); err != nil { + return 0, false, "self-check failed: " + err.Error() + } + return 1, true, "compile + self-check passed" +} + +// compileAndRunLanguages lists the languages the scorer currently supports. +var compileAndRunLanguages = map[string]struct{}{ + "go": {}, + "python": {}, + "javascript": {}, + "bash": {}, +} + +// IsCompileAndRunLanguage reports whether lang is supported. +func IsCompileAndRunLanguage(lang string) bool { + _, ok := compileAndRunLanguages[lang] + return ok +} + +// ScorerFromConfig builds a Scorer from a map[string]any configuration. +// It returns (nil, nil) when cfg is empty or no recognized type is given, +// so the caller can fall back to SuccessFlag. +func ScorerFromConfig(cfg map[string]any) (Scorer, error) { + if len(cfg) == 0 { + return nil, nil + } + typ, _ := cfg["type"].(string) + switch typ { + case "exact": + return ExactMatch{}, nil + case "contains": + th := float64(0) + if v, ok := cfg["pass_threshold"].(float64); ok { + th = v + } + return ContainsAll{PassThreshold: th}, nil + case "compile_and_run": + lang, _ := cfg["language"].(string) + if lang == "" { + lang, _ = cfg["lang"].(string) + } + if !IsCompileAndRunLanguage(lang) { + return nil, fmt.Errorf("compile_and_run: unsupported language %q", lang) + } + selfCheck, _ := cfg["self_check"].(string) + skip := false + if v, ok := cfg["skip_test"].(bool); ok { + skip = v + } + timeout := time.Duration(0) + if v, ok := cfg["timeout"].(string); ok && v != "" { + if d, err := time.ParseDuration(v); err == nil { + timeout = d + } + } + binary, _ := cfg["binary"].(string) + return CompileAndRun{ + Language: lang, + SelfCheck: selfCheck, + Timeout: timeout, + Binary: binary, + SkipTest: skip, + }, nil + default: + return nil, fmt.Errorf("unknown scorer type %q", typ) + } +} diff --git a/cmd/sin-code/internal/evalharness/scorer_compile_run_test.go b/cmd/sin-code/internal/evalharness/scorer_compile_run_test.go new file mode 100644 index 00000000..08626525 --- /dev/null +++ b/cmd/sin-code/internal/evalharness/scorer_compile_run_test.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: MIT +// Purpose: unit tests for CompileAndRun scorer. Requires Python 3 on +// PATH; otherwise the Python cases are skipped. Sandbox is used for +// every execution, so these tests exercise the same path as the CLI. +// Docs: scorer_compile_run_test.doc.md +package evalharness + +import ( + "os/exec" + "strings" + "testing" + "time" +) + +func pythonInterpreter() string { + for _, name := range []string{"python3", "python"} { + if _, err := exec.LookPath(name); err == nil { + return name + } + } + return "" +} + +func TestCompileAndRun_PythonFizzBuzz(t *testing.T) { + if pythonInterpreter() == "" { + t.Skip("python not available on PATH") + } + code := "```python\ndef fizzbuzz(n):\n if n % 15 == 0:\n return 'FizzBuzz'\n if n % 3 == 0:\n return 'Fizz'\n if n % 5 == 0:\n return 'Buzz'\n return str(n)\n```" + scorer := CompileAndRun{ + Language: "python", + SelfCheck: "assert fizzbuzz(15) == 'FizzBuzz'\nassert fizzbuzz(3) == 'Fizz'\nassert fizzbuzz(5) == 'Buzz'\nassert fizzbuzz(7) == '7'", + Timeout: 30 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 1.0 || !passed { + t.Fatalf("expected 1.0 pass, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "compile + self-check passed") { + t.Fatalf("unexpected detail: %s", detail) + } +} + +func TestCompileAndRun_PythonSyntaxError(t *testing.T) { + if pythonInterpreter() == "" { + t.Skip("python not available on PATH") + } + code := "```python\ndef fizzbuzz(n:\n return n\n```" + scorer := CompileAndRun{ + Language: "python", + SelfCheck: "assert fizzbuzz(1) == 1", + Timeout: 30 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 0.0 || passed { + t.Fatalf("expected 0.0 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "compile failed") { + t.Fatalf("expected compile failure, got detail=%s", detail) + } +} + +func TestCompileAndRun_SelfCheckFailure(t *testing.T) { + if pythonInterpreter() == "" { + t.Skip("python not available on PATH") + } + code := "```python\ndef fizzbuzz(n):\n return 'nope'\n```" + scorer := CompileAndRun{ + Language: "python", + SelfCheck: "assert fizzbuzz(15) == 'FizzBuzz'", + Timeout: 30 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 0.0 || passed { + t.Fatalf("expected 0.0 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "self-check failed") { + t.Fatalf("expected self-check failure, got detail=%s", detail) + } +} + +func TestCompileAndRun_SkipTestTrivialOneLiner(t *testing.T) { + if pythonInterpreter() == "" { + t.Skip("python not available on PATH") + } + code := "```python\ndict(zip(['a'], [1]))\n```" + scorer := CompileAndRun{ + Language: "python", + SkipTest: true, + Timeout: 30 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 1.0 || !passed { + t.Fatalf("expected 1.0 pass, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "trivial one-liner") { + t.Fatalf("expected trivial one-liner detail, got %s", detail) + } +} + +func TestCompileAndRun_NoSelfCheckScoresHalf(t *testing.T) { + if pythonInterpreter() == "" { + t.Skip("python not available on PATH") + } + code := "```python\ndef fizzbuzz(n):\n return str(n)\n```" + scorer := CompileAndRun{ + Language: "python", + Timeout: 30 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 0.5 || passed { + t.Fatalf("expected 0.5 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "no self-check") { + t.Fatalf("expected no self-check detail, got %s", detail) + } +} + +func TestCompileAndRun_NoCodeBlock(t *testing.T) { + scorer := CompileAndRun{Language: "python"} + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: "just some prose"}) + if score != 0.0 || passed { + t.Fatalf("expected 0.0 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "no code block") { + t.Fatalf("expected no code block detail, got %s", detail) + } +} + +func TestExtractCodeBlock(t *testing.T) { + cases := []struct { + in, want string + }{ + {"```python\nprint('hi')\n```", "print('hi')"}, + {"pre\n```go\npackage main\n```\npost", "package main"}, + {"no block here", ""}, + {"```\n\n```", ""}, + {"```bash\necho hi\n```", "echo hi"}, + } + for _, tc := range cases { + got := extractCodeBlock(tc.in) + if got != tc.want { + t.Errorf("extractCodeBlock(%q) = %q; want %q", tc.in, got, tc.want) + } + } +} + +func TestCompileAndRun_LanguageUnsupported(t *testing.T) { + scorer := CompileAndRun{Language: "rust"} + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: "```rust\nfn main(){}\n```"}) + if score != 0.0 || passed { + t.Fatalf("expected 0.0 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "unsupported language") { + t.Fatalf("expected unsupported language detail, got %s", detail) + } +} + +func TestIsCompileAndRunLanguage(t *testing.T) { + for _, lang := range []string{"go", "python", "javascript", "bash"} { + if !IsCompileAndRunLanguage(lang) { + t.Errorf("expected %q to be supported", lang) + } + } + if IsCompileAndRunLanguage("rust") { + t.Error("expected rust to be unsupported") + } +} + +func TestCompileAndRun_GoSkipTest(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not available on PATH") + } + code := "```go\npackage main\n\nimport \"fmt\"\n\nfunc main() {\n fmt.Println(\"hello\")\n}\n```" + scorer := CompileAndRun{ + Language: "go", + SkipTest: true, + Timeout: 60 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 1.0 || !passed { + t.Fatalf("expected 1.0 pass, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "trivial one-liner") && !strings.Contains(detail, "compile passed") { + t.Fatalf("unexpected detail: %s", detail) + } +} + +func TestCompileAndRun_GoSelfCheck(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not available on PATH") + } + code := "```go\npackage main\n\nfunc answer() int { return 42 }\n\nfunc main() {}\n```" + scorer := CompileAndRun{ + Language: "go", + SelfCheck: "func init() { if answer() != 42 { panic(\"wrong\") } }", + Timeout: 60 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 1.0 || !passed { + t.Fatalf("expected 1.0 pass, got score=%.2f pass=%v detail=%s", score, passed, detail) + } +} + +func TestCompileAndRun_GoSelfCheckFail(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not available on PATH") + } + code := "```go\npackage main\n\nfunc answer() int { return 42 }\n\nfunc main() {}\n```" + scorer := CompileAndRun{ + Language: "go", + SelfCheck: "func init() { if answer() != 7 { panic(\"wrong\") } }", + Timeout: 60 * time.Second, + } + score, passed, detail := scorer.Score(EvalCase{}, Output{Text: code}) + if score != 0.0 || passed { + t.Fatalf("expected 0.0 fail, got score=%.2f pass=%v detail=%s", score, passed, detail) + } + if !strings.Contains(detail, "self-check failed") { + t.Fatalf("expected self-check failure, got detail=%s", detail) + } +} + +func TestScorerFromConfig(t *testing.T) { + scorer, err := ScorerFromConfig(map[string]any{ + "type": "compile_and_run", + "language": "python", + "self_check": "assert True", + "skip_test": true, + }) + if err != nil { + t.Fatalf("ScorerFromConfig: %v", err) + } + car, ok := scorer.(CompileAndRun) + if !ok { + t.Fatalf("expected CompileAndRun, got %T", scorer) + } + if car.Language != "python" || car.SelfCheck != "assert True" || !car.SkipTest { + t.Fatalf("unexpected scorer: %+v", car) + } + + if _, err := ScorerFromConfig(map[string]any{"type": "compile_and_run", "language": "rust"}); err == nil { + t.Fatal("expected error for unsupported language") + } + if _, err := ScorerFromConfig(map[string]any{"type": "unknown"}); err == nil { + t.Fatal("expected error for unknown scorer type") + } +} diff --git a/cmd/sin-code/internal/evalharness/types.go b/cmd/sin-code/internal/evalharness/types.go index 9e010a9b..0063dec1 100644 --- a/cmd/sin-code/internal/evalharness/types.go +++ b/cmd/sin-code/internal/evalharness/types.go @@ -18,6 +18,12 @@ type EvalCase struct { Tags []string `json:"tags,omitempty"` // e.g. ["go","refactor"] Weight float64 `json:"weight,omitempty"` // default 1.0 Meta map[string]string `json:"meta,omitempty"` + + // Scorer is an optional machine-readable scorer configuration. + // Recognized keys depend on the scorer type; e.g.: + // {"type": "compile_and_run", "language": "python", "self_check": "..."} + // When empty the runner defaults to SuccessFlag. + Scorer map[string]any `json:"scorer,omitempty"` } // EvalSet is a named collection of cases.