evals: Go-native eval suites that gate CI

CHANGELOG.md

@@ -17,6 +17,21 @@ and [`agents/peggy/CHANGELOG.md`](agents/peggy/CHANGELOG.md).
 
 ## Unreleased
 
+- **Evals-as-gates harness (`evals`, `glue eval`).** A new Go-native eval
+  harness: define a `Suite` of `Case`s, run them against a `*glue.Agent`
+  with `Runner`, score each with `Scorer`s (built-ins: `Contains`,
+  `NotContains`, `Regex`, `Equals`, `Command` — the coding gate that runs
+  a build/test and scores on exit 0 — `Judge` LLM-as-judge, and
+  `ScorerFunc`), and gate CI with `Report.Gate(minPassRate)`. Suites can
+  also be declarative JSON (`SuiteSpec`, unknown fields/scorer-types
+  rejected) and run via `glue eval <suite.json> [--coding] [--min-pass-rate]
+  [--parallel] [--json]`, which builds the agent under test plus a
+  separate tool-less judge agent and exits non-zero when the gate fails.
+  This is the missing feedback loop for the v1.13.0 harness work.
+  Borrowed from Vercel's Eve framework; see
+  [ADR-0019](docs/adr/0019-evals-harness.md), [docs/evals.md](docs/evals.md),
+  and [examples/evals/smoke.json](examples/evals/smoke.json). (#360)
+
 - **TUI: `/` picker shows every command; `@` picker gains scroll
   indicators (`cmd/glue/tui`).** The slash-command popup previously
   reused the file picker's 8-row scroll window with no indicator, so a

cmd/glue/evalcmd.go

@@ -0,0 +1,172 @@
+// The "glue eval" subcommand: run a declarative eval suite against an
+// agent and gate (exit non-zero) on the aggregate pass rate. This is the
+// CI-facing front end to the evals package.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/erain/glue"
+	"github.com/erain/glue/evals"
+)
+
+func evalCommand(ctx context.Context, args []string, stdout, stderr io.Writer, newProvider providerFactory) error {
+	flags := flag.NewFlagSet("glue eval", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+
+	provider := flags.String("provider", defaultProvider, "provider name: codex, gemini, nvidia, or openrouter")
+	model := flags.String("model", "", "model id for the agent under test (default: provider default)")
+	judgeModel := flags.String("judge-model", "", "model id for judge scorers (default: --model)")
+	coding := flags.Bool("coding", false, "enable local coding tools for the agent under test")
+	workDir := flags.String("work", ".", "working directory for coding tools and command scorers")
+	storeDir := flags.String("store", ".glue/evals", "session store directory for eval runs")
+	minPassRate := flags.Float64("min-pass-rate", 1.0, "minimum suite pass rate to exit zero (0..1)")
+	parallel := flags.Int("parallel", 1, "max cases to run concurrently")
+	jsonOut := flags.Bool("json", false, "emit the report as JSON instead of text")
+	var allowedBinaries repeatedStrings
+	flags.Var(&allowedBinaries, "allow-binary", "allowed shell_exec binary basename for --coding; repeatable")
+	var envs envFiles
+	flags.Var(&envs, "env", "env file path; repeatable")
+
+	// The suite path is the first argument; flags follow it (Go's flag
+	// parser stops at the first positional, so we split it off by hand).
+	if len(args) == 0 || strings.HasPrefix(args[0], "-") {
+		return fmt.Errorf("usage: glue eval <suite.json> [flags]")
+	}
+	suitePath := args[0]
+	if err := flags.Parse(args[1:]); err != nil {
+		return err
+	}
+	if flags.NArg() != 0 {
+		return fmt.Errorf("unexpected arguments after suite path: %s", strings.Join(flags.Args(), " "))
+	}
+
+	if err := loadEnvFiles(envs); err != nil {
+		return err
+	}
+	providerName, effectiveModel, err := resolveProvider(*provider, *model)
+	if err != nil {
+		return err
+	}
+
+	// Agent under test: optional coding tools, auto-approving permission
+	// (eval runs are unattended — run them on a feature branch / worktree).
+	var tools []glue.Tool
+	if *coding {
+		tools, _, err = buildCodingTools(codingFlagConfig{
+			Enabled:         true,
+			WorkDir:         *workDir,
+			AllowedBinaries: append([]string(nil), allowedBinaries...),
+			AllowOverwrite:  true,
+		})
+		if err != nil {
+			return err
+		}
+	}
+	agent, err := newAgent(newProvider, agentConfig{
+		Provider:   providerName,
+		Model:      effectiveModel,
+		StoreDir:   *storeDir,
+		WorkDir:    *workDir,
+		Tools:      tools,
+		Permission: yoloPermission{},
+		Coding:     *coding,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Judge agent: independent of the agent under test, no tools, so an
+	// LLM-as-judge scorer never grades its own work.
+	judge, err := newAgent(newProvider, agentConfig{
+		Provider: providerName,
+		Model:    judgeModelOr(*judgeModel, effectiveModel),
+		StoreDir: *storeDir,
+	})
+	if err != nil {
+		return err
+	}
+
+	data, err := os.ReadFile(suitePath)
+	if err != nil {
+		return fmt.Errorf("read suite: %w", err)
+	}
+	suite, err := evals.ParseSuite(data, evals.BuildOptions{JudgeAgent: judge})
+	if err != nil {
+		return err
+	}
+
+	report, err := evals.Runner{Agent: agent, Parallelism: *parallel}.Run(ctx, suite)
+	if err != nil {
+		return err
+	}
+
+	if *jsonOut {
+		if err := writeJSONReport(stdout, report); err != nil {
+			return err
+		}
+	} else {
+		report.WriteText(stdout)
+	}
+
+	// Gate: a non-nil error here exits the process non-zero (CI gate).
+	return report.Gate(*minPassRate)
+}
+
+func judgeModelOr(judge, fallback string) string {
+	if strings.TrimSpace(judge) != "" {
+		return judge
+	}
+	return fallback
+}
+
+// writeJSONReport emits a stable, machine-readable view of the report for
+// CI consumers.
+func writeJSONReport(w io.Writer, r evals.Report) error {
+	type scoreJSON struct {
+		Scorer string  `json:"scorer"`
+		Value  float64 `json:"value"`
+		Reason string  `json:"reason,omitempty"`
+	}
+	type caseJSON struct {
+		Name   string      `json:"name"`
+		Passed bool        `json:"passed"`
+		Score  float64     `json:"score"`
+		Error  string      `json:"error,omitempty"`
+		Scores []scoreJSON `json:"scores,omitempty"`
+	}
+	out := struct {
+		Suite     string     `json:"suite"`
+		Threshold float64    `json:"threshold"`
+		Passed    int        `json:"passed"`
+		Total     int        `json:"total"`
+		PassRate  float64    `json:"pass_rate"`
+		Cases     []caseJSON `json:"cases"`
+	}{
+		Suite:     r.Suite,
+		Threshold: r.PassThreshold,
+		Passed:    r.Passed(),
+		Total:     r.Total(),
+		PassRate:  r.PassRate(),
+	}
+	for _, c := range r.Cases {
+		cj := caseJSON{Name: c.Name, Passed: c.Passed, Score: c.Score}
+		if c.Err != nil {
+			cj.Error = c.Err.Error()
+		}
+		for _, s := range c.Scores {
+			cj.Scores = append(cj.Scores, scoreJSON{Scorer: s.Scorer, Value: s.Value, Reason: s.Reason})
+		}
+		out.Cases = append(out.Cases, cj)
+	}
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(out)
+}

cmd/glue/evalcmd_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/erain/glue"
+)
+
+// fixedProvider always replies with the same text, regardless of how many
+// times it is called — convenient for eval tests where the agent under
+// test and the judge agent share one provider instance.
+type fixedProvider struct{ text string }
+
+func (p fixedProvider) Stream(_ context.Context, _ glue.ProviderRequest) (<-chan glue.ProviderEvent, error) {
+	ch := make(chan glue.ProviderEvent, 3)
+	ch <- glue.ProviderEvent{Type: glue.ProviderEventStart}
+	ch <- glue.ProviderEvent{Type: glue.ProviderEventTextDelta, Delta: p.text}
+	ch <- glue.ProviderEvent{Type: glue.ProviderEventDone, Message: &glue.Message{
+		Role:       glue.MessageRoleAssistant,
+		Content:    []glue.ContentPart{{Type: glue.ContentTypeText, Text: p.text}},
+		StopReason: glue.StopReasonStop,
+	}}
+	close(ch)
+	return ch, nil
+}
+
+func writeSuite(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "suite.json")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+const twoCaseSuite = `{
+  "name": "smoke",
+  "cases": [
+    {"name": "ok",   "prompt": "say hello", "scorers": [{"type": "contains", "values": ["hello"]}]},
+    {"name": "fail", "prompt": "say hello", "scorers": [{"type": "contains", "values": ["absent"]}]}
+  ]
+}`
+
+func runEval(t *testing.T, provider glue.Provider, args ...string) (code int, stdout, stderr string) {
+	t.Helper()
+	var out, errb bytes.Buffer
+	full := append([]string{"eval"}, args...)
+	code = runCLI(context.Background(), full, &out, &errb, fakeFactory(provider))
+	return code, out.String(), errb.String()
+}
+
+func TestEvalGateFailsBelowThreshold(t *testing.T) {
+	suite := writeSuite(t, twoCaseSuite)
+	store := t.TempDir()
+
+	// One of two cases passes; requiring 100% must exit non-zero.
+	code, stdout, stderr := runEval(t, fixedProvider{text: "hello!"},
+		suite, "--store", store, "--min-pass-rate", "1")
+	if code != 1 {
+		t.Fatalf("exit code = %d, want 1 (stderr: %s)", code, stderr)
+	}
+	if !strings.Contains(stdout, "[PASS] ok") || !strings.Contains(stdout, "[FAIL] fail") {
+		t.Errorf("report missing per-case marks:\n%s", stdout)
+	}
+	if !strings.Contains(stderr, "gate failed") {
+		t.Errorf("expected a gate-failed message on stderr, got: %s", stderr)
+	}
+}
+
+func TestEvalGatePassesAtRelaxedThreshold(t *testing.T) {
+	suite := writeSuite(t, twoCaseSuite)
+	code, stdout, stderr := runEval(t, fixedProvider{text: "hello!"},
+		suite, "--store", t.TempDir(), "--min-pass-rate", "0.5")
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0 (stderr: %s)", code, stderr)
+	}
+	if !strings.Contains(stdout, "1/2 passed") {
+		t.Errorf("report missing totals line:\n%s", stdout)
+	}
+}
+
+func TestEvalJSONOutput(t *testing.T) {
+	suite := writeSuite(t, twoCaseSuite)
+	code, stdout, _ := runEval(t, fixedProvider{text: "hello!"},
+		suite, "--store", t.TempDir(), "--min-pass-rate", "0", "--json")
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0", code)
+	}
+	if !strings.Contains(stdout, `"suite": "smoke"`) || !strings.Contains(stdout, `"pass_rate"`) {
+		t.Errorf("expected JSON report, got:\n%s", stdout)
+	}
+}
+
+func TestEvalRejectsMissingSuiteArg(t *testing.T) {
+	code, _, stderr := runEval(t, fixedProvider{text: "x"})
+	if code != 1 {
+		t.Fatalf("exit code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr, "usage: glue eval") {
+		t.Errorf("expected usage error, got: %s", stderr)
+	}
+}

cmd/glue/main.go

@@ -133,6 +133,12 @@ func runCLIWithDeps(ctx context.Context, args []string, stdin io.Reader, stdout
 			fmt.Fprintln(stderr, err)
 		}
 		return code
+	case "eval":
+		if err := evalCommand(ctx, args[1:], stdout, stderr, newProvider); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		return 0
 	default:
 		fmt.Fprintf(stderr, "unknown command %q\n\n", args[0])
 		printUsage(stderr)
@@ -245,6 +251,7 @@ func printUsage(w io.Writer) {
   glue run --prompt <text> [--provider <name>] [--id <id>] [--model <model>] [--store <dir>] [--work <dir>] [--coding] [--env <path>]
   glue goal "<objective>" [--provider <name>] [--model <model>] [--store <dir>] [--work <dir>] [--coding] [--yolo] [--worktree] [--max-iterations <n>] [--budget <tokens>] [--env <path>]
   glue goal --resume [<id>] | --list [--store <dir>]
+  glue eval <suite.json> [--provider <name>] [--model <model>] [--judge-model <model>] [--coding] [--work <dir>] [--min-pass-rate <0..1>] [--parallel <n>] [--json] [--env <path>]
   glue serve [--provider <name>] [--listen 127.0.0.1:0] [--metadata <path>] [--model <model>] [--store <dir>] [--work <dir>] [--coding] [--env <path>]
   glue connect --prompt <text> [--id <id>] [--metadata <path>] [--base-url <url>] [--token <token>]
   glue connect --skill <name> [--arg key=value] [--id <id>] [--metadata <path>] [--base-url <url>] [--token <token>]
@@ -267,6 +274,7 @@ func printUsage(w io.Writer) {
 Commands:
   run      Run a local agent on any registered provider, optionally with coding tools.
   goal     Run an autonomous goal loop headlessly (plan → make → verify until done or a guardrail trips); exit code reflects the outcome.
+  eval     Run a declarative eval suite (JSON) against an agent and gate on the pass rate; exits non-zero when the gate fails.
   serve    Start a local HTTP+SSE daemon for Glue sessions, optionally with coding tools.
   connect  Start a daemon prompt/skill run, or inspect daemon status/tools/skills/roles/MCP/recall surfaces.
   version  Print the binary's version, git revision, and Go toolchain (also: --version, -v).