fix: wire 8 new eval datasets into CI, update CHANGELOG

.github/workflows/eval-n8n.yml

@@ -25,7 +25,7 @@ on:
       datasets:
         description: 'Comma-separated dataset paths (default: all golden datasets)'
         required: false
-        default: 'evals/critical.json,evals/test-generation.json,evals/mutation.json,evals/fuzzing.json,evals/property.json,evals/quality-gate.json,evals/three-arm-example.json,evals/skill-code.json,evals/skill-debug.json,evals/skill-github.json'
+        default: 'evals/critical.json,evals/test-generation.json,evals/mutation.json,evals/fuzzing.json,evals/property.json,evals/quality-gate.json,evals/three-arm-example.json,evals/skill-code.json,evals/skill-debug.json,evals/skill-github.json,evals/skill-browser.json,evals/skill-design.json,evals/skill-ecosystem.json,evals/skill-infrastructure.json,evals/skill-memory.json,evals/skill-planning.json,evals/skill-process.json,evals/skill-shop.json'
       min_pass_rate:
         description: 'Minimum pass rate (0.0..1.0)'
         required: false

AGENTS.md

@@ -560,6 +560,9 @@ command hooks.
 | `fusion.min_quorum` | int | `1` | tournament — minimum passers before declaring winner |
 | `fusion.per_provider_timeout_s` | int | `120` | tournament — per-model wall-clock timeout |
 | `fusion.difficulty_gate` | bool | `true` | difficulty.go — filter stylistic failures to legacy retry |
+| `fusion.oracle_mode` | bool | `false` | loopbuilder — oracle-mode tournament with judge safety (issue #344) |
+
+Oracle mode is **opt-in and gated**: it only activates when `fusion.enabled = true`, `fusion.oracle_mode = true`, and the gate is in `oracle` mode. Unlike PoC mode, oracle mode does **not** use first-pass-wins; all candidates run to completion, a single judge evaluates all outputs in randomized order, and the highest-scoring candidate wins. Default cost cap is tighter ($2.00) and `fusion__oracle_tournament` is `ask` policy (M4).
 
 ### Verbosity / compression mode (issue #167)
 
@@ -623,7 +626,7 @@ Headless JSON contract (stable API — never break without major bump):
 | v3.20.0 | ✅ SHIPPED | Tool coverage, M6 enforcement, catalog, telemetry (issues #249, #253, #248, #250, #252, #251): agent profiles expose full `sin_*` + MCP prefix surface; system prompt injects SIN-tool preference fragment; runtime `ToolCoverageEnforcer` rejects missing/forbidden tool usage; `ledger tools` heatmap/coverage/unused; orchestrator planner emits mandatory `ToolChain` per intent; `sin-code catalog` unifies 46+ MCP tools, 17+ chat tools, and 14+ external MCP prefixes. |
 | v3.20.0 | ✅ SHIPPED | `sin-code image-graph` — SOTA ECharts chart generation (bar/line/pie/area); `skill-github-readme` bundled. 42 subcommands, 37 bundled skills. |
 | v3.21.0 | ✅ SHIPPED | Test-First Verify-Loop (RFC-test-automation.md): `sin_test` + `sin_test_generate` + `sin_quality_gate` + `sin_mutation` + `sin_fuzz` + `sin_property`; `tool.post` hook payload path; `test.*` config keys; `evals/test-generation.json` golden dataset. |
-| v3.22.0 | ACTIVE | SIN Fusion v1: Verify-Tournament (issue #290) — multi-model fan-out on verify.fail, Fireworks pool (6 models), thinking mode, cost-governor, difficulty gate, PoC-only |
+| v3.22.0 | ACTIVE | SIN Fusion v1: Verify-Tournament (issue #290) — multi-model fan-out on verify.fail, Fireworks pool (6 models), thinking mode, cost-governor, difficulty gate; oracle-mode tournament with judge safety (issue #344) |
 
 Each release tag ⇒ goreleaser builds linux/darwin/windows × amd64/arm64,
 updates `homebrew-sin` formula, and ships to GitHub Releases.

CHANGELOG.md

@@ -54,6 +54,29 @@ All notable changes to the SIN-Code unified binary will be documented in this fi
 - Profile distribution table: 8 → 11 targets.
 - CLI `--agent` flag help text and doc comments updated.
 
+### Added — SOTA Skill Infrastructure (PR #359, #360, #361)
+
+- **required_tools on all 37 skills**: 28 remaining skills now declare their
+  SIN tool dependencies. Every skill has a deterministic tool binding enforced
+  by the `ToolCoverageEnforcer` at runtime.
+- **skill-code-create teaches required_tools**: SKILL.md, frameworks/standards,
+  templates/prompt+output, tasks/workflow all updated to scaffold the
+  `required_tools` field in new skills.
+- **8 new eval datasets**: skill-browser, skill-design, skill-ecosystem,
+  skill-infrastructure, skill-memory, skill-planning, skill-process, skill-shop.
+  All 11 categories now have eval coverage. Wired into `eval-n8n.yml` CI.
+- **12 external skills: sources: field**: All `lifecycle: external` skills
+  now document their origin in `metadata.sources`.
+- **skill-code-graph**: empty dirs filled with .md files, .gitkeep removed,
+  lifecycle moved to top-level frontmatter.
+- **Stale counts fixed**: README.md (34→37 skills, 40→42 subcommands),
+  AGENTS.md (35→37), skill-code-create (36→37).
+- **skillmgr 100% coverage**: edge case test for empty/whitespace skill names.
+- **TUI WIP integrated**: 12 untracked files (model_switcher, help_overlay,
+  file_picker, mouse, render_cache, theme_custom, crash_recovery, etc.)
+  integrated. `go build ./cmd/sin-code/...` passes clean.
+- **Doc consistency**: skilldist.doc.md (4 fixes), release-notes/v3.20.0.md.
+
 ### Added — SIN Fusion v1: Verify-Tournament (issue #290)
 
 The largest release in SIN-Code history. Two epics (21 issues), ~50 new
@@ -459,6 +482,15 @@ gate (M3).
 - 264 tests, all `-race` clean (mandate M7)
 - `fusion.enabled = false` (default) → exact legacy behavior, zero regression
 
+### Added — SIN Fusion Oracle-mode tournament (issue #344)
+- **`internal/fusion/oracle.go`** — `Mode`/`ModeOracle` + `OracleJudgeFn` + `LLMOracleJudge` with structured JSON rubric (correctness/completeness/risk), randomized candidate order, and deterministic score tie-break
+- **`tournament.go`** — `Tournament.Mode` + `OracleJudge` fields; `Run()` dispatches to `runPoC()` or `runOracle()`; oracle mode runs all candidates to completion, then judges all outputs together
+- **`agentloop/loop.go`** — tournament now active in both PoC and oracle modes when configured
+- **`loopbuilder/builder.go`** — `FusionOracleMode` config + oracle tournament wiring; tighter default $2.00 cost cap for oracle mode
+- **`config.go`** — `fusion.oracle_mode` config key with `get`/`set`/`show`/`validate` handlers
+- **`permission_defaults.go`** — `fusion__oracle_tournament` = ask (M4)
+- Oracle-mode tests: judge correctness, all-run-to-completion, cost-cap, judge-error, race safety, position bias, nil judge, JSON/markdown verdict parsing, score clamping
+
 ### Added — Runtime DB migration to user-config-dir (issue #265)
 - `cmd/sin-code/internal/tui/agent_runner.go` no longer writes
   `lessons.db` / `sessions.db` directly inside the workspace's

cmd/sin-code/chat_cmd.go

@@ -140,7 +140,8 @@ func NewChatCmd() *cobra.Command {
   sin-code chat --lazy-tools             lazy tool loading via tool_search (issue #270)
   sin-code chat --fusion-on-verify-fail  enable SIN Fusion verify-tournament on verify.fail (issue #290)
   sin-code chat --fusion-providers <list> override Fireworks models for the tournament (comma-separated)
-  sin-code chat --fusion-max-cost <usd>   USD kill-switch per tournament invocation (default 5.0)`,
+  sin-code chat --fusion-max-cost <usd>   USD kill-switch per tournament invocation (default 5.0)
+  Oracle-mode fusion is experimental; set fusion.oracle_mode=true via config. Prefer PoC mode for verifiable tasks.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runChat(cmd.Context(), opts)
 		},

cmd/sin-code/daemon_cmd.go

@@ -19,7 +19,7 @@
 	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/agentloop"
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/autonomy"
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal"
@@ -129,7 +129,7 @@
 	cmd.Flags().BoolVar(&noBaseline, "no-baseline", false, "disable the always-on SinCode loop baseline (tests/debug/docs/completeness DoD); also via SIN_BASELINE=off")
 	cmd.Flags().StringVar(&requireTools, "require-tools", "", "comma-separated tool names the model must invoke before completion (issue #248)")
 	cmd.Flags().StringVar(&forbidTools, "forbid-tools", "", "comma-separated tool names that block completion if invoked (issue #248)")
-	cmd.Flags().BoolVar(&fusionOnVerifyFail, "fusion-on-verify-fail", false, "enable SIN Fusion verify-tournament on verify.fail (issue #290)")
+	cmd.Flags().BoolVar(&fusionOnVerifyFail, "fusion-on-verify-fail", false, "enable SIN Fusion verify-tournament on verify.fail (issue #290). Oracle mode is experimental; set fusion.oracle_mode=true via config.")
 	return cmd
 }
 

cmd/sin-code/internal/autonomy/goal_store.go

@@ -8,6 +8,7 @@ package autonomy
 import (
 	"context"
 	"fmt"
+	"time"
 )
 
 // GoalStore wraps an open Queue, exposing the subset of operations the
@@ -79,6 +80,15 @@ func (s *GoalStore) FailGoal(ctx context.Context, id int64, sessionID, errMsg st
 	return s.queue.Fail(ensureCtx(ctx), id, sessionID, errMsg)
 }
 
+// LeaseGoal atomically claims the highest-priority leasable goal. Returns
+// (nil, nil) when no goal is available.
+func (s *GoalStore) LeaseGoal(ctx context.Context, leaseDur time.Duration) (*Goal, error) {
+	if s == nil || s.queue == nil {
+		return nil, fmt.Errorf("autonomy: goal store not initialized")
+	}
+	return s.queue.Lease(ensureCtx(ctx), leaseDur)
+}
+
 // Close releases the underlying queue.
 func (s *GoalStore) Close() error {
 	if s == nil || s.queue == nil {

cmd/sin-code/internal/loopbuilder/fusion_integration_test.go

@@ -149,6 +149,37 @@ func TestFusionIntegration_WireFusionNoopWhenGateOff(t *testing.T) {
 	}
 }
 
+func TestFusionIntegration_WireFusionOracleMode(t *testing.T) {
+	loop := &agentloop.Loop{}
+	gate := verify.NewGate("oracle", nil, nil)
+	cfg := Config{
+		FusionEnabled:    true,
+		FusionOracleMode: true,
+		FusionProviders:  []string{"minimax-m3", "glm-5p2"},
+		FusionMaxCostUSD: 10.0,
+		FusionMinQuorum:  2,
+	}
+
+	WireFusion(loop, cfg, gate, nil, nil, nil, nil)
+
+	if loop.TournamentRunner == nil {
+		t.Fatal("expected TournamentRunner to be non-nil in oracle mode")
+	}
+	adapter, ok := loop.TournamentRunner.(*fusionAdapter)
+	if !ok {
+		t.Fatalf("expected *fusionAdapter, got %T", loop.TournamentRunner)
+	}
+	if adapter.t.Mode != fusion.ModeOracle {
+		t.Errorf("expected oracle mode, got %q", adapter.t.Mode)
+	}
+	if adapter.t.OracleJudge == nil {
+		t.Error("expected OracleJudge to be wired in oracle mode")
+	}
+	if adapter.t.MaxCostUSD != 2.0 {
+		t.Errorf("expected oracle max cost capped at 2.0, got %v", adapter.t.MaxCostUSD)
+	}
+}
+
 func TestFusionIntegration_TournamentEndToEndAllFail(t *testing.T) {
 	mockRunFunc := func(ctx context.Context, prov fusion.ProviderConfig, sess *session.Session, prompt string) (*agentloop.Result, error) {
 		return &agentloop.Result{

cmd/sin-code/internal/todo/auto_discover.go

@@ -97,9 +97,11 @@ func (d *AutoDiscover) ScanDir(root string, maxDepth int) ([]TodoMarker, error)
 			}
 			if maxDepth > 0 {
 				rel, _ := filepath.Rel(root, path)
-				depth := strings.Count(rel, string(filepath.Separator))
-				if depth >= maxDepth {
-					return filepath.SkipDir
+				if rel != "." {
+					depth := strings.Count(rel, string(filepath.Separator)) + 1
+					if depth >= maxDepth {
+						return filepath.SkipDir
+					}
 				}
 			}
 			return nil

cmd/sin-code/internal/todo/goal_bridge_test.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/autonomy"
 )
@@ -141,7 +142,13 @@ func TestSyncStatusGoalFailedReopensTodo(t *testing.T) {
 	_ = ts.Add(td)
 	g, _ := b.TodoToGoal(td)
 	gid := strconv.FormatInt(g.ID, 10)
-	_ = gs.FailGoal(context.Background(), g.ID, "", "boom")
+	for i := 0; i < 3; i++ {
+		leased, _ := gs.LeaseGoal(context.Background(), time.Minute)
+		if leased == nil {
+			break
+		}
+		_ = gs.FailGoal(context.Background(), leased.ID, "", "boom")
+	}
 	if err := b.SyncStatus(gid, td.ID); err != nil {
 		t.Fatalf("SyncStatus: %v", err)
 	}

cmd/sin-code/tui/todos_refresh_test.go

@@ -12,77 +12,97 @@ import (
 	"github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/todo"
 )
 
-// tempTodoDB creates a temp bbolt DB for the todo store and returns a
-// cleanup function that restores the original hook.
-func tempTodoDB(t *testing.T) (*todo.Store, func()) {
+// setupTodoDB creates a temp bbolt DB, seeds it with the given todos,
+// closes it, and wires todoOpenHook to reopen it. Returns a cleanup fn.
+// The store must be closed before RefreshTodosCmd is called to avoid
+// bbolt lock contention.
+func setupTodoDB(t *testing.T, seed ...*todo.Todo) func() {
 	t.Helper()
 	dir := t.TempDir()
 	path := filepath.Join(dir, "todo.db")
 	store, err := todo.Open(path)
 	if err != nil {
 		t.Fatalf("Open: %v", err)
 	}
+	for _, td := range seed {
+		if err := store.Add(td); err != nil {
+			t.Fatalf("Add: %v", err)
+		}
+	}
+	_ = store.Close()
+
 	orig := todoOpenHook
 	todoOpenHook = func(p string) (*todo.Store, error) {
 		if p == "" {
 			return todo.Open(path)
 		}
 		return todo.Open(p)
 	}
-	return store, func() {
+	return func() {
 		todoOpenHook = orig
-		_ = store.Close()
 	}
 }
 
 // TestRefreshTodosCmdReturnsRealCounts verifies that RefreshTodosCmd
 // queries the real store and returns non-zero counts when todos exist.
 func TestRefreshTodosCmdReturnsRealCounts(t *testing.T) {
-	store, cleanup := tempTodoDB(t)
-	defer cleanup()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "todo.db")
+	store, err := todo.Open(path)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
 
-	// Seed the store with real todos.
-	if err := store.Add(&todo.Todo{Title: "Task A", Priority: todo.PriorityP0, Status: todo.StatusOpen}); err != nil {
-		t.Fatal(err)
+	// Create two todos: A (open) blocks B (open) → B is "blocked".
+	todoA := &todo.Todo{Title: "Task A", Priority: todo.PriorityP0, Status: todo.StatusOpen}
+	todoB := &todo.Todo{Title: "Task B", Priority: todo.PriorityP1, Status: todo.StatusOpen}
+	if err := store.Add(todoA); err != nil {
+		t.Fatalf("Add A: %v", err)
 	}
-	if err := store.Add(&todo.Todo{Title: "Task B", Priority: todo.PriorityP1, Status: todo.StatusOpen}); err != nil {
-		t.Fatal(err)
+	if err := store.Add(todoB); err != nil {
+		t.Fatalf("Add B: %v", err)
 	}
-	if err := store.Add(&todo.Todo{Title: "Task C", Priority: todo.PriorityP2, Status: todo.StatusBlocked}); err != nil {
-		t.Fatal(err)
+	if err := store.AddDep(todo.Dependency{From: todoB.ID, To: todoA.ID, Type: todo.DepBlocks}); err != nil {
+		t.Fatalf("AddDep: %v", err)
 	}
+	_ = store.Close()
 
-	// The default todoDataHook uses todoOpenHook, which is overridden
-	// above to point at our temp DB. No need to reset todoDataHook.
+	orig := todoOpenHook
+	todoOpenHook = func(p string) (*todo.Store, error) {
+		if p == "" {
+			return todo.Open(path)
+		}
+		return todo.Open(p)
+	}
+	defer func() { todoOpenHook = orig }()
 
 	cmd := RefreshTodosCmd()
 	msg := cmd()
 
 	refresh, ok := msg.(TodosRefreshMsg)
 	if !ok {
-		// Graceful degradation might return CountsMsg if store is nil.
-		// But with our hook, it should return TodosRefreshMsg.
 		t.Fatalf("expected TodosRefreshMsg, got %T", msg)
 	}
 
-	if refresh.Counts.Open != 3 {
-		t.Errorf("expected Open=3, got %d", refresh.Counts.Open)
+	if refresh.Counts.Open != 2 {
+		t.Errorf("expected Open=2, got %d", refresh.Counts.Open)
 	}
 	if refresh.Counts.Blocked != 1 {
 		t.Errorf("expected Blocked=1, got %d", refresh.Counts.Blocked)
 	}
+	if refresh.Counts.Ready != 1 {
+		t.Errorf("expected Ready=1, got %d", refresh.Counts.Ready)
+	}
 }
 
 // TestRefreshTodosCmdReturnsRealItems verifies that RefreshTodosCmd
 // returns real todo items from the store, not an empty list.
 func TestRefreshTodosCmdReturnsRealItems(t *testing.T) {
-	store, cleanup := tempTodoDB(t)
+	cleanup := setupTodoDB(t,
+		&todo.Todo{Title: "Real Todo", Priority: todo.PriorityP0, Status: todo.StatusOpen, Type: todo.TypeBug},
+	)
 	defer cleanup()
 
-	if err := store.Add(&todo.Todo{Title: "Real Todo", Priority: todo.PriorityP0, Status: todo.StatusOpen, Type: todo.TypeBug}); err != nil {
-		t.Fatal(err)
-	}
-
 	cmd := RefreshTodosCmd()
 	msg := cmd()
 
@@ -105,24 +125,15 @@ func TestRefreshTodosCmdReturnsRealItems(t *testing.T) {
 // TestRefreshTodosCmdOverdueComputed verifies that overdue todos (open
 // with DueAt in the past) are counted correctly.
 func TestRefreshTodosCmdOverdueComputed(t *testing.T) {
-	store, cleanup := tempTodoDB(t)
-	defer cleanup()
-
 	past := time.Now().Add(-24 * time.Hour)
 	future := time.Now().Add(24 * time.Hour)
 
-	// Overdue: open with past DueAt.
-	if err := store.Add(&todo.Todo{Title: "Overdue", Priority: todo.PriorityP0, Status: todo.StatusOpen, DueAt: &past}); err != nil {
-		t.Fatal(err)
-	}
-	// Not overdue: open with future DueAt.
-	if err := store.Add(&todo.Todo{Title: "Future", Priority: todo.PriorityP1, Status: todo.StatusOpen, DueAt: &future}); err != nil {
-		t.Fatal(err)
-	}
-	// Not overdue: done with past DueAt (closed todos don't count).
-	if err := store.Add(&todo.Todo{Title: "Closed", Priority: todo.PriorityP2, Status: todo.StatusDone, DueAt: &past}); err != nil {
-		t.Fatal(err)
-	}
+	cleanup := setupTodoDB(t,
+		&todo.Todo{Title: "Overdue", Priority: todo.PriorityP0, Status: todo.StatusOpen, DueAt: &past},
+		&todo.Todo{Title: "Future", Priority: todo.PriorityP1, Status: todo.StatusOpen, DueAt: &future},
+		&todo.Todo{Title: "Closed", Priority: todo.PriorityP2, Status: todo.StatusDone, DueAt: &past},
+	)
+	defer cleanup()
 
 	cmd := RefreshTodosCmd()
 	msg := cmd()
@@ -144,7 +155,6 @@ func TestRefreshTodosCmdOverdueComputed(t *testing.T) {
 // returns CountsMsg{} (zeros) when the store is unavailable, rather
 // than crashing.
 func TestRefreshTodosCmdGracefulDegradation(t *testing.T) {
-	// Point the hook at a non-existent path to force an error.
 	origOpen := todoOpenHook
 	todoOpenHook = func(p string) (*todo.Store, error) {
 		return nil, os.ErrNotExist
@@ -154,7 +164,6 @@ func TestRefreshTodosCmdGracefulDegradation(t *testing.T) {
 	cmd := RefreshTodosCmd()
 	msg := cmd()
 
-	// Should return CountsMsg{} (graceful degradation, not a crash).
 	c, ok := msg.(CountsMsg)
 	if !ok {
 		t.Fatalf("expected CountsMsg on error, got %T", msg)
@@ -197,7 +206,6 @@ func TestTodosRefreshMsgUpdatesModel(t *testing.T) {
 		t.Errorf("expected first item ID 'st-1', got %q", m.TodoItems[0].ID)
 	}
 
-	// Kanban view should also be populated.
 	if m.KanbanView == nil {
 		t.Fatal("expected KanbanView to be initialized")
 	}

cmd/sin-code/tui/tui_coverage_test.go

@@ -2218,8 +2218,8 @@ func TestPreviousViewCoverage(t *testing.T) {
 	m := NewModel()
 	m.ViewKind = ViewTools
 	m.PreviousView()
-	if m.ViewKind != ViewMemory {
-		t.Errorf("PreviousView = %v, want Memory", m.ViewKind)
+	if m.ViewKind != ViewKanban {
+		t.Errorf("PreviousView = %v, want Kanban", m.ViewKind)
 	}
 }