feat(skills): runnable determinism gate + fix broken gate providers + plugin leak (#62)

make determinism [SKILL=<name>] runs the loop's regression gate. Fix the 4 non-trace-review determinism configs whose exec provider path never resolved (promptfoo runs exec from the config dir; give each skill a local eval-run.sh wrapper). Exclude dev-only eval-run.sh from the shipped plugin.

Author: AndreJorgeLopes

CLAUDE.md

@@ -108,6 +108,14 @@ computable work, on two axes: **offload** an AI decide/parse/score step to bash/
 (assert with promptfoo `is-json`/`regex`/no-fences on stdout). Leave genuinely-creative
 steps model-driven. The `determinize-skill` (proof-of-skill) automates this audit.
 
+Each converted skill carries a `skills/<name>/determinism.promptfooconfig.yaml` (dev-only, not
+shipped in the plugin). Run the gate with **`make determinism`** (all skills) or
+**`make determinism SKILL=<name>`** (one). It needs API keys (the skill-spawning configs) and a
+reachable Langfuse (trace-review), so it is deliberately NOT part of `make test` / CI. promptfoo
+runs each config's `exec:` provider from that config's own directory, so provider paths are
+config-dir-relative (`../../eval/lib/run-skill.sh` for the claude-spawning skills, a local
+`eval-run.sh` for trace-review). This is the loop's pre-edit regression gate.
+
 ## Hooks Architecture
 
 Hook scripts live in `lib/hooks/`, registered in `~/.claude/settings.json` via `devflow init` (step 5d).

Makefile

@@ -4,7 +4,7 @@ LIBDIR := $(PREFIX)/share/devflow
 VERSION := 0.13.0
 TARBALL := devflow-$(VERSION).tar.gz
 
-.PHONY: install uninstall link test test-unit brew-local release help plugin-dev plugin-unlink plugin-install check-version check-formula version-bump flows flows-check skills-sync skills-check skills-guard
+.PHONY: install uninstall link test test-unit brew-local release help plugin-dev plugin-unlink plugin-install check-version check-formula version-bump flows flows-check skills-sync skills-check skills-guard determinism
 
 help: ## Show this help
 	@grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*## "}; {printf "  %-14s %s\n", $$1, $$2}'
@@ -64,6 +64,21 @@ skills-check: ## Fail if generated plugin skills/commands drift from repo-root s
 skills-guard: ## Detect skill edits made in the WRONG (generated) tree and fold them into the source
 	@bash scripts/skills-guard.sh
 
+determinism: ## Run the promptfoo determinism gate for skills with a config (needs API keys + Langfuse; not in CI). One skill: make determinism SKILL=<name>
+	@command -v npx >/dev/null 2>&1 || { echo "determinism: npx (Node) is required"; exit 1; }
+	@[ -f "$(HOME)/.config/zsh/secrets" ] && . "$(HOME)/.config/zsh/secrets" 2>/dev/null || true; \
+	fail=0; ran=0; \
+	for cfg in skills/*/determinism.promptfooconfig.yaml; do \
+		[ -f "$$cfg" ] || continue; \
+		name="$$(basename "$$(dirname "$$cfg")")"; \
+		if [ -n "$(SKILL)" ] && [ "$(SKILL)" != "$$name" ]; then continue; fi; \
+		echo "=== determinism gate: $$name ==="; ran=1; \
+		npx -y promptfoo@latest eval -c "$$cfg" --no-cache || fail=1; \
+	done; \
+	[ "$$ran" = 1 ] || { echo "determinism: no config matched$(if $(SKILL), for SKILL=$(SKILL),)"; exit 1; }; \
+	[ "$$fail" = 0 ] || { echo "determinism gate FAILED"; exit 1; }; \
+	echo "determinism gate passed"
+
 flows: ## Regenerate flow mini-plugins from canonical sources
 	@bash scripts/build-flows.sh
 

devflow-plugin/skills/trace-review/eval-run.sh

@@ -28,10 +28,13 @@ command -v jq >/dev/null 2>&1 || { echo "build-skills: jq is required" >&2; exit
 # VERSION always comes from the real plugin manifest, even when rendering to a temp OUT.
 VERSION="$(jq -r .version "${ROOT}/devflow-plugin/.claude-plugin/plugin.json")"
 
-# Files that live beside a skill at repo root for dev/CI but must NOT ship in the plugin.
+# Files that live beside a skill at repo root for dev/CI but must NOT ship in the plugin:
+# the promptfoo determinism config and its exec-provider harness (references bin/devflow via a
+# repo-relative path that does not exist in a plugin install).
 _is_dev_only() {
   case "$(basename "$1")" in
     determinism.promptfooconfig.yaml) return 0 ;;
+    eval-run.sh) return 0 ;;
     *) return 1 ;;
   esac
 }

scripts/build-skills.sh

@@ -9,7 +9,7 @@ prompts:
   - '{{input}}'
 
 providers:
-  - id: 'exec: bash eval/lib/run-skill.sh'
+  - id: 'exec: bash eval-run.sh'
     label: finish-feature@head
 
 tests:

skills/finish-feature/determinism.promptfooconfig.yaml

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# promptfoo `exec:` provider for this skill's determinism asserts.
+# promptfoo runs the exec command from THIS config's directory and validates the file
+# locally, so it must be a local file — it then execs the shared headless runner with an
+# absolute path (no CWD assumptions). Dev-only: excluded from the shipped plugin.
+set -euo pipefail
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec bash "${here}/../../eval/lib/run-skill.sh" "$@"

skills/finish-feature/eval-run.sh

@@ -8,7 +8,7 @@ prompts:
 
 providers:
   # run-skill.sh clears CLAUDECODE so the nested claude launches (sdk#573). Run from repo root.
-  - id: 'exec: bash eval/lib/run-skill.sh'
+  - id: 'exec: bash eval-run.sh'
     label: new-feature@head
 
 defaultTest:

skills/new-feature/determinism.promptfooconfig.yaml

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# promptfoo `exec:` provider for this skill's determinism asserts.
+# promptfoo runs the exec command from THIS config's directory and validates the file
+# locally, so it must be a local file — it then execs the shared headless runner with an
+# absolute path (no CWD assumptions). Dev-only: excluded from the shipped plugin.
+set -euo pipefail
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec bash "${here}/../../eval/lib/run-skill.sh" "$@"

skills/new-feature/eval-run.sh

@@ -8,7 +8,7 @@ prompts:
   - '{{input}}'
 
 providers:
-  - id: 'exec: bash eval/lib/run-skill.sh'
+  - id: 'exec: bash eval-run.sh'
     label: resolve-repo@head
 
 tests:

skills/resolve-repo/determinism.promptfooconfig.yaml

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# promptfoo `exec:` provider for this skill's determinism asserts.
+# promptfoo runs the exec command from THIS config's directory and validates the file
+# locally, so it must be a local file — it then execs the shared headless runner with an
+# absolute path (no CWD assumptions). Dev-only: excluded from the shipped plugin.
+set -euo pipefail
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec bash "${here}/../../eval/lib/run-skill.sh" "$@"