diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..dee29d3 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,14 @@ +{ + "name": "local-skills", + "description": "Local development marketplace for the pandavas skill", + "owner": { "name": "pandavas", "email": "noreply@example.com" }, + "plugins": [ + { + "name": "pandavas", + "description": "Autonomous, test-gated multi-agent code resolution where deterministic verbs rule on correctness.", + "version": "0.1.0", + "source": "./", + "author": { "name": "pandavas", "email": "noreply@example.com" } + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..836aa04 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "pandavas", + "description": "Autonomous, test-gated multi-agent code resolution: a /pandavas command + skill where the host model plays the research/worker/judge agents and deterministic Python verbs rule on whether the code is correct.", + "version": "0.1.0", + "author": { "name": "pandavas", "email": "noreply@example.com" }, + "license": "MIT", + "keywords": ["multi-agent", "coding-agent", "tdd", "test-gated", "orchestration", "claude-code", "cursor"] +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..9806adf --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "pandavas", + "displayName": "Pandavas", + "description": "Autonomous, test-gated multi-agent code resolution where deterministic verbs rule on correctness. (Cursor has no subagent primitive, so the judge runs as a fresh-pass self-review; the deterministic gates are identical to Claude Code.)", + "version": "0.1.0", + "author": { "name": "pandavas", "email": "noreply@example.com" }, + "license": "MIT", + "keywords": ["multi-agent", "coding-agent", "tdd", "test-gated", "orchestration", "cursor"], + "skills": "./skills/" +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f077a2a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Keep shell installers LF (must run under bash) and PowerShell installers CRLF. +*.sh text eol=lf +*.ps1 text eol=crlf diff --git a/.gitignore b/.gitignore index ad81f42..098a426 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ Thumbs.db runs/ traces/ *.log + +# --- Skill-mode working/state dir --- +.pandavas/ diff --git a/CLAUDE.md b/CLAUDE.md index 1f32200..329868e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,6 +93,15 @@ moving to the next. ## Learned Rules -> Accumulate project-specific conventions here as they're discovered. Empty for now. - -- +> Accumulate project-specific conventions here as they're discovered. + +- **Skill mode is additive (keyless `/pandavas`).** The host harness's model + (Claude Code / Cursor) plays the LLM agents while deterministic CLI verbs in + `src/pandavas/spine.py` (`baseline`, `resolve-brief`, `run-tests`, `judge-gate`, + `decide`, `restore`, `commit`, ...) own every gate. The verbs reuse `run`'s logic + (`nodes.make_bhima_test` / `_better` / `_restore_snapshot`) and pass state through a + gitignored `.pandavas/` dir; `decide` is the sole writer of `loop.json`. The + standalone `python -m pandavas run` path and the six deterministic modules are + unchanged. Packaging (`commands/`, `skills/`, `agents/`, `scripts/install-*`) is + top-level and excluded from the wheel (pyproject packages `src/` only). Keep verb + stdout ASCII-only (`json.dumps(..., ensure_ascii=True)`). Design: `docs/SKILL_MODE.md`. diff --git a/README.md b/README.md index c3f1b96..049e71e 100644 --- a/README.md +++ b/README.md @@ -176,17 +176,46 @@ no API key: python examples/demo.py ``` +## Skill mode (keyless, in-editor) + +pandavas also runs **without an API key**, as a `/pandavas` command inside a coding +harness (Claude Code, Cursor). The harness's own model plays the three LLM agents; +deterministic `python -m pandavas` verbs play the test runner and the orchestrator, so +the same regression-aware gate still rules on correctness. You get the in-editor +ergonomics; the engine keeps the guarantees. + +Install the engine plus the skill, then use it in any repo whose test suite runs: + +```bash +pip install -e . # so `python -m pandavas` works +bash scripts/install-claude.sh # Claude Code (or scripts/install-claude.ps1) +bash scripts/install-cursor.sh # Cursor (or scripts/install-cursor.ps1) +``` + +``` +/pandavas . Fix the off-by-one in paginate() +``` + +The host model researches into an anchored brief (resolve-gated) and writes the change; +the verbs decide: `run-tests` (exit 0 only if no new failures vs baseline), `judge-gate`, +then `decide` (converge / retry / stop). On Claude Code the judge runs as a genuine +fresh-context subagent; on Cursor it degrades to a fresh-pass self-review (the +deterministic gates are identical). One honest caveat: skill mode can't *structurally* +force the gates the way `run` does — the model must call them and obey their exit codes +— so `python -m pandavas run` stays the hard-determinism path. Full design: +[`docs/SKILL_MODE.md`](docs/SKILL_MODE.md). + ## Tests ```bash python -m pytest ``` -93 tests cover the deterministic core directly: the executor and test-command +121 tests cover the deterministic core directly: the executor and test-command detection, the JUnit parser, the resolve gate, the path-safe edit applier, snapshot and diffing, the regression and red→green logic, oscillation and -best-attempt restore, the git helpers, the CLI, and an end-to-end red→green run on -the fixture. 92 run offline and pass; one is a live-API smoke test that runs only +best-attempt restore, the git helpers, the CLI, the keyless verb spine and a skill-mode end-to-end run, and an end-to-end red→green run on +the fixture. 120 run offline and pass; one is a live-API smoke test that runs only when `GROQ_API_KEY` is set. CI runs the suite on Python 3.10, 3.11, and 3.12. ## Tech stack diff --git a/agents/pandavas-judge.md b/agents/pandavas-judge.md new file mode 100644 index 0000000..f403a4a --- /dev/null +++ b/agents/pandavas-judge.md @@ -0,0 +1,20 @@ +--- +name: pandavas-judge +description: Independent fresh-context reviewer (Sahadeva) for pandavas skill mode. Judges a proposed change against acceptance criteria and the true test state and returns {"approved", "feedback"}. Invoked by the pandavas skill behind the deterministic judge-gate - it must never see the worker's reasoning. +tools: Read, Grep +--- + +You are Sahadeva, an INDEPENDENT code reviewer. You did NOT write this code and must not assume it is correct. + +You are given only: the acceptance criteria, the cumulative diff under review (often written to `.pandavas/diffs/cumulative.diff`), and the list of task-relevant tests still failing (if any). A deterministic gate has already confirmed the suite has no NEW failures versus baseline, so your job is correctness against intent - not re-running tests. + +Review: +- Does the change actually satisfy EVERY acceptance criterion? +- If a test was added or modified, does it genuinely verify the requirement, or is it vacuous (trivially passing, asserts nothing meaningful)? +- Is any test relevant to the task still failing? If so, the change is INCOMPLETE. + +Approve ONLY if the change satisfies the acceptance criteria AND no task-relevant test remains failing. + +Output ONLY a JSON object - no prose, no markdown, no code fences: + +{"approved": , "feedback": ""} diff --git a/commands/pandavas.md b/commands/pandavas.md new file mode 100644 index 0000000..da7958b --- /dev/null +++ b/commands/pandavas.md @@ -0,0 +1,12 @@ +--- +description: Autonomously resolve one test-gated bug fix or small feature on a local repo - research, worker, test, judge, orchestrate, where deterministic Python verbs rule on correctness. +argument-hint: +allowed-tools: Bash, Read, Edit, Write, Grep, Glob, Task +--- +Use the **pandavas** skill. + +Task (repo + what to fix or build): $ARGUMENTS + +Baseline the suite, research once into an anchored brief and resolve-gate it, then loop worker -> run-tests -> judge-gate -> fresh-context judge -> decide until the verbs converge (`decide` exit 0) or honestly report they could not. The Python verbs rule on correctness: never claim success without `run-tests` exit 0, never claim convergence without `decide` exit 0. + +Close by showing the diff, the red->green tests, and the deterministic verdicts (real exit codes). STOP for my approval before committing. diff --git a/docs/SKILL_MODE.md b/docs/SKILL_MODE.md new file mode 100644 index 0000000..df0c866 --- /dev/null +++ b/docs/SKILL_MODE.md @@ -0,0 +1,115 @@ +# Skill mode (keyless `/pandavas`) + +> Design source of truth is [`SPEC.md`](SPEC.md). This document describes the +> additive **skill mode** layered on top of it. The standalone +> `python -m pandavas run` path is unchanged. + +## Why two modes + +pandavas runs in two ways that share one deterministic core: + +| | Standalone (`python -m pandavas run`) | Skill (`/pandavas`) | +|---|---|---| +| LLM agents | pandavas's own LLM client + API key | the **host harness's model** (Claude Code / Cursor) - **no API key** | +| Orchestration | LangGraph state machine, one process | the host model follows a protocol, calling verbs via the shell | +| Determinism | **hard** - the process forces every gate | **soft** - the model must call the verbs and obey their exit codes | +| Best for | a CI-able, reproducible proof of the system | a frictionless, keyless fix from inside your editor | + +Both call the **same** deterministic code, so a fix delivered in skill mode is gated +by the same regression-aware test logic the standalone mode is CI-tested against. + +## The split across the process boundary + +A subprocess cannot borrow the harness's model, so skill mode does not run the agents +as a subprocess. Instead it splits the work: + +- **The host model owns the LLM-shaped steps:** research (Nakula), writing the edit + (Arjuna), judging (Sahadeva). +- **The deterministic decisions stay in pandavas's Python**, exposed as LLM-free CLI + verbs the skill calls: the resolve gate, the regression-aware test verdict, + oscillation detection, the iteration cap, best-attempt restore, git delivery. + +The model is the hands and eyes; the verbs are the rules. The skill protocol lives in +[`../skills/pandavas/SKILL.md`](../skills/pandavas/SKILL.md). + +## The verb surface + +Every verb takes `--repo`, prints one ASCII JSON line, and sets a process exit code. +They are implemented in [`../src/pandavas/spine.py`](../src/pandavas/spine.py), reusing +the same functions as `run` (`executor`, `testresults`, `edits`, `diffing`, `gitutils`, +`brief`, plus `nodes.make_bhima_test` / `_better` / `_restore_snapshot`). + +| Verb | Role | Exit codes | +|---|---|---| +| `baseline` | Bhima | run the suite once; capture baseline + snapshots; init run state | 0 / 1 no test cmd | +| `detect-test` | - | print the auto-detected test command | 0 / 1 none | +| `resolve-brief` | gate | every anchor must resolve to real files/lines | 0 resolved / 1 dangling | +| `run-tests` | Bhima | regression-aware verdict vs baseline | **0 iff no new failures** / 1 | +| `diff` | - | render the cumulative / per-iteration diff to a file | 0 | +| `judge-gate` | Sahadeva | deterministic pre-LLM reject (failing tests / empty diff) | 0 review / 30 reject | +| `decide` | Yudhishthira | trace, best-attempt, oscillation, cap, restore-on-stop | 0 converged / 10 continue / 20 stop | +| `restore` | - | restore the best or baseline snapshot | 0 / 1 | +| `commit` | - | deliver a converged change to a new branch | 0 / 1 | +| `apply-edits` | - | optional path-safe full-file writer | 0 / 1 rejected | + +## The `.pandavas/` working directory + +Because the verbs are stateless processes, run state persists in a `.pandavas/` +directory under the target repo: + +``` +.pandavas/ + baseline.json test command, junit path, baseline per-test, exit code + brief.json the resolved ResearchBrief (research-once cache) + loop.json iteration / signatures / best-attempt / trace / status + last_test.json the latest regression verdict + snapshots/ baseline.json, pre_iter.json, best.json + diffs/ cumulative.diff, iter.diff +``` + +It is kept out of your repo three ways: `baseline` appends `.pandavas/` to +`/info/exclude` (so a `commit` never stages it), every snapshot drops +`.pandavas/` keys, and `.pandavas` is in `retrieval.IGNORE_DIRS`. `decide` is the sole +writer of `loop.json` and recomputes diffs/signatures from snapshots, so a fabricated +model diff cannot game the loop. + +## Fresh-context judge, per harness + +The judge (Sahadeva) must not review its own work. Behind the deterministic +`judge-gate`: + +- **Claude Code** spawns the **`pandavas-judge` subagent** + ([`../agents/pandavas-judge.md`](../agents/pandavas-judge.md)) with only the + acceptance criteria, the cumulative diff, and the still-failing tests - genuine + fresh context. +- **Cursor** has no equal subagent primitive, so the judge runs as a fresh-pass + self-review in shared context. This is **honestly weaker** (the reviewer has seen + the worker's reasoning). The deterministic gates are identical on both; only the + LLM-review independence degrades. For strict isolation, use Claude Code or the + standalone `run`. + +## Honest limitation: soft vs hard determinism + +In standalone `run`, the process calls Yudhishthira unconditionally - the gates cannot +be skipped. In skill mode the host model must *choose* to call `run-tests` / +`judge-gate` / `decide` and *obey* their exit codes. An over-eager model could skip a +gate and claim success. This is mitigated (the Iron Law, the mandatory procedure, a +proof step that pastes real exit codes, `commit` gated behind `decide` exit 0 plus +human approval, and `decide` recomputing from snapshots) but not eliminated. + +**`python -m pandavas run` remains the hard-determinism proof; `/pandavas` is the +keyless ergonomic path.** + +## Install & use + +```bash +pip install -e . # the engine, so `python -m pandavas` works +bash scripts/install-claude.sh # or scripts/install-claude.ps1 (Claude Code) +bash scripts/install-cursor.sh # or scripts/install-cursor.ps1 (Cursor) +``` + +Then, in any repo whose test suite already runs: + +``` +/pandavas . Fix the off-by-one in paginate() +``` diff --git a/scripts/install-claude.ps1 b/scripts/install-claude.ps1 new file mode 100644 index 0000000..af0cd55 --- /dev/null +++ b/scripts/install-claude.ps1 @@ -0,0 +1,12 @@ +# Install the pandavas skill into Claude Code (run from the repo root). +$dest = Join-Path $HOME ".claude" +New-Item -ItemType Directory -Force -Path "$dest\commands","$dest\skills","$dest\agents" | Out-Null +Copy-Item -Recurse -Force commands\* "$dest\commands\" +Copy-Item -Recurse -Force skills\* "$dest\skills\" +Copy-Item -Recurse -Force agents\* "$dest\agents\" +Write-Host "Installed the pandavas skill into Claude Code -> $dest" +Write-Host "" +Write-Host "Also install the engine so 'python -m pandavas' works:" +Write-Host " pip install -e . # from this repo, or: pip install pandavas" +Write-Host "" +Write-Host "Then open Claude Code in any repo and type: /pandavas . " diff --git a/scripts/install-claude.sh b/scripts/install-claude.sh new file mode 100644 index 0000000..a192bb3 --- /dev/null +++ b/scripts/install-claude.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Install the pandavas skill into Claude Code (run from the repo root). +set -euo pipefail +DEST="${HOME}/.claude" +mkdir -p "$DEST/commands" "$DEST/skills" "$DEST/agents" +cp -r commands/* "$DEST/commands/" +cp -r skills/* "$DEST/skills/" +cp -r agents/* "$DEST/agents/" +echo "Installed the pandavas skill into Claude Code." +echo " commands -> $DEST/commands skills -> $DEST/skills agents -> $DEST/agents" +echo +echo "Also install the engine so 'python -m pandavas' works:" +echo " pip install -e . # from this repo, or: pip install pandavas" +echo +echo "Then open Claude Code in any repo and type: /pandavas . " diff --git a/scripts/install-cursor.ps1 b/scripts/install-cursor.ps1 new file mode 100644 index 0000000..f56d135 --- /dev/null +++ b/scripts/install-cursor.ps1 @@ -0,0 +1,15 @@ +# Install the pandavas skill into Cursor (run from the repo root). +$dest = Join-Path $HOME ".cursor" +New-Item -ItemType Directory -Force -Path "$dest\commands","$dest\skills" | Out-Null +Copy-Item -Recurse -Force commands\* "$dest\commands\" +Copy-Item -Recurse -Force skills\* "$dest\skills\" +Write-Host "Installed the pandavas skill into Cursor -> $dest" +Write-Host "" +Write-Host "Note: Cursor has no subagent primitive, so the judge (Sahadeva) runs as a" +Write-Host "fresh-pass self-review in shared context - weaker independence than Claude" +Write-Host "Code's subagent. The deterministic gates (run-tests, judge-gate, decide) are" +Write-Host "identical on both harnesses. For strict judge isolation use Claude Code, or" +Write-Host "run the standalone engine: python -m pandavas run --repo . --task '...'" +Write-Host "" +Write-Host "Also install the engine: pip install -e . (so 'python -m pandavas' works)" +Write-Host "Then in Cursor, in any repo, type: /pandavas . " diff --git a/scripts/install-cursor.sh b/scripts/install-cursor.sh new file mode 100644 index 0000000..2ba5be9 --- /dev/null +++ b/scripts/install-cursor.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Install the pandavas skill into Cursor (run from the repo root). +set -euo pipefail +DEST="${HOME}/.cursor" +mkdir -p "$DEST/commands" "$DEST/skills" +cp -r commands/* "$DEST/commands/" +cp -r skills/* "$DEST/skills/" +echo "Installed the pandavas skill into Cursor." +echo " commands -> $DEST/commands skills -> $DEST/skills" +echo +echo "Note: Cursor has no subagent primitive, so the judge (Sahadeva) runs as a" +echo "fresh-pass self-review in shared context - weaker independence than Claude" +echo "Code's subagent. The deterministic gates (run-tests, judge-gate, decide) are" +echo "identical on both harnesses. For strict judge isolation use Claude Code, or" +echo "run the standalone engine: python -m pandavas run --repo . --task \"...\"" +echo +echo "Also install the engine: pip install -e . (so 'python -m pandavas' works)" +echo "Then in Cursor, in any repo, type: /pandavas . " diff --git a/skills/pandavas/SKILL.md b/skills/pandavas/SKILL.md new file mode 100644 index 0000000..5534f83 --- /dev/null +++ b/skills/pandavas/SKILL.md @@ -0,0 +1,81 @@ +--- +name: pandavas +description: Use when you want an autonomous, test-gated fix or small feature on a local repo - a five-agent loop (research, worker, test, judge, orchestrate) where deterministic Python verbs rule on whether the code is correct. Requires a repo whose test suite already runs locally. +--- + +# Pandavas + +## Overview + +pandavas resolves ONE coding task - a bug fix or a small, testable feature - on a repo you already have locally. You (the host model) play the LLM agents: Nakula (research), Arjuna (worker), Sahadeva (judge). Deterministic `python -m pandavas` verbs play Bhima (the test runner) and Yudhishthira (the loop controller). You propose; the verbs dispose. + +**Core principle:** A model never decides whether the code is correct. `run-tests` and `decide` decide, by process exit code. Tests are ground truth; research hands over checked pointers, not prose. + +## The Iron Law + +``` +THE PYTHON VERBS RULE. +No "it passes" without `run-tests` exit 0. No "converged" without `decide` exit 0. +No delivery without `decide` exit 0 AND the human's approval. +``` + +## Procedure + +Run every verb from the repo root with `--repo .`. Each prints one JSON line and sets a process exit code - **branch on the exit code (`$?`), not on your own judgement.** (`python -m pandavas` requires the engine installed: `pip install -e .` or `pip install pandavas`.) + +0. **Parse** `$ARGUMENTS` into a repo path (default `.`) and the task text. + +1. **Baseline.** `python -m pandavas baseline --repo . --task ""` + - Add `--test-command ""` if detection is wrong, `--junit-xml ` for per-test rigor on non-pytest frameworks. + - Read `baseline_failures`: these pre-existing failures are **tolerated** - do not try to fix them. + - A clean error here means no test command could be found; pass `--test-command`. + +2. **Research ONCE (Nakula).** Explore with Read/Grep/Glob. Derive concrete, testable `acceptance_criteria` and anchors (real repo-relative paths + 1-indexed line ranges you actually read). Write `.pandavas/brief.json` (schema below), then `python -m pandavas resolve-brief --repo .` + - Exit 1: read `failures`, fix the anchors, rewrite, retry a few times. + - Cannot anchor your claims to real lines? **STOP and say so** - never guess. + - The brief is cached; do NOT re-research on later iterations. + + ```json + {"task":"...","acceptance_criteria":["..."], + "relevant_code":[{"path":"f.py","line_start":1,"line_end":9,"snippet":"...","why":"..."}], + "conventions":[],"integration_points":[],"constraints":[],"open_questions":[],"confidence":{}} + ``` + +3. **Iterate** - worker -> test -> judge -> decide: + a. **Worker (Arjuna).** Re-read the REAL code at the anchors (snippets only orient). Make the change with Edit/Write. If it is a bug with no test covering it, ADD a reproduction test as part of the change (fails before, passes after). On a retry, address the previous `decide` feedback. + b. **Test (Bhima).** `python -m pandavas run-tests --repo .` - exit 0 = no new failures vs baseline; exit 1 = you introduced a failure or regression. Fix it; do NOT proceed on exit 1. + c. **Diff.** `python -m pandavas diff --repo .` (renders `.pandavas/diffs/cumulative.diff` for review). + d. **Judge-gate.** `python -m pandavas judge-gate --repo .` - exit 30 = auto-reject (failing tests or no change): skip the LLM judge, go to (f) with `--approved false` and the printed `feedback`. Exit 0 = go to (e). + e. **Judge (Sahadeva, fresh context).** Spawn the **pandavas-judge** subagent (Task tool) with ONLY: the `acceptance_criteria`, the cumulative diff, and any task-relevant tests still failing; it returns `{"approved": bool, "feedback": str}`. If your harness has no subagent primitive (e.g. Cursor), instead do one clean review pass as an independent reviewer who did NOT write the change, weighing only the diff + criteria + test state. Prefer the subagent when available - it is genuinely independent; the fallback is weaker. + f. **Decide (Yudhishthira).** `python -m pandavas decide --repo . --approved --feedback ""` - branch on exit code: + - **0** -> converged. Go to 4. + - **10** -> continue. Read `feedback`, loop to (a). + - **20** -> stopped (`did_not_converge`); `decide` has already restored the best attempt to disk. Go to 5. + +4. **Deliver (converged).** Show the cumulative diff and the red->green tests (`newly_passing`). Then ask the human before committing: `python -m pandavas commit --repo . --task ""` (or `--branch `). **Never commit without approval.** + +5. **Honest failure (did_not_converge).** Report the `termination_reason` (`cap` / `oscillation`), the best attempt now on disk, and which task-relevant tests still fail. Never dress a failing result as a pass. + +## Output + +``` +### Task + on + +### Loop +baseline failures (tolerated): <...> +iterations: status: converged | did_not_converge () + +### Change + +red->green: + +### Proof (deterministic verdicts - the "verified" column) +$ python -m pandavas run-tests --repo . -> exit 0 +$ python -m pandavas decide ... --approved true -> status converged (exit 0) + +### Not verified + +``` + +Close with the **agent-suggested vs. deterministically-verified** split: the LLM steps (research, the edit, the judge's reasoning) are *suggested*; the verb exit codes (`resolve-brief`, `run-tests`, `decide`) are *verified*. If a claim has no exit code behind it, label it unverified. diff --git a/src/pandavas/cli.py b/src/pandavas/cli.py index 46f13bb..7e1d4ab 100644 --- a/src/pandavas/cli.py +++ b/src/pandavas/cli.py @@ -14,7 +14,7 @@ import sys from typing import Optional, Sequence -from . import gitutils, nodes, orchestrator +from . import gitutils, nodes, orchestrator, spine from .executor import LocalExecutor from .llm import LLMClient @@ -71,6 +71,9 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Do not create a git branch/commit on a converged change.", ) + + # Keyless deterministic verbs for skill mode (baseline, run-tests, decide, ...). + spine.add_subparsers(sub) return parser @@ -143,6 +146,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser = _build_parser() args = parser.parse_args(argv) + # Skill-mode verbs are deterministic and self-contained; dispatch them before + # any run-specific argument is touched. The `run` path below is unchanged. + if args.command != "run": + return spine.dispatch(args) + executor = LocalExecutor(junit_xml=args.junit_xml) run_kwargs = { "task": args.task, diff --git a/src/pandavas/retrieval.py b/src/pandavas/retrieval.py index 09c463b..05ada71 100644 --- a/src/pandavas/retrieval.py +++ b/src/pandavas/retrieval.py @@ -13,6 +13,7 @@ IGNORE_DIRS = { ".git", + ".pandavas", # pandavas skill-mode working/state dir (never repo content) "node_modules", "__pycache__", ".venv", diff --git a/src/pandavas/spine.py b/src/pandavas/spine.py new file mode 100644 index 0000000..be9bc51 --- /dev/null +++ b/src/pandavas/spine.py @@ -0,0 +1,632 @@ +"""Deterministic verb spine for pandavas "skill mode" (keyless). + +This module exposes the project's LLM-free machinery as a set of small, +stateless CLI subcommands. A coding harness (Claude Code, Cursor, ...) plays the +LLM agents (Nakula research, Arjuna worker, Sahadeva judge) with its own model +and calls these verbs via the shell; the verbs are the *rules* the host model +must obey. There is no LLM and no API key here -- everything is a thin wrapper +over executor / testresults / edits / diffing / gitutils / brief plus three pure +helpers from nodes (make_bhima_test, _better, _restore_snapshot), so skill mode +shares the EXACT determinism that `python -m pandavas run` is CI-tested against. + +Design rules: + * State persists between stateless verb calls in a ``.pandavas/`` working + directory under the target repo (excluded from the user's git). + * ``decide`` is the single writer of ``loop.json`` and the single authority for + diffs / signatures / best-attempt: it recomputes them from snapshots and + never trusts a model-supplied diff, so the loop cannot be gamed. + * All stdout is a single ASCII JSON object (``ensure_ascii=True``); large + payloads (diffs, test logs) are written to files and only their paths + + ASCII summaries are printed. True errors print ``{"error": ...}`` to stderr + and exit 1. + * Exit codes are distinct so the host model can branch on ``$?``: + 0 ok / converged / review + 1 error / no-test-command / dangling-anchor / not-committed + 10 decide: continue (retry) + 20 decide: stop (did_not_converge) + 30 judge-gate: auto-reject (skip the LLM judge) +""" + +from __future__ import annotations + +import datetime +import hashlib +import json +import os +import subprocess +import sys +from typing import Optional + +from . import diffing, edits, gitutils +from .brief import ResearchBrief, resolve_brief +from .executor import LocalExecutor, detect_test_command +from .nodes import _better, _restore_snapshot, make_bhima_test +from .testresults import failures + +STATE_DIR_NAME = ".pandavas" +SCHEMA_VERSION = 1 +_TAIL_LIMIT = 2000 + + +# --- state-dir plumbing -------------------------------------------------------- + + +def _state_dir(repo: str) -> str: + return os.path.join(repo, STATE_DIR_NAME) + + +def _ensure_state(repo: str) -> str: + state = _state_dir(repo) + os.makedirs(os.path.join(state, "snapshots"), exist_ok=True) + os.makedirs(os.path.join(state, "diffs"), exist_ok=True) + return state + + +def _sp(repo: str, *parts: str) -> str: + """Path inside the repo's .pandavas/ state dir.""" + return os.path.join(_state_dir(repo), *parts) + + +def _read_json(path: str) -> Optional[dict]: + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return None + + +def _write_json(path: str, data: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + # utf-8 file; readability over ASCII (ASCII only constrains stdout). + json.dump(data, f, indent=2, ensure_ascii=False) + + +def _write_text(path: str, text: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(text) + + +def _read_text(path: str) -> str: + try: + with open(path, "r", encoding="utf-8") as f: + return f.read() + except OSError: + return "" + + +def _emit(payload: dict) -> int: + """Print one ASCII JSON object to stdout. Returns 0 (caller may override).""" + print(json.dumps(payload, ensure_ascii=True)) + return 0 + + +def _err(message: str) -> int: + """Print an error as JSON to stderr and return exit code 1.""" + print(json.dumps({"error": message}, ensure_ascii=True), file=sys.stderr) + return 1 + + +def _now() -> str: + return datetime.datetime.now().isoformat(timespec="seconds") + + +def _repo_snapshot(repo: str) -> dict: + """Snapshot the repo, defensively dropping any ``.pandavas/`` keys so the + state dir never pollutes diffs, signatures, best-attempt, or restore.""" + snap = diffing.snapshot(repo) + prefix = STATE_DIR_NAME + "/" + return { + k: v + for k, v in snap.items() + if k != STATE_DIR_NAME and not k.startswith(prefix) + } + + +def _git_dir(repo: str) -> Optional[str]: + """Resolve the repo's git dir (handles worktrees/submodules), or None.""" + try: + out = subprocess.run( + ["git", "rev-parse", "--git-dir"], + cwd=repo, + capture_output=True, + text=True, + ) + except (OSError, FileNotFoundError): + return None + if out.returncode != 0: + return None + gitdir = out.stdout.strip() + if not gitdir: + return None + return gitdir if os.path.isabs(gitdir) else os.path.join(repo, gitdir) + + +def _exclude_pandavas(repo: str) -> bool: + """Append ``.pandavas/`` to /info/exclude (local, non-invasive) so the + state dir is never staged and the user's tracked .gitignore is untouched.""" + gitdir = _git_dir(repo) + if gitdir is None: + return False + info = os.path.join(gitdir, "info") + exclude = os.path.join(info, "exclude") + line = STATE_DIR_NAME + "/" + existing = _read_text(exclude) + if line in existing.splitlines(): + return True + try: + os.makedirs(info, exist_ok=True) + with open(exclude, "a", encoding="utf-8") as f: + if existing and not existing.endswith("\n"): + f.write("\n") + f.write(line + "\n") + return True + except OSError: + return False + + +def _failing_count(per_test: Optional[dict], test_passed: bool) -> int: + if per_test is not None: + return len(failures(per_test)) + return 0 if test_passed else 1 + + +# --- verbs --------------------------------------------------------------------- + + +def cmd_baseline(args) -> int: + """Run the suite once, capture baseline + snapshots, init loop.json.""" + repo = args.repo + if not os.path.isdir(repo): + return _err(f"repo path is not a directory: {repo!r}") + + loop_path = _sp(repo, "loop.json") + if os.path.isfile(loop_path) and not args.force: + return _err( + "an existing .pandavas/loop.json was found; pass --force to " + "re-baseline and discard the current run state." + ) + + command = args.test_command or detect_test_command(repo) + if command is None: + return _err( + f"could not detect a test command for {repo!r}; pass --test-command." + ) + + _ensure_state(repo) + executor = LocalExecutor(junit_xml=args.junit_xml) + result = executor.run_tests(repo, command) + + snap = _repo_snapshot(repo) + _write_json(_sp(repo, "snapshots", "baseline.json"), snap) + _write_json(_sp(repo, "snapshots", "pre_iter.json"), snap) + + _write_json( + _sp(repo, "baseline.json"), + { + "version": SCHEMA_VERSION, + "task": args.task, + "repo_path": os.path.abspath(repo), + "test_command": result.command, + "junit_xml": args.junit_xml, + "baseline_per_test": result.per_test, + "baseline_passed": result.passed, + "baseline_exit_code": result.exit_code, + }, + ) + _write_json( + loop_path, + { + "version": SCHEMA_VERSION, + "task": args.task, + "repo_path": os.path.abspath(repo), + "iteration": 1, + "max_iterations": args.max_iterations, + "status": "running", + "termination_reason": None, + "iteration_signatures": [], + "best_attempt": None, + "trace": [], + "updated_at": _now(), + }, + ) + excluded = _exclude_pandavas(repo) + + base_fail = sorted(failures(result.per_test)) if result.per_test else [] + return _emit( + { + "test_command": result.command, + "baseline_passed": result.passed, + "baseline_failures": base_fail, + "per_test_available": result.per_test is not None, + "max_iterations": args.max_iterations, + "git_excluded": excluded, + "state_dir": STATE_DIR_NAME, + } + ) + + +def cmd_detect_test(args) -> int: + command = detect_test_command(args.repo) + _emit({"test_command": command}) + return 0 if command is not None else 1 + + +def cmd_resolve_brief(args) -> int: + """Resolve gate: every anchor must point at real files/lines in the repo.""" + repo = args.repo + brief_path = args.brief + if not os.path.isabs(brief_path): + brief_path = os.path.join(repo, brief_path) + + data = _read_json(brief_path) + if data is None: + return _err(f"could not read brief JSON at {brief_path!r}") + + try: + brief = ResearchBrief(**data) + except Exception as exc: # pydantic ValidationError / TypeError + return _emit_fail_resolve([f"invalid ResearchBrief JSON: {exc}"]) + + fails = resolve_brief(brief, repo) + if fails: + return _emit_fail_resolve(fails) + + # Cache the validated brief at the canonical path (research-once). + _ensure_state(repo) + _write_json(_sp(repo, "brief.json"), brief.model_dump()) + _emit({"resolved": True, "failures": [], "anchors": len(brief.relevant_code)}) + return 0 + + +def _emit_fail_resolve(fails: list) -> int: + _emit({"resolved": False, "failures": fails}) + return 1 + + +def cmd_apply_edits(args) -> int: + """Optional path-safe full-file writer (Arjuna may edit files directly).""" + repo = args.repo + edits_path = args.edits + if not os.path.isabs(edits_path): + edits_path = os.path.join(repo, edits_path) + data = _read_json(edits_path) + if data is None or not isinstance(data, list): + return _err(f"edits JSON at {edits_path!r} must be a list of {{path, content}}") + + written, rejected = edits.apply_edits(repo, data) + _emit({"written": written, "rejected": rejected}) + return 0 if not rejected else 1 + + +def cmd_run_tests(args) -> int: + """Regression-aware verdict vs the baseline. Exit 0 iff no new failures.""" + repo = args.repo + base = _read_json(_sp(repo, "baseline.json")) + if base is None: + return _err("no .pandavas/baseline.json; run `pandavas baseline` first.") + + executor = LocalExecutor(junit_xml=base.get("junit_xml")) + state = { + "repo_path": repo, + "test_command": base.get("test_command"), + "baseline_test_results": base.get("baseline_per_test"), + } + out = make_bhima_test(executor)(state) + result = out["last_test_result"] + test_passed = bool(out["test_passed"]) + failing_count = _failing_count(result.per_test, test_passed) + + _write_text( + _sp(repo, "last_test_output.txt"), + (result.stdout or "") + (result.stderr or ""), + ) + _write_json( + _sp(repo, "last_test.json"), + { + "version": SCHEMA_VERSION, + "test_passed": test_passed, + "exit_code": result.exit_code, + "command": result.command, + "duration_s": result.duration_s, + "regressions": out["regressions"], + "new_failures": out["new_failures"], + "newly_passing": out["newly_passing"], + "failing_count": failing_count, + "per_test": result.per_test, + }, + ) + _emit( + { + "test_passed": test_passed, + "exit_code": result.exit_code, + "regressions": out["regressions"], + "new_failures": out["new_failures"], + "newly_passing": out["newly_passing"], + "failing_count": failing_count, + "output_path": STATE_DIR_NAME + "/last_test_output.txt", + } + ) + return 0 if test_passed else 1 + + +def cmd_diff(args) -> int: + """Render a cumulative (vs baseline) or per-iteration (vs pre_iter) diff.""" + repo = args.repo + base_name = "baseline.json" if args.mode == "cumulative" else "pre_iter.json" + before = _read_json(_sp(repo, "snapshots", base_name)) + if before is None: + return _err(f"missing snapshot {base_name}; run `pandavas baseline` first.") + after = _repo_snapshot(repo) + d = diffing.compute_diff(before, after) + changed = sorted(diffing.changed_files(before, after)) + out_path = _sp(repo, "diffs", f"{args.mode}.diff") + _write_text(out_path, d) + _emit( + { + "mode": args.mode, + "diff_path": STATE_DIR_NAME + f"/diffs/{args.mode}.diff", + "changed_files": changed, + "n_changed": len(changed), + } + ) + return 0 + + +def cmd_judge_gate(args) -> int: + """Deterministic pre-LLM judge gate: reject failing/empty-diff without tokens. + + Exit 0 -> proceed to the LLM judge (tests pass and a real change exists). + Exit 30 -> auto-reject; feed the printed feedback to `decide --approved false`. + """ + repo = args.repo + last = _read_json(_sp(repo, "last_test.json")) + if last is None: + return _err("no .pandavas/last_test.json; run `pandavas run-tests` first.") + + if not last["test_passed"]: + tail = _read_text(_sp(repo, "last_test_output.txt")).strip()[-_TAIL_LIMIT:] + feedback = "Tests are failing:\n" + tail + if last.get("new_failures"): + feedback += "\nNew failures: " + ", ".join(last["new_failures"]) + if last.get("regressions"): + feedback += "\nRegressions: " + ", ".join(last["regressions"]) + _emit({"gate": "reject", "reason": "failing_tests", "feedback": feedback}) + return 30 + + before = _read_json(_sp(repo, "snapshots", "baseline.json")) or {} + cum = diffing.compute_diff(before, _repo_snapshot(repo)) + if cum.strip() == "": + _emit( + { + "gate": "reject", + "reason": "empty_diff", + "feedback": ( + "No change was made (empty diff); a fix task cannot be " + "satisfied without a change." + ), + } + ) + return 30 + + _emit({"gate": "review", "reason": None}) + return 0 + + +def cmd_decide(args) -> int: + """Loop control (Yudhishthira): trace, best-attempt, oscillation, cap, restore. + + Faithful port of nodes.yudhishthira over on-disk state. Sole writer of + loop.json; recomputes diffs/signatures from snapshots. + + Exit 0 -> converged. + Exit 10 -> continue (retry); the next worker pass should address `feedback`. + Exit 20 -> stop (did_not_converge); best attempt restored to disk. + """ + repo = args.repo + loop = _read_json(_sp(repo, "loop.json")) + last = _read_json(_sp(repo, "last_test.json")) + baseline_snap = _read_json(_sp(repo, "snapshots", "baseline.json")) + pre_iter_snap = _read_json(_sp(repo, "snapshots", "pre_iter.json")) + if loop is None or last is None or baseline_snap is None or pre_iter_snap is None: + return _err( + "missing run state (loop.json / last_test.json / snapshots); run " + "`pandavas baseline` then `pandavas run-tests` before `decide`." + ) + + it = loop["iteration"] + max_it = loop["max_iterations"] + approved = args.approved == "true" + feedback = args.feedback or "" + + after = _repo_snapshot(repo) + per_iter_diff = diffing.compute_diff(pre_iter_snap, after) + cumulative = diffing.compute_diff(baseline_snap, after) + + new_failures = list(last.get("new_failures") or []) + record = { + "iteration": it, + "test_passed": last.get("test_passed"), + "judge_approved": approved, + "judge_feedback": feedback, + "diff": per_iter_diff, + "regressions": list(last.get("regressions") or []), + "new_failures": new_failures, + "newly_passing": list(last.get("newly_passing") or []), + } + trace = list(loop.get("trace") or []) + [record] + + cand = { + "iteration": it, + "new_failures": new_failures, + "new_failures_count": len(new_failures), + "failing_count": last.get("failing_count", 0), + "diff": cumulative, + } + best = loop.get("best_attempt") + if _better(cand, best): + best = cand + _write_json(_sp(repo, "snapshots", "best.json"), after) + + if approved: + loop.update( + { + "status": "converged", + "trace": trace, + "best_attempt": best, + "updated_at": _now(), + } + ) + _write_json(_sp(repo, "loop.json"), loop) + _emit( + { + "status": "converged", + "iteration": it, + "newly_passing": record["newly_passing"], + "diff_path": STATE_DIR_NAME + "/diffs/cumulative.diff", + } + ) + _write_text(_sp(repo, "diffs", "cumulative.diff"), cumulative) + return 0 + + sig = [hashlib.sha256(per_iter_diff.encode("utf-8")).hexdigest(), sorted(new_failures)] + seen = loop.get("iteration_signatures") or [] + updated_sigs = seen + [sig] + + def _stop(reason: str) -> int: + restored = False + if best is not None and best["iteration"] != it: + best_snap = _read_json(_sp(repo, "snapshots", "best.json")) + # Only restore against a real snapshot; never restore to an empty one + # (a missing best.json must not wipe the repo). + if best_snap is not None: + _restore_snapshot(repo, best_snap, after) + restored = True + loop.update( + { + "status": "did_not_converge", + "termination_reason": reason, + "iteration_signatures": updated_sigs, + "trace": trace, + "best_attempt": best, + "updated_at": _now(), + } + ) + _write_json(_sp(repo, "loop.json"), loop) + _emit( + { + "status": "did_not_converge", + "termination_reason": reason, + "iteration": it, + "best_iteration": best["iteration"] if best else None, + "restored_best": restored, + "new_failures": best["new_failures"] if best else new_failures, + } + ) + return 20 + + if sig in seen: + return _stop("oscillation") + if it >= max_it: + return _stop("cap") + + # Continue: roll pre_iter forward so the next pass diffs against this state. + _write_json(_sp(repo, "snapshots", "pre_iter.json"), after) + loop.update( + { + "iteration": it + 1, + "status": "running", + "iteration_signatures": updated_sigs, + "trace": trace, + "best_attempt": best, + "updated_at": _now(), + } + ) + _write_json(_sp(repo, "loop.json"), loop) + _emit({"status": "running", "iteration": it + 1, "feedback": feedback}) + return 10 + + +def cmd_restore(args) -> int: + name = "best.json" if args.to == "best" else "baseline.json" + snap = _read_json(_sp(args.repo, "snapshots", name)) + if snap is None: + return _err(f"no snapshot to restore ({name} missing).") + _restore_snapshot(args.repo, snap, _repo_snapshot(args.repo)) + _emit({"restored": args.to}) + return 0 + + +def cmd_commit(args) -> int: + branch = args.branch or gitutils.slug_branch(args.task) + message = args.message or f"pandavas: {args.task}" + result = gitutils.commit_change(args.repo, branch, message) + _emit(result) + return 0 if result["committed"] else 1 + + +# --- parser wiring ------------------------------------------------------------- + + +def add_subparsers(sub) -> None: + """Register every spine verb on the shared subparsers object from cli.py.""" + + def _with_repo(p): + p.add_argument("--repo", required=True, help="Path to the local repository.") + return p + + p = _with_repo(sub.add_parser("baseline", help="Run the suite once; init run state.")) + p.add_argument("--task", required=True, help="Task description.") + p.add_argument("--test-command", default=None, help="Override the test command.") + p.add_argument("--junit-xml", default=None, help="JUnit XML path for per-test rigor.") + p.add_argument("--max-iterations", type=int, default=6, help="Hard retry cap.") + p.add_argument("--force", action="store_true", help="Discard existing run state.") + p.set_defaults(func=cmd_baseline) + + p = _with_repo(sub.add_parser("detect-test", help="Print the detected test command.")) + p.set_defaults(func=cmd_detect_test) + + p = _with_repo(sub.add_parser("resolve-brief", help="Resolve-gate a research brief.")) + p.add_argument( + "--brief", + default=os.path.join(STATE_DIR_NAME, "brief.json"), + help="Brief JSON path (default .pandavas/brief.json, relative to --repo).", + ) + p.set_defaults(func=cmd_resolve_brief) + + p = _with_repo(sub.add_parser("apply-edits", help="Path-safe full-file writes.")) + p.add_argument("--edits", required=True, help="JSON list of {path, content}.") + p.set_defaults(func=cmd_apply_edits) + + p = _with_repo(sub.add_parser("run-tests", help="Regression-aware test verdict.")) + p.set_defaults(func=cmd_run_tests) + + p = _with_repo(sub.add_parser("diff", help="Render cumulative/iter diff to a file.")) + p.add_argument( + "--mode", choices=["cumulative", "iter"], default="cumulative" + ) + p.set_defaults(func=cmd_diff) + + p = _with_repo(sub.add_parser("judge-gate", help="Deterministic pre-LLM judge gate.")) + p.set_defaults(func=cmd_judge_gate) + + p = _with_repo(sub.add_parser("decide", help="Loop control: converge/retry/stop.")) + p.add_argument("--approved", choices=["true", "false"], required=True) + p.add_argument("--feedback", default=None, help="Judge feedback for the next pass.") + p.set_defaults(func=cmd_decide) + + p = _with_repo(sub.add_parser("restore", help="Restore best/baseline snapshot.")) + p.add_argument("--to", choices=["best", "baseline"], default="best") + p.set_defaults(func=cmd_restore) + + p = _with_repo(sub.add_parser("commit", help="Commit a converged change to a branch.")) + p.add_argument("--task", required=True, help="Task (for the default branch/message).") + p.add_argument("--branch", default=None, help="Branch name (default: auto from task).") + p.add_argument("--message", default=None, help="Commit message.") + p.set_defaults(func=cmd_commit) + + +def dispatch(args) -> int: + """Run the spine verb selected by argparse (its func default).""" + return args.func(args) diff --git a/tests/test_skill_e2e.py b/tests/test_skill_e2e.py new file mode 100644 index 0000000..56ea74b --- /dev/null +++ b/tests/test_skill_e2e.py @@ -0,0 +1,125 @@ +"""End-to-end skill-mode test: a scripted 'host model' drives the deterministic +verbs over the real fixture with real pytest -- and NO API key. + +This is the verb-surface analogue of examples/demo.py and the --offline check: it +proves baseline -> resolve-brief -> (edit) -> run-tests -> diff -> judge-gate -> +decide wire together and gate correctly, with the host's LLM steps replaced by a +script. The positive case converges red->green and commits; the negative case +(a no-op worker) does not converge. +""" + +import json +import os +import shutil +import subprocess + +from pandavas.cli import main + +FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "sample_buggy_repo") + + +def _last(capsys) -> dict: + return json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + + +def _copy_fixture(tmp_path) -> str: + repo = os.path.join(str(tmp_path), "repo") + shutil.copytree(FIXTURE, repo) + return repo + + +def _git_init(repo, *, initial_commit=True): + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "t@t.t"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "t"], cwd=repo, check=True) + if initial_commit: + subprocess.run(["git", "add", "-A"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=repo, check=True) + + +def _read_loop(repo) -> dict: + with open(os.path.join(repo, ".pandavas", "loop.json"), encoding="utf-8") as f: + return json.load(f) + + +def test_skill_mode_converges_red_to_green_and_commits(tmp_path, capsys): + repo = _copy_fixture(tmp_path) + _git_init(repo) + + # 1. baseline -- the bug's test is failing at baseline (tolerated by the gate). + assert main(["baseline", "--repo", repo, "--task", "fix add"]) == 0 + base = _last(capsys) + assert base["baseline_passed"] is False + assert "test_calc::test_add" in base["baseline_failures"] + + # 2. research (host model): an anchored brief -> the resolve gate passes. + with open(os.path.join(repo, ".pandavas", "brief.json"), "w", encoding="utf-8") as f: + json.dump( + { + "task": "fix add", + "acceptance_criteria": ["add(2, 3) == 5"], + "relevant_code": [ + {"path": "calc.py", "line_start": 1, "line_end": 2, + "snippet": "def add", "why": "the buggy function"} + ], + "conventions": [], "integration_points": [], "constraints": [], + "open_questions": [], "confidence": {}, + }, + f, + ) + assert main(["resolve-brief", "--repo", repo]) == 0 + capsys.readouterr() + + # 3. worker (host model): apply the fix at the anchor. + calc = os.path.join(repo, "calc.py") + with open(calc, encoding="utf-8") as f: + src = f.read() + with open(calc, "w", encoding="utf-8") as f: + f.write(src.replace("a - b", "a + b")) + + # 4. test -- green now, and the fix surfaces as red->green. + assert main(["run-tests", "--repo", repo]) == 0 + assert _last(capsys)["newly_passing"] == ["test_calc::test_add"] + + # 5. diff + 6. judge-gate (deterministic) -> review. + assert main(["diff", "--repo", repo]) == 0 + capsys.readouterr() + assert main(["judge-gate", "--repo", repo]) == 0 + assert _last(capsys)["gate"] == "review" + + # 7. judge (host model) approves -> decide converges. + assert main(["decide", "--repo", repo, "--approved", "true"]) == 0 + assert _last(capsys)["status"] == "converged" + assert _read_loop(repo)["status"] == "converged" + + # 8. deliver -- committed to a branch, and .pandavas/ is NOT in the commit. + assert main(["commit", "--repo", repo, "--task", "fix add"]) == 0 + assert _last(capsys)["committed"] is True + show = subprocess.run( + ["git", "show", "HEAD", "--name-only", "--format="], + cwd=repo, capture_output=True, text=True, + ) + committed = [line for line in show.stdout.splitlines() if line.strip()] + assert "calc.py" in committed + assert not any(p.startswith(".pandavas") for p in committed) + + +def test_skill_mode_no_op_worker_does_not_converge(tmp_path, capsys): + repo = _copy_fixture(tmp_path) + + assert main(["baseline", "--repo", repo, "--task", "fix add"]) == 0 + capsys.readouterr() + + # No-op "worker": nothing changes. The pre-existing failure is tolerated... + assert main(["run-tests", "--repo", repo]) == 0 + capsys.readouterr() + # ...but there is no change, so the deterministic gate rejects (skips the LLM). + assert main(["judge-gate", "--repo", repo]) == 30 + capsys.readouterr() + + # First decide continues; a second identical pass is detected as oscillation. + assert main(["decide", "--repo", repo, "--approved", "false"]) == 10 + capsys.readouterr() + assert main(["decide", "--repo", repo, "--approved", "false"]) == 20 + assert _last(capsys)["termination_reason"] == "oscillation" + assert _read_loop(repo)["status"] == "did_not_converge" diff --git a/tests/test_spine.py b/tests/test_spine.py new file mode 100644 index 0000000..eb1c3f3 --- /dev/null +++ b/tests/test_spine.py @@ -0,0 +1,432 @@ +"""Tests for the deterministic verb spine (skill mode). All keyless, no network. + +Each verb is invoked through the real CLI entrypoint (`main([...])`) so argument +parsing + dispatch are exercised too. The executor is faked where deterministic +per-test results matter; git verbs use a real local repo. +""" + +import json +import os +import subprocess +import sys + +import pytest + +from pandavas import spine +from pandavas.cli import main +from pandavas.executor import Executor, TestResult + +STATE = ".pandavas" + + +# --- helpers ------------------------------------------------------------------- + + +def _out(capsys) -> dict: + """Parse the last stdout line as JSON (and assert it is pure ASCII).""" + captured = capsys.readouterr() + text = captured.out.strip() + assert text, f"no stdout (stderr={captured.err!r})" + text.encode("ascii") # stdout must be ASCII-only (Windows-first) + return json.loads(text.splitlines()[-1]) + + +def _err(capsys) -> str: + return capsys.readouterr().err + + +def _read(repo, *parts): + with open(os.path.join(repo, STATE, *parts), "r", encoding="utf-8") as f: + return json.load(f) + + +class _FakeExec(Executor): + """Returns a controllable TestResult without running anything.""" + + def __init__(self): + self.per_test = None + self.passed = True + self.exit_code = 0 + + def run_tests(self, repo_path, test_command=None) -> TestResult: + return TestResult( + passed=self.passed, + exit_code=self.exit_code, + stdout="", + stderr="", + command="pytest", + duration_s=0.0, + per_test=self.per_test, + ) + + +def _use_fake(monkeypatch) -> _FakeExec: + fake = _FakeExec() + monkeypatch.setattr(spine, "LocalExecutor", lambda junit_xml=None: fake) + return fake + + +def _git_init(repo): + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "t@t.t"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "t"], cwd=repo, check=True) + + +def _seed(repo, *, iteration=1, max_iterations=6, signatures=None, best=None, + test_passed=True, new_failures=None, failing_count=0, + baseline=None, pre_iter=None): + """Write a minimal, consistent .pandavas/ state for decide/judge-gate tests.""" + repo = str(repo) + sd = os.path.join(repo, STATE) + os.makedirs(os.path.join(sd, "snapshots"), exist_ok=True) + os.makedirs(os.path.join(sd, "diffs"), exist_ok=True) + + def w(rel, data): + with open(os.path.join(sd, rel), "w", encoding="utf-8") as f: + json.dump(data, f) + + w("loop.json", { + "version": 1, "task": "t", "repo_path": repo, "iteration": iteration, + "max_iterations": max_iterations, "status": "running", + "termination_reason": None, "iteration_signatures": signatures or [], + "best_attempt": best, "trace": [], "updated_at": "x", + }) + w("last_test.json", { + "version": 1, "test_passed": test_passed, + "exit_code": 0 if test_passed else 1, "command": "pytest", + "duration_s": 0.0, "regressions": [], "new_failures": new_failures or [], + "newly_passing": [], "failing_count": failing_count, "per_test": None, + }) + base = baseline if baseline is not None else {} + w(os.path.join("snapshots", "baseline.json"), base) + w(os.path.join("snapshots", "pre_iter.json"), + pre_iter if pre_iter is not None else base) + + +# --- detect-test --------------------------------------------------------------- + + +def test_detect_test_finds_pytest(tmp_path, capsys): + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + code = main(["detect-test", "--repo", str(tmp_path)]) + assert code == 0 + assert _out(capsys)["test_command"] == "pytest" + + +def test_detect_test_none_exits_1(tmp_path, capsys): + code = main(["detect-test", "--repo", str(tmp_path)]) + assert code == 1 + assert _out(capsys)["test_command"] is None + + +# --- baseline ------------------------------------------------------------------ + + +def test_baseline_writes_state_and_git_exclude(tmp_path, capsys, monkeypatch): + repo = tmp_path + (repo / "a.txt").write_text("hello\n", encoding="utf-8") + _git_init(str(repo)) + fake = _use_fake(monkeypatch) + fake.per_test = {"t1": "failed"} + fake.passed = False + + code = main(["baseline", "--repo", str(repo), "--task", "fix it", + "--test-command", "pytest"]) + data = _out(capsys) + + assert code == 0 + assert data["baseline_passed"] is False + assert data["baseline_failures"] == ["t1"] + assert data["git_excluded"] is True + loop = _read(str(repo), "loop.json") + assert loop["iteration"] == 1 and loop["status"] == "running" + # .pandavas/ is locally excluded so a later commit never stages it. + exclude = (repo / ".git" / "info" / "exclude").read_text(encoding="utf-8") + assert ".pandavas/" in exclude.splitlines() + + +def test_baseline_refuses_to_clobber_without_force(tmp_path, capsys, monkeypatch): + repo = tmp_path + _use_fake(monkeypatch) + main(["baseline", "--repo", str(repo), "--task", "t", "--test-command", "pytest"]) + capsys.readouterr() + code = main(["baseline", "--repo", str(repo), "--task", "t", + "--test-command", "pytest"]) + assert code == 1 + assert "force" in _err(capsys).lower() + + +def test_baseline_no_test_command_clean_error(tmp_path, capsys): + # Empty repo, no override -> clean JSON error to stderr, no traceback, exit 1. + code = main(["baseline", "--repo", str(tmp_path), "--task", "t"]) + err = _err(capsys) + assert code == 1 + assert "error" in err and "Traceback" not in err + + +# --- resolve-brief ------------------------------------------------------------- + + +def _write_brief(repo, anchors): + brief = { + "task": "t", "acceptance_criteria": ["c"], "relevant_code": anchors, + "conventions": [], "integration_points": [], "constraints": [], + "open_questions": [], "confidence": {}, + } + os.makedirs(os.path.join(repo, STATE), exist_ok=True) + with open(os.path.join(repo, STATE, "brief.json"), "w", encoding="utf-8") as f: + json.dump(brief, f) + + +def test_resolve_brief_ok(tmp_path, capsys): + (tmp_path / "calc.py").write_text("def add(a, b):\n return a - b\n", encoding="utf-8") + _write_brief(str(tmp_path), [ + {"path": "calc.py", "line_start": 1, "line_end": 2, "snippet": "x", "why": "y"} + ]) + code = main(["resolve-brief", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 0 and data["resolved"] is True and data["failures"] == [] + + +def test_resolve_brief_dangling_anchor(tmp_path, capsys): + (tmp_path / "calc.py").write_text("def add(a, b):\n return a - b\n", encoding="utf-8") + _write_brief(str(tmp_path), [ + {"path": "calc.py", "line_start": 1, "line_end": 999, "snippet": "x", "why": "y"} + ]) + code = main(["resolve-brief", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 1 and data["resolved"] is False and data["failures"] + + +def test_resolve_brief_path_escape_rejected(tmp_path, capsys): + _write_brief(str(tmp_path), [ + {"path": "../secret.py", "line_start": 1, "line_end": 1, "snippet": "x", "why": "y"} + ]) + code = main(["resolve-brief", "--repo", str(tmp_path)]) + assert code == 1 and _out(capsys)["resolved"] is False + + +# --- apply-edits --------------------------------------------------------------- + + +def test_apply_edits_writes_and_rejects_escape(tmp_path, capsys): + edits = [ + {"path": "src/new.py", "content": "x = 1\n"}, + {"path": "../escape.py", "content": "bad"}, + ] + p = tmp_path / "edits.json" + p.write_text(json.dumps(edits), encoding="utf-8") + code = main(["apply-edits", "--repo", str(tmp_path), "--edits", str(p)]) + data = _out(capsys) + assert code == 1 # a rejection occurred + assert data["written"] == ["src/new.py"] + assert data["rejected"] == ["../escape.py"] + assert (tmp_path / "src" / "new.py").read_text(encoding="utf-8") == "x = 1\n" + + +# --- run-tests (regression-aware verdict) -------------------------------------- + + +def _baseline_with(repo, fake, per_test, *, passed, capsys): + fake.per_test = per_test + fake.passed = passed + fake.exit_code = 0 if passed else 1 + main(["baseline", "--repo", str(repo), "--task", "t", "--test-command", "pytest"]) + capsys.readouterr() + + +def test_run_tests_tolerates_preexisting_failure(tmp_path, capsys, monkeypatch): + fake = _use_fake(monkeypatch) + _baseline_with(tmp_path, fake, {"t1": "passed", "t2": "failed"}, + passed=False, capsys=capsys) + # Same failures as baseline -> no NEW failures -> tolerated. + fake.per_test = {"t1": "passed", "t2": "failed"} + code = main(["run-tests", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 0 and data["test_passed"] is True and data["new_failures"] == [] + + +def test_run_tests_detects_regression(tmp_path, capsys, monkeypatch): + fake = _use_fake(monkeypatch) + _baseline_with(tmp_path, fake, {"t1": "passed", "t2": "failed"}, + passed=False, capsys=capsys) + fake.per_test = {"t1": "failed", "t2": "failed"} # t1 regressed + code = main(["run-tests", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 1 and data["test_passed"] is False + assert data["regressions"] == ["t1"] + + +def test_run_tests_reports_newly_passing_red_to_green(tmp_path, capsys, monkeypatch): + fake = _use_fake(monkeypatch) + _baseline_with(tmp_path, fake, {"t1": "passed", "t2": "failed"}, + passed=False, capsys=capsys) + fake.per_test = {"t1": "passed", "t2": "passed"} # t2 fixed + code = main(["run-tests", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 0 and data["newly_passing"] == ["t2"] + assert _read(str(tmp_path), "last_test.json")["failing_count"] == 0 + + +# --- diff ---------------------------------------------------------------------- + + +def test_diff_cumulative_lists_changed_file(tmp_path, capsys, monkeypatch): + repo = tmp_path + (repo / "calc.py").write_text("a\n", encoding="utf-8") + _use_fake(monkeypatch) + main(["baseline", "--repo", str(repo), "--task", "t", "--test-command", "pytest"]) + capsys.readouterr() + (repo / "calc.py").write_text("b\n", encoding="utf-8") + + code = main(["diff", "--repo", str(repo)]) + data = _out(capsys) + assert code == 0 and data["changed_files"] == ["calc.py"] and data["n_changed"] == 1 + assert os.path.isfile(os.path.join(str(repo), STATE, "diffs", "cumulative.diff")) + + +# --- judge-gate ---------------------------------------------------------------- + + +def test_judge_gate_rejects_failing_tests(tmp_path, capsys): + _seed(tmp_path, test_passed=False, new_failures=["t1"]) + os.makedirs(os.path.join(str(tmp_path), STATE), exist_ok=True) + code = main(["judge-gate", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 30 and data["gate"] == "reject" and data["reason"] == "failing_tests" + + +def test_judge_gate_rejects_empty_diff(tmp_path, capsys): + (tmp_path / "calc.py").write_text("same\n", encoding="utf-8") + _seed(tmp_path, test_passed=True, baseline={"calc.py": "same\n"}) + code = main(["judge-gate", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 30 and data["reason"] == "empty_diff" + + +def test_judge_gate_review_when_passing_and_changed(tmp_path, capsys): + (tmp_path / "calc.py").write_text("new\n", encoding="utf-8") + _seed(tmp_path, test_passed=True, baseline={"calc.py": "old\n"}) + code = main(["judge-gate", "--repo", str(tmp_path)]) + data = _out(capsys) + assert code == 0 and data["gate"] == "review" + + +# --- decide (loop control) ----------------------------------------------------- + + +def test_decide_converges_on_approval(tmp_path, capsys): + (tmp_path / "calc.py").write_text("new\n", encoding="utf-8") + _seed(tmp_path, baseline={"calc.py": "old\n"}) + code = main(["decide", "--repo", str(tmp_path), "--approved", "true"]) + data = _out(capsys) + assert code == 0 and data["status"] == "converged" + assert _read(str(tmp_path), "loop.json")["status"] == "converged" + + +def test_decide_continues_below_cap(tmp_path, capsys): + (tmp_path / "calc.py").write_text("new\n", encoding="utf-8") + _seed(tmp_path, iteration=1, max_iterations=6, baseline={"calc.py": "old\n"}) + code = main(["decide", "--repo", str(tmp_path), "--approved", "false", + "--feedback", "try again"]) + data = _out(capsys) + assert code == 10 and data["status"] == "running" and data["iteration"] == 2 + assert _read(str(tmp_path), "loop.json")["iteration"] == 2 + + +def test_decide_stops_at_cap(tmp_path, capsys): + (tmp_path / "calc.py").write_text("changed\n", encoding="utf-8") + _seed(tmp_path, iteration=2, max_iterations=2, baseline={"calc.py": "old\n"}) + code = main(["decide", "--repo", str(tmp_path), "--approved", "false"]) + data = _out(capsys) + assert code == 20 and data["termination_reason"] == "cap" + + +def test_decide_detects_oscillation(tmp_path, capsys): + # No change -> identical (empty diff, no new failures) signature twice. + (tmp_path / "a.txt").write_text("x\n", encoding="utf-8") + _seed(tmp_path, iteration=1, max_iterations=9, baseline={"a.txt": "x\n"}) + assert main(["decide", "--repo", str(tmp_path), "--approved", "false"]) == 10 + capsys.readouterr() + code = main(["decide", "--repo", str(tmp_path), "--approved", "false"]) + data = _out(capsys) + assert code == 20 and data["termination_reason"] == "oscillation" + + +def test_decide_restores_best_attempt_at_cap(tmp_path, capsys): + repo = str(tmp_path) + # Best (iteration 1) had zero new failures; current (iteration 2) is worse. + _seed(repo, iteration=2, max_iterations=2, baseline={"calc.py": "orig\n"}, + best={"iteration": 1, "new_failures": [], "new_failures_count": 0, + "failing_count": 0, "diff": "d"}, + test_passed=False, new_failures=["x"], failing_count=1) + # The recorded best snapshot, and a worse working tree with a stray file. + with open(os.path.join(repo, STATE, "snapshots", "best.json"), "w", + encoding="utf-8") as f: + json.dump({"calc.py": "GOOD\n"}, f) + (tmp_path / "calc.py").write_text("WORSE\n", encoding="utf-8") + (tmp_path / "stray.txt").write_text("junk\n", encoding="utf-8") + + code = main(["decide", "--repo", repo, "--approved", "false"]) + data = _out(capsys) + + assert code == 20 and data["restored_best"] is True + assert (tmp_path / "calc.py").read_text(encoding="utf-8") == "GOOD\n" + assert not (tmp_path / "stray.txt").exists() # stray removed on restore + + +# --- restore ------------------------------------------------------------------- + + +def test_restore_to_baseline(tmp_path, capsys): + repo = str(tmp_path) + os.makedirs(os.path.join(repo, STATE, "snapshots"), exist_ok=True) + with open(os.path.join(repo, STATE, "snapshots", "baseline.json"), "w", + encoding="utf-8") as f: + json.dump({"calc.py": "orig\n"}, f) + (tmp_path / "calc.py").write_text("messed up\n", encoding="utf-8") + + code = main(["restore", "--repo", repo, "--to", "baseline"]) + assert code == 0 and _out(capsys)["restored"] == "baseline" + assert (tmp_path / "calc.py").read_text(encoding="utf-8") == "orig\n" + + +def test_restore_missing_snapshot_errors(tmp_path, capsys): + code = main(["restore", "--repo", str(tmp_path), "--to", "best"]) + assert code == 1 and "error" in _err(capsys) + + +# --- commit -------------------------------------------------------------------- + + +def test_commit_creates_branch(tmp_path, capsys): + repo = tmp_path + (repo / "calc.py").write_text("x = 1\n", encoding="utf-8") + _git_init(str(repo)) + subprocess.run(["git", "add", "-A"], cwd=str(repo), check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=str(repo), check=True) + (repo / "calc.py").write_text("x = 2\n", encoding="utf-8") + + code = main(["commit", "--repo", str(repo), "--task", "tweak x"]) + data = _out(capsys) + assert code == 0 and data["committed"] is True + assert data["branch"] == "pandavas/tweak-x" + + +def test_commit_non_git_repo_reports_error(tmp_path, capsys): + code = main(["commit", "--repo", str(tmp_path), "--task", "x"]) + data = _out(capsys) + assert code == 1 and data["committed"] is False + + +# --- constraints --------------------------------------------------------------- + + +def test_repo_snapshot_excludes_state_dir(tmp_path): + (tmp_path / "real.py").write_text("code\n", encoding="utf-8") + os.makedirs(os.path.join(str(tmp_path), STATE), exist_ok=True) + (tmp_path / STATE / "loop.json").write_text("{}", encoding="utf-8") + + snap = spine._repo_snapshot(str(tmp_path)) + assert "real.py" in snap + assert not any(k == STATE or k.startswith(STATE + "/") for k in snap)