diff --git a/.codies-memory b/.codies-memory new file mode 100644 index 0000000..66b3304 --- /dev/null +++ b/.codies-memory @@ -0,0 +1 @@ +wishful diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index 205b0fe..0000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - prompt: | - REPO: ${{ github.repository }} - PR NUMBER: ${{ github.event.pull_request.number }} - - Please review this pull request and provide feedback on: - - Code quality and best practices - - Potential bugs or issues - - Performance considerations - - Security concerns - - Test coverage - - Use the repository's CLAUDE.md for guidance on style and conventions. Be constructive and helpful in your feedback. - - Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. - - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://docs.claude.com/en/docs/claude-code/cli-reference for available options - claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' - diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index 412cef9..0000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - - # This is an optional setting that allows Claude to read CI results on PRs - additional_permissions: | - actions: read - - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://docs.claude.com/en/docs/claude-code/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr:*)' - diff --git a/.gitignore b/.gitignore index ff9fae9..2043d19 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ node_modules/ docs-site/node_modules/ docs-site/.astro/ docs-site/dist/ +.bmad/ +.claude/ +.memsearch/ +.pyro/ diff --git a/AGENTS.md b/AGENTS.md index e583f77..7b3fcff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,6 +173,21 @@ Key properties: - **Safety‑checked**: generated code is parsed and checked for obviously dangerous constructs. - **Cache‑backed**: generated modules live as `.py` files and can be edited or committed. +### Active Product Direction: Code Search Workbench + +The current serious direction is not "one-shot helper generation." It is bounded +code search: target artifact + mutation space + fitness function + budget + +accept/rollback. Before extending `evolve`, `context`, CLI, demos, or the +dashboard, read: + +- `docs/specs/001-wishful-evolve/implementation-plan.md` +- `docs/specs/002-wishful-context/implementation-plan.md` +- `docs/specs/003-wishful-code-search-workbench/concept-plan.md` + +The key product primitive is "function with lineage": import address, current +source, context, variants, scores, failures, winner, evidence scope, and +accept/rollback state. + --- ## Repository Layout diff --git a/README.md b/README.md index 39f6ce7..7d556dd 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ PyPI version Python 3.12+ License: MIT - Tests - Coverage + Tests + Coverage Code style: ruff

@@ -246,6 +246,77 @@ It's turtles all the way down. 🐒 --- +## 🧬 Evolve: Improve a Function Over Generations + +`explore()` tries several fresh variants. `evolve()` goes one step further: it +keeps a history of prior attempts, scores, and failures, then passes that +history back into the next mutation prompt. + +```python +import wishful + + +def normalize_scores(values): + values = [float(value) for value in values] + if not values: + return [] + minimum = min(values) + maximum = max(values) + if maximum == minimum: + return [0.0 for _ in values] + return [(value - minimum) / (maximum - minimum) for value in values] + + +normalize_scores.__wishful_source__ = """ +def normalize_scores(values): + values = [float(value) for value in values] + if not values: + return [] + minimum = min(values) + maximum = max(values) + if maximum == minimum: + return [0.0 for _ in values] + return [(value - minimum) / (maximum - minimum) for value in values] +""".strip() + + +def is_correct(fn): + return ( + fn([10, 20, 30]) == [0.0, 0.5, 1.0] + and fn([2, 2, 2]) == [0.0, 0.0, 0.0] + ) + + +def score(fn): + return 1_000.0 - len(fn.__wishful_source__) if is_correct(fn) else 0.0 + + +evolved = wishful.evolve( + normalize_scores, + fitness=score, + test=is_correct, + generations=3, + variants=4, + mutation_prompt="Keep behavior identical but make the code concise.", +) + +print(evolved.__wishful_evolution__) +``` + +The returned function carries: + +- `__wishful_source__`: the winning source code +- `__wishful_evolution__`: original score, final score, improvement, generation + summaries, and every attempted variant + +Try the deterministic offline demo: + +```bash +WISHFUL_FAKE_LLM=1 uv run python examples/14_evolve.py +``` + +--- + ## πŸ—„οΈ Cache Ops: Because Sometimes Wishes Need Revising ```python diff --git a/docs/specs/001-wishful-evolve/implementation-plan.md b/docs/specs/001-wishful-evolve/implementation-plan.md new file mode 100644 index 0000000..c0bfcc4 --- /dev/null +++ b/docs/specs/001-wishful-evolve/implementation-plan.md @@ -0,0 +1,259 @@ +# Implementation Plan: wishful.evolve + +## Validation Checklist + +- [x] All specification file paths are correct and exist +- [x] Context priming section is complete +- [x] All implementation phases are defined +- [x] Each phase follows TDD: Prime β†’ Test β†’ Implement β†’ Validate +- [x] Dependencies between phases are clear (no circular dependencies) +- [x] Parallel work is properly tagged with `[parallel: true]` +- [x] Activity hints provided for specialist selection `[activity: type]` +- [x] Every phase references relevant design sections +- [x] Every test references design acceptance criteria +- [x] Integration & E2E tests defined in final phase +- [x] Project commands match actual project setup +- [x] A developer could follow this plan independently + +--- + +## Specification Compliance Guidelines + +### How to Ensure Specification Adherence + +1. **Before Each Phase**: Complete the Pre-Implementation Specification Gate +2. **During Implementation**: Reference specific design sections in each task +3. **After Each Task**: Run Specification Compliance checks +4. **Phase Completion**: Verify all specification requirements are met + +### Deviation Protocol + +If implementation cannot follow specification exactly: +1. Document the deviation and reason +2. Get approval before proceeding +3. Update design docs if the deviation is an improvement +4. Never deviate without documentation + +## Metadata Reference + +- `[parallel: true]` - Tasks that can run concurrently +- `[component: component-name]` - For multi-component features +- `[ref: document/section; lines: 1, 2-3]` - Links to specifications, patterns, or interfaces and (if applicable) line(s) +- `[activity: type]` - Activity hint for specialist agent selection + +--- + +## Context Priming + +*GATE: You MUST fully read all files mentioned in this section before starting any implementation.* + +**Design Documentation**: + +- `docs/specs/001-wishful-evolve/implementation-plan.md` - this implementation plan +- `docs/specs/002-wishful-context/product-requirements.md` - planned context layer +- `docs/specs/002-wishful-context/solution-design.md` - context integration point for evolve prompts +- `docs/specs/003-wishful-code-search-workbench/concept-plan.md` - product direction for function lineage and evidence casefiles + +**Key Design Decisions**: + +1. **History-as-Context is CRITICAL** - This is what differentiates `evolve()` from `explore()`. The LLM receives full history of previous attempts with fitness scores, enabling informed mutations (AlphaEvolve paradigm). + +2. **Parameters for History Control** - `keep_history: bool = True` and `history_limit: int = 10` control whether/how much history is passed to the LLM. + +3. **Sync API with Async Internals** - Follow the `explore()` pattern: sync public API wrapping async internals for responsive progress updates. + +4. **Reuse Existing Infrastructure** - Use `wishful.llm.client.generate_module_code()` for LLM calls, `wishful.safety.validator` for code validation. + +5. **Rich Metadata** - Evolved functions get `__wishful_evolution__` (stats) and `__wishful_source__` (code) attributes. + +**Implementation Context**: + +- **Test command**: `uv run pytest tests/test_evolve.py -v` +- **Full test suite**: `uv run pytest tests/ -v` +- **Type checking**: `uv run mypy src/wishful/evolve/` +- **Example**: `uv run python examples/14_evolve.py` +- **Fake LLM mode**: `WISHFUL_FAKE_LLM=1 uv run pytest tests/test_evolve.py -v` + +**Patterns to Follow**: +- `src/wishful/explore/` - Reference for async/sync pattern, progress display, variant handling +- `src/wishful/llm/client.py` - LLM client usage patterns +- `src/wishful/safety/validator.py` - Code validation pattern + +--- + +## Implementation Phases + +### Phase 1: Foundation - Exceptions and History Tracking + +- [x] T1 Phase 1: Foundation Layer `[component: evolve-foundation]` + + - [x] T1.1 Prime Context + - [x] T1.1.1 Read design for EvolutionError requirements + - [x] T1.1.2 Read design for EvolutionHistory requirements + - [x] T1.1.3 Review existing explore exceptions pattern `[ref: src/wishful/explore/exceptions.py]` + + - [x] T1.2 Write Tests `[activity: test-execution]` + - [x] T1.2.1 Create `tests/test_evolve.py` with `TestEvolutionError` class + - [x] T1.2.2 Add `TestEvolveMetadata` tests + - [x] T1.2.3 Add `TestEvolveHistory` tests + + - [x] T1.3 Implement `[activity: domain-modeling]` + - [x] T1.3.1 Create `src/wishful/evolve/exceptions.py` with EvolutionError + - [x] T1.3.2 Create `src/wishful/evolve/history.py` with VariantRecord, GenerationRecord, EvolutionHistory + - [x] T1.3.3 Implement `get_context_for_llm(limit)` method - THE KEY ALPHAEVOLVE MECHANISM + + - [x] T1.4 Validate `[activity: run-tests]` + - [x] T1.4.1 Run `uv run pytest tests/test_evolve.py::TestEvolutionError -v` + - [x] T1.4.2 Run `uv run pytest tests/test_evolve.py::TestEvolveMetadata -v` + - [x] T1.4.3 Run `uv run pytest tests/test_evolve.py::TestEvolveHistory -v` + +--- + +### Phase 2: Mutation Module - The AlphaEvolve Core + +- [x] T2 Phase 2: LLM Mutation with History Context `[component: evolve-mutation]` + + - [x] T2.1 Prime Context + - [x] T2.1.1 Read mutation module design + - [x] T2.1.2 Study `_build_evolution_context()` function - formats history for LLM + - [x] T2.1.3 Review existing LLM client usage `[ref: src/wishful/llm/client.py]` + + - [x] T2.2 Write Tests `[activity: test-execution]` + - [x] T2.2.1 Add `TestBuildEvolutionContext` tests (7 tests) + - [x] T2.2.2 Add `TestTruncateSource` tests (3 tests) + - [x] T2.2.3 Add `TestGetFunctionSource` tests (3 tests) + - [x] T2.2.4 Add `TestMutateWithLLM` tests (3 tests) + + - [x] T2.3 Implement `[activity: component-development]` + - [x] T2.3.1 Create `src/wishful/evolve/mutation.py` + - [x] T2.3.2 Implement `mutate_with_llm(source, mutation_prompt, function_name, history)` + - [x] T2.3.3 Implement `_build_evolution_context()` with rich history formatting + - [x] T2.3.4 Implement `_truncate_source()` utility + - [x] T2.3.5 Implement `get_function_source()` utility + + - [x] T2.4 Validate `[activity: run-tests]` + - [x] T2.4.1 Run `uv run pytest tests/test_evolve.py -v` - 33 tests passing + - [x] T2.4.2 Run `uv run mypy src/wishful/evolve/` - No issues + - [x] T2.4.3 Create tryout scripts: 05, 06, 07 + +--- + +### Phase 3: Core Evolver - The Evolution Loop + +- [x] T3 Phase 3: Main evolve() Function `[component: evolve-core]` + + - [x] T3.1 Prime Context + - [x] T3.1.1 Read live evolve and workbench specs + - [x] T3.1.2 Study evolution loop with history context passing + - [x] T3.1.3 Review explore() implementation pattern `[ref: src/wishful/explore/explorer.py]` + + - [x] T3.2 Write Tests `[activity: test-execution]` + - [x] T3.2.1 Add `TestEvolveBasic` tests + - [x] T3.2.2 Add `TestEvolveWithTest` tests + - [x] T3.2.3 Add public import integration tests + + - [x] T3.3 Implement `[activity: component-development]` + - [x] T3.3.1 Create `src/wishful/evolve/evolver.py` + - [x] T3.3.2 Implement `evolve()` with `keep_history` and `history_limit` + - [x] T3.3.3 Implement evolution loop with history context passing to mutation + - [x] T3.3.4 Implement `_compile_function()` helper + + - [x] T3.4 Validate `[activity: run-tests]` + - [x] T3.4.1 Run `uv run pytest tests/test_evolve.py::TestEvolveBasic -q` + - [x] T3.4.2 Run `uv run pytest tests/test_evolve.py::TestEvolveWithTest -q` + - [x] T3.4.3 Run `uv run pytest tests/test_evolve.py -q` + +--- + +### Phase 4: Public API and Integration + +- [x] T4 Phase 4: Public API Integration `[component: evolve-api]` + + - [x] T4.1 Prime Context + - [x] T4.1.1 Read public API requirements from this plan + - [x] T4.1.2 Read integration requirements from this plan + - [x] T4.1.3 Review existing wishful `__init__.py` exports `[ref: src/wishful/__init__.py]` + + - [x] T4.2 Write Tests `[activity: test-execution]` + - [x] T4.2.1 Add `TestEvolveIntegration` tests + - [x] T4.2.2 Add import tests for public API + + - [x] T4.3 Implement `[activity: component-development]` + - [x] T4.3.1 Create `src/wishful/evolve/__init__.py` with exports + - [x] T4.3.2 Update `src/wishful/__init__.py` to export `evolve` and `EvolutionError` + + - [x] T4.4 Validate `[activity: run-tests]` + - [x] T4.4.1 Run `uv run pytest tests/test_evolve.py::TestEvolveIntegration -q` + - [x] T4.4.2 Test import: `from wishful import evolve, EvolutionError` + +--- + +### Phase 5: Example and Documentation + +- [x] T5 Phase 5: Example and Final Polish `[component: evolve-docs]` + + - [x] T5.1 Prime Context + - [x] T5.1.1 Read existing example patterns + - [x] T5.1.2 Review existing examples pattern `[ref: examples/]` + + - [x] T5.2 Create Example `[activity: component-development]` + - [x] T5.2.1 Create `examples/14_evolve.py` + - [x] T5.2.2 Test example with fake LLM: `WISHFUL_FAKE_LLM=1 uv run python examples/14_evolve.py` + + - [x] T5.3 Validate `[activity: run-tests]` + - [x] T5.3.1 Run full test suite: `uv run pytest tests/test_evolve.py -q` + - [x] T5.3.2 Check coverage: `uv run pytest tests/test_evolve.py --cov=src/wishful/evolve --cov-report=term-missing` + - [x] T5.3.3 Type check: `uv run mypy src/wishful/evolve/` + +--- + +### Phase 6: Integration & End-to-End Validation + +- [x] T6 Final Validation + + - [x] T6.1 All unit tests passing: `uv run pytest tests/test_evolve.py -q` + - [x] T6.2 Integration with existing wishful features verified + - [x] T6.3 Example runs successfully with fake LLM + - [x] T6.4 Test coverage β‰₯90% for new code + - [x] T6.5 Type hints complete, mypy passes + - [x] T6.6 Docstrings complete for all public APIs + - [x] T6.7 Full test suite still passes: `uv run pytest --cov=wishful tests/` + - [x] T6.8 `from wishful import evolve, EvolutionError` works + - [x] T6.9 Definition of Done checklist verified against this plan + +--- + +## Definition of Done + +From the design specification: + +- [x] All tests in `test_evolve.py` pass (25+ tests) +- [x] Coverage for new code is β‰₯90% +- [x] `wishful.evolve` is importable from main package +- [x] History context is properly passed to LLM (the AlphaEvolve innovation) +- [x] Works with `WISHFUL_FAKE_LLM=1` +- [x] Documentation strings are complete +- [x] Example file demonstrates basic usage +- [x] Type hints complete, `mypy` passes + +--- + +## Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| LLM not using history context | Rich prompt formatting, clear instructions in `_build_evolution_context()` | +| History too long for context | `history_limit` parameter, truncated source in `_truncate_source()` | +| Infinite loops in fitness | `timeout_per_variant` parameter | +| Memory from many generations | Only keep top N variants via `history_limit` | +| Flaky tests | All tests use monkeypatched mutations (deterministic) | + +--- + +## References + +- [AlphaEvolve Blog Post](https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/) β€” Google DeepMind +- [AlphaEvolve Paper](https://arxiv.org/abs/2506.13131) β€” arXiv +- [OpenEvolve](https://huggingface.co/blog/codelion/openevolve) β€” Open source implementation +- Current design direction: `docs/specs/003-wishful-code-search-workbench/concept-plan.md` +- Context follow-up: `docs/specs/002-wishful-context/implementation-plan.md` diff --git a/docs/specs/002-wishful-context/implementation-plan.md b/docs/specs/002-wishful-context/implementation-plan.md new file mode 100644 index 0000000..2ff2acf --- /dev/null +++ b/docs/specs/002-wishful-context/implementation-plan.md @@ -0,0 +1,189 @@ +# Implementation Plan: wishful.context + +## Validation Checklist + +- [x] All specification file paths are correct and exist +- [x] Context priming section is complete +- [x] All implementation phases are defined +- [x] Each phase follows TDD: Prime β†’ Test β†’ Implement β†’ Validate +- [x] Dependencies between phases are clear +- [x] Activity hints provided for specialist selection +- [x] Every phase references relevant SDD sections +- [x] Project commands match actual project setup +- [x] A developer could follow this plan independently + +--- + +## Context Priming + +*GATE: Read all files before starting implementation.* + +**Specification**: +- `docs/specs/002-wishful-context/product-requirements.md` - PRD +- `docs/specs/002-wishful-context/solution-design.md` - SDD + +**Reference Implementation**: +- `src/wishful/types/registry.py` - Pattern to mirror exactly +- `src/wishful/types/__init__.py` - Export pattern + +**Key Design Decisions**: +1. Mirror `@wishful.type` pattern exactly +2. Use `for_=` parameter name +3. Global registry only (no scopes) +4. List targets register for each with priority order + +**Commands**: +```bash +uv run pytest tests/test_context.py -v # Context tests +uv run pytest tests/test_evolve.py -v # Verify evolve still works +uv run mypy src/wishful/context/ # Type check +``` + +--- + +## Implementation Phases + +### Phase 1: Context Registry Foundation + +- [ ] T1 Phase 1: Registry and Decorator `[component: context]` + + - [ ] T1.1 Prime Context + - [ ] T1.1.1 Read `src/wishful/types/registry.py` for pattern `[ref: SDD; lines: 22-35]` + - [ ] T1.1.2 Read SDD Data Models section `[ref: SDD; lines: 95-114]` + + - [ ] T1.2 Write Tests `[activity: test-execution]` + - [ ] T1.2.1 Create `tests/test_context.py` with `TestContextRegistry` class + - [ ] T1.2.2 Test: `test_register_with_function_reference` + - [ ] T1.2.3 Test: `test_register_with_string_path` + - [ ] T1.2.4 Test: `test_register_with_list_targets` + - [ ] T1.2.5 Test: `test_get_context_returns_sources` + - [ ] T1.2.6 Test: `test_get_context_empty_for_unknown` + - [ ] T1.2.7 Test: `test_clear_registry` + - [ ] T1.2.8 Test: `test_multiple_contexts_same_target` + - [ ] T1.2.9 Test: `test_docstring_extraction` + + - [ ] T1.3 Implement `[activity: component-development]` + - [ ] T1.3.1 Create `src/wishful/context/__init__.py` with exports + - [ ] T1.3.2 Create `src/wishful/context/registry.py` with: + - [ ] `ContextEntry` dataclass + - [ ] `ContextRegistry` class + - [ ] `_registry` global instance + - [ ] `context(for_=)` decorator + - [ ] `get_context_for(target)` function + - [ ] `clear_context_registry()` function + + - [ ] T1.4 Validate `[activity: run-tests]` + - [ ] T1.4.1 Run `uv run pytest tests/test_context.py -v` + - [ ] T1.4.2 Run `uv run mypy src/wishful/context/` + - [ ] T1.4.3 Verify import: `python -c "from wishful.context import context, get_context_for"` + +--- + +### Phase 2: Public API Export + +- [ ] T2 Phase 2: Wishful Package Integration `[component: wishful]` + + - [ ] T2.1 Prime Context + - [ ] T2.1.1 Read `src/wishful/__init__.py` for export pattern + + - [ ] T2.2 Write Tests `[activity: test-execution]` + - [ ] T2.2.1 Test: `test_context_importable_from_wishful` + - [ ] T2.2.2 Test: `test_get_context_for_importable_from_wishful` + + - [ ] T2.3 Implement `[activity: component-development]` + - [ ] T2.3.1 Modify `src/wishful/__init__.py` to export `context`, `get_context_for` + + - [ ] T2.4 Validate `[activity: run-tests]` + - [ ] T2.4.1 Run `python -c "from wishful import context, get_context_for; print('OK')"` + - [ ] T2.4.2 Run full test suite: `uv run pytest tests/ -v` + +--- + +### Phase 3: Evolve Integration + +- [ ] T3 Phase 3: Integration with evolve module `[component: evolve]` + + - [ ] T3.1 Prime Context + - [ ] T3.1.1 Read `src/wishful/evolve/mutation.py` current implementation + - [ ] T3.1.2 Read SDD Integration section `[ref: SDD; lines: 192-227]` + + - [ ] T3.2 Write Tests `[activity: test-execution]` + - [ ] T3.2.1 Test: `test_build_evolution_context_includes_registered_context` + - [ ] T3.2.2 Test: `test_build_evolution_context_no_context_registered` + - [ ] T3.2.3 Test: `test_context_docstring_included_in_prompt` + + - [ ] T3.3 Implement `[activity: component-development]` + - [ ] T3.3.1 Add `target_function: Callable | None = None` param to `_build_evolution_context()` + - [ ] T3.3.2 Add context lookup and formatting logic + - [ ] T3.3.3 Update `mutate_with_llm()` to pass target function + + - [ ] T3.4 Validate `[activity: run-tests]` + - [ ] T3.4.1 Run `uv run pytest tests/test_evolve.py -v` + - [ ] T3.4.2 Run `uv run pytest tests/test_context.py -v` + - [ ] T3.4.3 Run full suite: `uv run pytest tests/ -v` + +--- + +### Phase 4: Documentation & Spec Updates + +- [ ] T4 Phase 4: Update 001-wishful-evolve Documentation `[component: docs]` + + - [ ] T4.1 Update Evolve Implementation Plan + - [ ] T4.1.1 Add note about `@wishful.context` integration in `docs/specs/001-wishful-evolve/implementation-plan.md` + - [ ] T4.1.2 Update Phase 3 (Core Evolver) to pass `target_function` to mutation + + - [ ] T4.2 Update Evolve Design Docs + - [ ] T4.2.1 Add context integration section to `docs/specs/001-wishful-evolve/implementation-plan.md` + - [ ] T4.2.2 Add context follow-up notes to `docs/specs/003-wishful-code-search-workbench/concept-plan.md` + + - [ ] T4.3 Update CLAUDE.md + - [ ] T4.3.1 Add `wishful.context` to architecture section + - [ ] T4.3.2 Update session history + + - [ ] T4.4 Create Tryout Scripts + - [ ] T4.4.1 Create `examples/15_context_basic.py` + - [ ] T4.4.2 Create `examples/16_context_with_evolve.py` + - [ ] T4.4.3 Document context examples in `README.md` + +--- + +### Phase 5: Final Validation + +- [ ] T5 Integration & End-to-End Validation + + - [ ] T5.1 All unit tests passing: `uv run pytest tests/test_context.py -v` + - [ ] T5.2 All evolve tests passing: `uv run pytest tests/test_evolve.py -v` + - [ ] T5.3 Full test suite passing: `uv run pytest tests/ -v` + - [ ] T5.4 Type checking passes: `uv run mypy src/wishful/context/ src/wishful/evolve/` + - [ ] T5.5 Import verification: `from wishful import context, get_context_for` + - [ ] T5.6 Test coverage β‰₯90% for new code + - [ ] T5.7 Tryout scripts run successfully + - [ ] T5.8 PRD acceptance criteria verified: + - [ ] `@wishful.context(for_=target)` works + - [ ] Supports function references + - [ ] Supports string paths + - [ ] Supports list of targets + - [ ] `get_context_for()` returns source + docstring + - [ ] Integrates with `_build_evolution_context()` + +--- + +## Definition of Done + +- [ ] All tests pass (β‰₯15 new tests) +- [ ] Test coverage β‰₯90% for `src/wishful/context/` +- [ ] `wishful.context` importable from main package +- [ ] Works with evolve's `_build_evolution_context()` +- [ ] Documentation updated (CLAUDE.md, evolve docs) +- [ ] Tryout scripts demonstrate usage +- [ ] Type hints complete, mypy passes + +--- + +## Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| Circular import with evolve | Lazy import in `_build_evolution_context()` | +| Source extraction fails | Fallback to `repr()` with warning | +| Target resolution ambiguous | Clear error messages, documentation | diff --git a/docs/specs/002-wishful-context/product-requirements.md b/docs/specs/002-wishful-context/product-requirements.md new file mode 100644 index 0000000..e9b46bc --- /dev/null +++ b/docs/specs/002-wishful-context/product-requirements.md @@ -0,0 +1,153 @@ +# Product Requirements Document: wishful.context + +## Validation Checklist + +- [x] All required sections are complete +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Problem statement is specific and measurable +- [x] Problem is validated by evidence (not assumptions) +- [x] Context β†’ Problem β†’ Solution flow makes sense +- [x] Every feature has testable acceptance criteria +- [x] No technical implementation details included + +--- + +## Product Overview + +### Vision +Enable developers to declaratively attach contextual information (fitness functions, constraints, examples, hints) to wishful-generated functions, making LLM generation smarter and more targeted. + +### Problem Statement +When wishful generates or evolves code, the LLM lacks crucial context: +- **evolve()**: LLM sees fitness *scores* but not *what's being measured* +- **explore()**: LLM generates variants without knowing constraints or goals +- **static**: Generation happens without domain-specific hints or examples + +Without this context, the LLM is essentially guessing what "good" means. + +**Evidence**: During evolve() Phase 2 implementation, we discovered that `_build_evolution_context()` passes history with scores (90, 55, 25) but never explains WHY those scores differ. The LLM must reverse-engineer the fitness function from code patterns. + +### Value Proposition +`@wishful.context` provides a declarative, composable way to give the LLM exactly the context it needs - making generation smarter without changing the core wishful API. + +--- + +## Feature Requirements + +### Must Have Features + +#### Feature 1: Context Decorator +- **User Story:** As a developer, I want to attach context to a wishful function so that the LLM understands what I'm optimizing for. +- **Acceptance Criteria:** + - [ ] `@wishful.context(for_=target)` decorator registers context + - [ ] Supports function references: `@wishful.context(for_=sort)` + - [ ] Supports string paths: `@wishful.context(for_="module.func")` + - [ ] Works with functions, classes, and any callable + +#### Feature 2: Multiple Targets +- **User Story:** As a developer, I want one context to serve multiple functions so that I don't repeat myself. +- **Acceptance Criteria:** + - [ ] `@wishful.context(for_=[sort, search])` registers for both targets + - [ ] Order in list determines priority (first = highest) + +#### Feature 3: Context Registry +- **User Story:** As wishful internals, I need to look up all context for a given target. +- **Acceptance Criteria:** + - [ ] `get_context_for(target)` returns list of context sources + - [ ] Returns source code of decorated items + - [ ] Respects priority ordering + - [ ] Returns empty list if no context registered + +#### Feature 4: Integration Point +- **User Story:** As wishful.evolve (and future features), I need to include context in LLM prompts. +- **Acceptance Criteria:** + - [ ] Context can be retrieved and formatted for LLM consumption + - [ ] Works with `_build_evolution_context()` in evolve module + - [ ] Pattern matches existing `@wishful.type` infrastructure + +### Should Have Features + +#### Feature 5: Docstring Extraction +- **User Story:** As a developer, I want my context function's docstring to be included as human-readable explanation. +- **Acceptance Criteria:** + - [ ] Docstrings are extracted and included in context + - [ ] Source code AND docstring both available + +### Won't Have (This Phase) + +- **Scoped registries** - Start global only, scoped contexts for later +- **Context validation** - No runtime checking that context is "correct" +- **Auto-discovery** - No scanning modules for context, must be explicit +- **Priority override** - No way to re-order after registration + +--- + +## Detailed Feature Specification + +### Feature: Context Decorator + Registry + +**User Flow:** +```python +# 1. Developer decorates context providers +@wishful.context(for_=sort) +def sorting_benchmark(fn) -> float: + """Measures speed on large reversed arrays. Higher = faster.""" + ... + +@wishful.context(for_=sort) +class SortingConstraints: + """Edge cases that must be handled.""" + EMPTY = [] + DUPLICATES = [1, 1, 1] + +# 2. Wishful internally looks up context +contexts = wishful.context.get_context_for(sort) +# Returns: [source of sorting_benchmark, source of SortingConstraints] + +# 3. Context included in LLM prompt during generation/evolution +``` + +**Business Rules:** +- Rule 1: Context is registered at decoration time (import time) +- Rule 2: Multiple contexts for same target are allowed +- Rule 3: Order of registration = priority order +- Rule 4: String targets resolved at lookup time, not registration + +**Edge Cases:** +- Target doesn't exist yet (string path) β†’ Register anyway, resolve later +- No context for target β†’ Return empty list (not an error) +- Same context decorated twice β†’ Register twice (user's choice) + +--- + +## Success Criteria + +| Criterion | Measurement | +|-----------|-------------| +| Works with evolve() | Context appears in LLM prompt during evolution | +| Pattern consistency | API feels like `@wishful.type` | +| No breaking changes | Existing wishful code unaffected | +| Test coverage | β‰₯90% for new code | + +--- + +## Constraints and Assumptions + +### Constraints +- Must follow existing wishful patterns (`@wishful.type`, `@wishful.static`) +- Must not require changes to existing user code +- Must work with Python 3.12+ + +### Assumptions +- Global registry is sufficient for initial release +- Users will provide context as functions or classes (not arbitrary objects) +- Source code extraction via `inspect.getsource()` or `__wishful_source__` is reliable + +--- + +## Open Questions + +- [x] Naming: `for_=` vs `target=` β†’ **Decision: `for_=`** (most natural English) +- [x] Multiple targets: Yes, via list +- [x] Priority: Order in list +- [x] Scope: Global only for now diff --git a/docs/specs/002-wishful-context/solution-design.md b/docs/specs/002-wishful-context/solution-design.md new file mode 100644 index 0000000..1d43ae7 --- /dev/null +++ b/docs/specs/002-wishful-context/solution-design.md @@ -0,0 +1,273 @@ +# Solution Design Document: wishful.context + +## Validation Checklist + +- [x] All required sections are complete +- [x] Architecture pattern clearly stated +- [x] Every component has directory mapping +- [x] All architecture decisions confirmed +- [x] A developer could implement from this design + +--- + +## Constraints + +- **CON-1**: Must follow existing wishful patterns (`@wishful.type` in `src/wishful/types/`) +- **CON-2**: Python 3.12+, no external dependencies beyond existing +- **CON-3**: Must not break existing wishful API + +## Implementation Context + +### Required Context Sources + +```yaml +- file: src/wishful/types/registry.py + relevance: CRITICAL + why: "Pattern to mirror exactly - registry class, decorator, global instance" + +- file: src/wishful/types/__init__.py + relevance: HIGH + why: "Export pattern to follow" + +- file: src/wishful/evolve/mutation.py + relevance: HIGH + why: "Integration point - _build_evolution_context() will use context registry" +``` + +### Implementation Boundaries + +- **Must Preserve**: Existing `@wishful.type` behavior, `evolve()` API +- **Can Modify**: `_build_evolution_context()` to include registered context +- **Must Not Touch**: Core wishful generation logic + +### Project Commands + +```bash +# Testing +uv run pytest tests/test_context.py -v # Context tests +uv run pytest tests/test_evolve.py -v # Verify evolve still works +uv run pytest tests/ -v # Full suite + +# Type checking +uv run mypy src/wishful/context/ + +# Validation +python -c "from wishful import context; print('OK')" +``` + +--- + +## Solution Strategy + +- **Architecture Pattern**: Mirror `@wishful.type` exactly +- **Integration Approach**: Add optional context lookup to LLM prompt building +- **Justification**: Proven pattern, predictable for users, minimal new concepts + +--- + +## Building Block View + +### Components + +```mermaid +graph LR + User[Developer] -->|decorates| Context[@wishful.context] + Context -->|registers| Registry[ContextRegistry] + Evolve[evolve/mutation.py] -->|looks up| Registry + Registry -->|returns| Sources[Context Sources] + Sources -->|included in| LLMPrompt[LLM Prompt] +``` + +### Directory Map + +``` +src/wishful/ +β”œβ”€β”€ context/ # NEW MODULE +β”‚ β”œβ”€β”€ __init__.py # NEW: Public exports +β”‚ └── registry.py # NEW: ContextRegistry + decorator +β”œβ”€β”€ __init__.py # MODIFY: Add context exports +└── evolve/ + └── mutation.py # MODIFY: Integrate context lookup +tests/ +└── test_context.py # NEW: Unit tests +``` + +### Data Models + +```python +@dataclass +class ContextEntry: + """Single context provider entry.""" + source: str # Source code of provider + docstring: str | None # Extracted docstring + provider: Any # Reference to original (for debugging) + +class ContextRegistry: + """Global registry for context providers.""" + _contexts: dict[str, list[ContextEntry]] # target_key -> entries + + def register(self, provider: Any, for_: Any) -> None: ... + def get_for(self, target: Any) -> list[ContextEntry]: ... + def clear(self) -> None: ... + def _resolve_key(self, target: Any) -> str: ... + def _extract_source(self, provider: Any) -> str: ... +``` + +### API Surface + +```python +# Public API (from wishful.context) +def context(*, for_: Any) -> Callable: + """Decorator to register context for a target.""" + +def get_context_for(target: Any) -> list[dict]: + """Get all context entries for a target.""" + +def clear_context_registry() -> None: + """Clear registry (for testing).""" +``` + +--- + +## Runtime View + +### Registration Flow + +``` +1. Module imports β†’ @wishful.context(for_=sort) executes +2. Decorator calls _registry.register(provider, for_=sort) +3. Registry resolves target to key: "mymodule.sort" +4. Registry extracts source + docstring from provider +5. Registry stores ContextEntry in _contexts["mymodule.sort"] +``` + +### Lookup Flow + +``` +1. evolve() calls _build_evolution_context(target_function=sort) +2. _build_evolution_context calls get_context_for(sort) +3. Registry resolves sort β†’ "mymodule.sort" +4. Registry returns list of ContextEntry dicts +5. Context included in LLM prompt under "ADDITIONAL CONTEXT" +``` + +### Target Resolution Logic + +```python +def _resolve_key(self, target: Any) -> str: + if isinstance(target, str): + return target # Already a string path + elif callable(target): + return f"{target.__module__}.{target.__qualname__}" + else: + raise TypeError(f"Target must be callable or string, got {type(target)}") +``` + +--- + +## Architecture Decisions + +- [x] **ADR-1**: Mirror @wishful.type pattern exactly + - Rationale: Proven, consistent, less cognitive load for users + - Trade-offs: Less flexibility, but simplicity wins + - User confirmed: βœ… (from earlier discussion) + +- [x] **ADR-2**: Use `for_=` parameter name + - Rationale: Most natural English ("context for X") + - Trade-offs: Underscore needed due to reserved word + - User confirmed: βœ… (from earlier discussion) + +- [x] **ADR-3**: Global registry only (no scopes) + - Rationale: Keep simple, add scopes later if needed + - Trade-offs: Can't have module-local contexts + - User confirmed: βœ… (from earlier discussion) + +- [x] **ADR-4**: List targets = register for each with priority order + - Rationale: DRY for shared context, priority by position + - Trade-offs: Implicit priority might surprise some users + - User confirmed: βœ… (from earlier discussion) + +--- + +## Integration with evolve() + +### Changes to mutation.py + +```python +# In _build_evolution_context(), add after history section: + +def _build_evolution_context( + source: str, + mutation_prompt: str, + function_name: str, + history: List[dict], + target_function: Callable | None = None, # NEW parameter +) -> str: + ... + # NEW: Include registered context + if target_function: + from wishful.context import get_context_for + contexts = get_context_for(target_function) + if contexts: + parts.extend([ + "-" * 60, + "ADDITIONAL CONTEXT (provided by developer):", + "-" * 60, + "", + ]) + for ctx in contexts: + if ctx.get("docstring"): + parts.append(f"# {ctx['docstring']}") + parts.extend([ + "```python", + ctx["source"], + "```", + "", + ]) +``` + +--- + +## Test Specifications + +### Critical Test Scenarios + +```gherkin +Scenario: Register context with function reference + Given a function `sort` exists + When developer decorates a fitness function with @wishful.context(for_=sort) + Then get_context_for(sort) returns the fitness function's source + +Scenario: Register context with string path + Given no function exists yet + When developer decorates with @wishful.context(for_="future.module.func") + Then context is registered for that string key + +Scenario: Multiple contexts for same target + Given two contexts registered for `sort` + When get_context_for(sort) is called + Then both contexts returned in registration order + +Scenario: No context registered + When get_context_for(unknown_func) is called + Then empty list returned (not an error) +``` + +### Test Coverage Requirements + +- Registry: register, get_for, clear, key resolution +- Decorator: with function, with string, with list +- Source extraction: functions, classes, with/without docstrings +- Integration: context appears in evolve prompt + +--- + +## Files to Create/Modify + +| File | Action | Description | +|------|--------|-------------| +| `src/wishful/context/__init__.py` | CREATE | Public exports | +| `src/wishful/context/registry.py` | CREATE | ContextRegistry + decorator | +| `src/wishful/__init__.py` | MODIFY | Add `context, get_context_for` | +| `src/wishful/evolve/mutation.py` | MODIFY | Add `target_function` param | +| `tests/test_context.py` | CREATE | Unit tests | diff --git a/docs/specs/003-wishful-code-search-workbench/concept-plan.md b/docs/specs/003-wishful-code-search-workbench/concept-plan.md new file mode 100644 index 0000000..a06ba4f --- /dev/null +++ b/docs/specs/003-wishful-code-search-workbench/concept-plan.md @@ -0,0 +1,504 @@ +# Wishful Code Search Workbench: Concept Plan + +## Status + +This document captures the 2026-06-03 synthesis on why Wishful becomes serious +when it stops competing with one-shot code generation and starts owning bounded +code search. + +Read this after: + +- `AGENTS.md` +- `docs/specs/001-wishful-evolve/implementation-plan.md` +- `docs/specs/002-wishful-context/product-requirements.md` +- `docs/specs/002-wishful-context/solution-design.md` + +Current branch reality: + +- `wishful.evolve` has history records and LLM mutation context. +- The public `evolve()` loop is implemented and exported. +- `wishful.context` is specified but not implemented. +- The CLI/dashboard wireframe exists, but evidence/casefile mechanics do not. + +## Core Thesis + +Wishful's serious wedge is not "generate the missing helper function once." +Modern coding agents can often do that directly. + +Wishful becomes valuable when the task has this shape: + +```text +target artifact + mutation space + fitness function + budget + accept/rollback +``` + +That is code search. The developer or coding agent identifies a bounded search +problem, then Wishful runs the repeatable loop, preserves the evidence, and +turns the winner into reusable local code. + +Short positioning: + +```text +When one-shot codegen plateaus, Wishful keeps going. +``` + +Agent-facing positioning: + +```text +Codie finds the search problem. Wishful runs the search. +``` + +## Why Not Just Ask A Coding Agent? + +For a one-off optimization, a coding agent can loop manually: + +1. Inspect slow function. +2. Edit candidate implementation. +3. Run benchmark. +4. Read failures. +5. Edit again. +6. Repeat until better or tired. + +That works, but the loop state lives in chat context and reviewer patience. It +is hard to reproduce, hard to batch, hard to inspect later, and easy to lose +across sessions. + +Wishful should make the loop infrastructure: + +- persistent lineage: every variant, failure, score, and winner is retained +- rerunnable fitness: tests, benchmarks, or evals define "better" +- reusable artifact: the winner becomes the imported function/module +- batch mode: many bounded searches can run under budgets +- review surface: a casefile explains why the winner was allowed to count +- future improvement: the next run starts from known attempts, not amnesia + +The analogy is `pytest` versus "Codie, check that this works." Agents can do +manual checks, but durable software work still wants the repeatable harness. + +## KG Synthesis Findings + +The project-level synthesis from the knowledge graph: + +- Acceptance/reviewability: final green checks collapse process quality. Wishful + should preserve trajectory evidence at function scale, not only the winning + source. +- Surface-first development: the developer-facing surface is the contract. + Wishful's contract is the import path, signature, types, context, and fitness + function. +- Deterministic/probabilistic split: LLM mutation can be probabilistic, but + critical-path acceptance must depend on deterministic or reproducible sensors + with declared scope. +- Creative leverage: generate candidates at the level where selection has the + most downstream control. For Wishful, that level is the importable function or + small module boundary. +- Proofroom relation: Proofroom asks whether agent work may count at work-unit + or PR scale. Wishful can ask the same question at generated-function scale and + feed the answer back into the next generation. + +The product primitive this implies: + +```text +function with lineage +``` + +A Wishful function should eventually carry: + +- import address +- current source +- declared context +- tests, benchmarks, or evals +- variant history +- failures and error summaries +- winning rationale +- evidence scope +- accept/rollback state + +## Demo Selection Filter + +Bad demos: + +- `generate slugify()` +- `write a CSV parser` +- `make a password validator` +- anything a coding agent can one-shot cleanly + +Good demos satisfy all of this: + +- one-shot is plausible but not enough +- 20-100 attempts can become meaningfully better +- fitness is cheap enough to run many times +- failures teach the next attempt +- the winner becomes reusable code +- the evidence is interesting to inspect + +Kill a demo if: + +- the fitness signal is mostly vibes +- the search space is repo-wide rather than function/module-sized +- each evaluation is too expensive for iteration +- one passing implementation is already enough +- the best result cannot be reused as a stable artifact + +## Demo Candidates + +### 1. NanoGPT/Nanochat Optimizer Or Scheduler Speedrun + +Prompt: + +```text +Find a task-specialized optimizer or learning-rate schedule that beats tuned +AdamW on this dataset/training setup under fixed budget, seeds, and holdout +checks. +``` + +Why it is strong: + +- One-shot optimizer invention is usually nonsense. +- Repeated measured variants can find local wins. +- The audience immediately understands "beat AdamW on this bounded setup." +- It connects to AutoResearch-style overnight experimentation. + +Risks: + +- Easy to overfit a tiny benchmark. +- Expensive if the fitness loop is too large. +- Needs tuned AdamW baseline, multiple seeds, and holdout checks or it becomes + benchmark glitter. + +Use this as the sexy flagship after the core harness works. + +### 2. Parser Gauntlet + +Prompt: + +```text +Given 200 ugly real vendor/export files, evolve a parser that maximizes fixture +pass rate while staying readable. +``` + +Why it is strong: + +- Normal coding agents can write parser v1. +- They struggle with the long tail of ugly cases. +- Every failed fixture becomes useful evolutionary pressure. +- Fitness is cheap and deterministic. +- The result is boringly useful to real developers. + +This is probably the best first proof demo. + +### 3. Hot Path Forge + +Prompt: + +```text +Make this normalization/scoring/matching function 5x faster without changing +outputs. +``` + +Fitness: + +- property tests for correctness +- benchmark for speed +- optional readability or complexity guard + +Why it is strong: + +- Shows the difference between "optimize once" and real search. +- Easy to understand in a README or CLI video. +- Can run quickly with fake or local deterministic examples. + +### 4. Extractor Or Prompt Arena + +Prompt: + +```text +Evolve a prompt or small extractor against a labeled eval set. +``` + +Fitness: + +- precision +- recall +- F1 +- latency or cost guard + +Why it is strong: + +- Shows Wishful beyond pure Python algorithms. +- Still has a measurable artifact and stable eval. + +Risk: + +- If an LLM judge is the primary fitness function, the demo becomes weaker. + Prefer labeled evals. + +### 5. SQL/Rewriter Duel + +Prompt: + +```text +Generate semantically equivalent query variants and score by correctness plus +runtime on a sample database. +``` + +Why it is strong: + +- Hard to one-shot well. +- Easy to measure. +- Useful in real production-adjacent work. + +Risk: + +- Needs a safe database fixture and query equivalence checks. + +### 6. UI Microcomponent With Visual Fitness + +Prompt: + +```text +Evolve a small component until Playwright, accessibility, and screenshot checks +pass. +``` + +Why it is interesting: + +- First credible step toward another modality. +- Fitness can combine deterministic checks and visual diffs. + +Risk: + +- More fragile than code-only demos. +- Should wait until the code-search harness is solid. + +## Recommended Demo Order + +1. Parser Gauntlet: deterministic, useful, cheap, hard for one-shot agents. +2. Hot Path Forge: obvious value from repeated measured optimization. +3. NanoGPT/Nanochat Optimizer: public "holy shit" demo once overfitting guards + and casefiles are strong. +4. Extractor/Prompt Arena: shows the artifact model can include prompts. +5. SQL/Rewriter Duel: practical but needs careful fixtures. +6. UI Microcomponent: later, after evidence mechanics can handle non-code + artifacts cleanly. + +## Proposed User Surface + +Python surface: + +```python +import wishful + +@wishful.context(for_="optimizers.WishfulAdam") +def optimizer_context(): + """Beat tuned AdamW on a fixed nanochat run under equal wall-clock budget.""" + return { + "baseline": "AdamW", + "metrics": ["val_loss", "time_to_threshold", "stability"], + "seeds": [1, 2, 3, 4, 5], + "holdout": "heldout_training_config", + } + +result = wishful.evolve( + "optimizers.WishfulAdam", + fitness=score_optimizer, + generations=30, + variants=8, + budget="overnight", + evidence=True, +) + +result.accept() +``` + +CLI surface: + +```bash +uv run wishful evolve optimizers.WishfulAdam \ + --fitness benchmarks/optimizer.py:score_optimizer \ + --generations 30 \ + --variants 8 \ + --casefile + +uv run wishful inspect optimizers.WishfulAdam +uv run wishful diff optimizers.WishfulAdam --run latest +uv run wishful accept optimizers.WishfulAdam --run latest +uv run wishful rollback optimizers.WishfulAdam --to previous +``` + +The CLI should not be dashboard-first. It should expose the evidence model first. + +## Evidence Casefile + +Minimal casefile directory: + +```text +.wishful/evidence/ + optimizers.WishfulAdam/ + 2026-06-03T21-42-10Z/ + casefile.json + casefile.md + config.json + winner.py + variants/ + gen-001-var-001.py + gen-001-var-002.py + scores.csv + failures.md +``` + +`casefile.json` is for tools. `casefile.md` is for humans. + +The casefile should answer: + +- What target was evolved? +- What context was attached? +- What fitness function was used? +- What budget was spent? +- Which variants were tried? +- Which variants failed and why? +- Which variant won? +- What evidence supports the winner? +- What known blind spots remain? +- Was the winner accepted into the cache or only proposed? + +Recommended default: + +- `evolve(..., evidence=True)` writes a casefile every run. +- Winner is not final until accepted when the run is high-risk. +- Low-risk compatibility with `explore()` can still auto-cache the winner. + +## Implementation Path + +### Phase 0: Finish Existing Branch Spine + +Do not start with dashboard work. + +Complete: + +- `docs/specs/001-wishful-evolve/implementation-plan.md` Phase 3: public + `evolve()` loop +- Phase 4: public API exports +- Phase 5: example/docs + +Then complete: + +- `docs/specs/002-wishful-context/implementation-plan.md` +- `@wishful.context` registry and evolve integration + +### Phase 1: Evidence-First Evolve Result + +Add an `EvolutionResult` or equivalent return object that exposes: + +- target +- winner +- history +- best_score +- casefile_path +- accepted flag +- `accept()` +- `rollback()` or companion CLI rollback + +Keep this minimal. The point is to stop losing lineage. + +### Phase 2: Casefile Writer + +Implement a deterministic local writer for: + +- JSON casefile +- Markdown summary +- winner source +- variant sources +- score table +- failure summaries + +This can be local filesystem only. No server, no dashboard. + +### Phase 3: CLI Evidence Surface + +Add commands in this order: + +1. `wishful evolve` +2. `wishful inspect` +3. `wishful diff` +4. `wishful accept` +5. `wishful rollback` + +The commands should make a terminal session compelling before any GUI exists. + +### Phase 4: Demo Pack + +Create examples under `examples/` or a dedicated `demos/` directory: + +- parser gauntlet +- hot path forge +- nanochat optimizer once stable + +Each demo needs: + +- baseline +- fitness function +- expected run command +- expected casefile screenshot/snippet +- explanation of why one-shot agents struggle + +### Phase 5: Batch And Budget Controls + +Only after one-target evolution feels good: + +- per-run budgets +- variant/generation caps +- timeout per variant +- run queue +- overnight mode +- resume interrupted run + +### Phase 6: Dashboard + +The existing Excalidraw dashboard should wait until the CLI evidence model is +real. The dashboard should visualize casefiles, not invent a separate state +model. + +## Acceptance Criteria For The Product Direction + +Wishful has crossed from joke to serious when: + +- a user can define a target and fitness function in under five minutes +- Wishful can run many variants without chat supervision +- the winner is inspectable as normal Python code +- the evidence explains why the winner won +- failures are preserved and reused +- rerunning the same case is reproducible enough to review +- a coding agent would prefer using Wishful for bounded searches instead of + manually looping in chat + +## Open Decisions For The Next Session + +1. Should `evolve()` auto-cache winners by default, or require explicit + `accept()` once evidence exists? +2. Should `fitness` return only a float, or a richer object with metrics and + notes? +3. Should `@wishful.context` allow returning structured data, or only source and + docstring for v1? +4. Should casefiles live under `.wishful/evidence/` or `.wishful/runs/`? +5. Should demos live in `examples/` or `demos/`? +6. Which first flagship demo do we build: parser gauntlet or hot path forge? + +## Restart Checklist + +When resuming this work: + +```bash +cd /home/pyro/projects/private/wishful +git switch feat/001-wishful-evolve +uv sync +WISHFUL_FAKE_LLM=1 uv run pytest tests/test_evolve.py -v +``` + +Then read: + +1. `AGENTS.md` +2. `docs/specs/001-wishful-evolve/implementation-plan.md` +3. `docs/specs/002-wishful-context/implementation-plan.md` +4. this file + +Start by finishing the public `evolve()` loop before touching casefiles, +CLI, demos, or dashboard work. diff --git a/docs/wireframes/wishful-cli-dashboard.excalidraw b/docs/wireframes/wishful-cli-dashboard.excalidraw new file mode 100644 index 0000000..1dbeb9f --- /dev/null +++ b/docs/wireframes/wishful-cli-dashboard.excalidraw @@ -0,0 +1,1572 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "bmad-workflow", + "elements": [ + { + "id": "title", + "type": "text", + "x": 20, + "y": 20, + "width": 600, + "height": 35, + "text": "Wishful CLI Management Dashboard - Wireframes", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "screen1-label", + "type": "text", + "x": 20, + "y": 70, + "width": 200, + "height": 25, + "text": "Screen 1: Main Dashboard", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "screen1-frame", + "type": "rectangle", + "x": 20, + "y": 100, + "width": 700, + "height": 500, + "strokeColor": "#444444", + "backgroundColor": "#1e1e1e", + "fillStyle": "solid", + "strokeWidth": 3, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen1-titlebar", + "type": "rectangle", + "x": 20, + "y": 100, + "width": 700, + "height": 30, + "strokeColor": "#444444", + "backgroundColor": "#2d2d2d", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen1-title-text", + "type": "text", + "x": 40, + "y": 105, + "width": 400, + "height": 20, + "text": "Wishful Manager v1.0 [?] Help [q] Quit", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-stats-box", + "type": "rectangle", + "x": 40, + "y": 145, + "width": 200, + "height": 100, + "strokeColor": "#00ff00", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-stats-title", + "type": "text", + "x": 50, + "y": 150, + "width": 180, + "height": 18, + "text": "[ Cache Stats ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-stats-content", + "type": "text", + "x": 50, + "y": 175, + "width": 180, + "height": 60, + "text": "Modules: 24\nTotal Size: 142 KB\nLast Gen: 2m ago\nModel: gpt-4", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-actions-box", + "type": "rectangle", + "x": 260, + "y": 145, + "width": 440, + "height": 100, + "strokeColor": "#00ffff", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-actions-title", + "type": "text", + "x": 270, + "y": 150, + "width": 180, + "height": 18, + "text": "[ Quick Actions ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-actions-content", + "type": "text", + "x": 270, + "y": 175, + "width": 420, + "height": 60, + "text": "[r] Regenerate Module [c] Clear All Cache [i] Inspect\n[e] Explore Variants [t] Type Registry [l] View Logs\n[s] Settings [h] Help [q] Quit", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-box", + "type": "rectangle", + "x": 40, + "y": 260, + "width": 660, + "height": 280, + "strokeColor": "#666666", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-title", + "type": "text", + "x": 50, + "y": 265, + "width": 200, + "height": 18, + "text": "[ Cached Modules ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ffff55", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-header", + "type": "text", + "x": 50, + "y": 290, + "width": 640, + "height": 18, + "text": " Module Path Status Generated Size Actions", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-divider", + "type": "text", + "x": 50, + "y": 308, + "width": 640, + "height": 18, + "text": " ─────────────────────────────────────────────────────────────────────", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#444444", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row1-bg", + "type": "rectangle", + "x": 45, + "y": 323, + "width": 650, + "height": 20, + "strokeColor": "transparent", + "backgroundColor": "#3a3a5a", + "fillStyle": "solid", + "strokeWidth": 0, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row1", + "type": "text", + "x": 50, + "y": 325, + "width": 640, + "height": 18, + "text": "> wishful.static.text [OK] 5 min ago 2.1KB [v][e][d]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row2", + "type": "text", + "x": 50, + "y": 345, + "width": 640, + "height": 18, + "text": " wishful.static.json [OK] 1 hr ago 1.8KB [v][e][d]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row3", + "type": "text", + "x": 50, + "y": 365, + "width": 640, + "height": 18, + "text": " wishful.static.http [OK] 2 hr ago 3.4KB [v][e][d]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row4", + "type": "text", + "x": 50, + "y": 385, + "width": 640, + "height": 18, + "text": " wishful.static.math [ERR] 3 hr ago 0KB [v][r]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ff5555", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row5", + "type": "text", + "x": 50, + "y": 405, + "width": 640, + "height": 18, + "text": " wishful.static.validation [OK] yesterday 1.2KB [v][e][d]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row6", + "type": "text", + "x": 50, + "y": 425, + "width": 640, + "height": 18, + "text": " wishful.static.formatting [OK] yesterday 0.9KB [v][e][d]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row7", + "type": "text", + "x": 50, + "y": 445, + "width": 640, + "height": 18, + "text": " wishful.dynamic.story [DYN] on-demand N/A [v]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ffff55", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-modules-row8", + "type": "text", + "x": 50, + "y": 465, + "width": 640, + "height": 18, + "text": " wishful.dynamic.creative [DYN] on-demand N/A [v]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ffff55", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen1-scrollbar", + "type": "rectangle", + "x": 685, + "y": 290, + "width": 8, + "height": 240, + "strokeColor": "#333333", + "backgroundColor": "#333333", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null, + "roundness": {"type": 3, "value": 4} + }, + { + "id": "screen1-scrollbar-thumb", + "type": "rectangle", + "x": 686, + "y": 295, + "width": 6, + "height": 60, + "strokeColor": "#666666", + "backgroundColor": "#666666", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null, + "roundness": {"type": 3, "value": 3} + }, + { + "id": "screen1-footer", + "type": "text", + "x": 40, + "y": 555, + "width": 660, + "height": 18, + "text": "[↑↓] Navigate [Enter] View Module [Tab] Switch Panel [/] Search [q] Quit", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen1"], + "boundElements": null + }, + { + "id": "screen2-label", + "type": "text", + "x": 760, + "y": 70, + "width": 250, + "height": 25, + "text": "Screen 2: Module Detail View", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "screen2-frame", + "type": "rectangle", + "x": 760, + "y": 100, + "width": 700, + "height": 500, + "strokeColor": "#444444", + "backgroundColor": "#1e1e1e", + "fillStyle": "solid", + "strokeWidth": 3, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen2-titlebar", + "type": "rectangle", + "x": 760, + "y": 100, + "width": 700, + "height": 30, + "strokeColor": "#444444", + "backgroundColor": "#2d2d2d", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen2-title-text", + "type": "text", + "x": 780, + "y": 105, + "width": 500, + "height": 20, + "text": "Module: wishful.static.text [Esc] Back [?] Help", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-info-box", + "type": "rectangle", + "x": 780, + "y": 145, + "width": 320, + "height": 120, + "strokeColor": "#00ff00", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-info-title", + "type": "text", + "x": 790, + "y": 150, + "width": 200, + "height": 18, + "text": "[ Module Info ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-info-content", + "type": "text", + "x": 790, + "y": 175, + "width": 300, + "height": 85, + "text": "Path: .wishful/text.py\nSize: 2.1 KB (847 bytes)\nGenerated: 2024-01-15 14:32:45\nModel: gpt-4-turbo\nTokens: 342 prompt / 289 completion\nCost: $0.0023", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-actions-box", + "type": "rectangle", + "x": 1120, + "y": 145, + "width": 320, + "height": 120, + "strokeColor": "#00ffff", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-actions-title", + "type": "text", + "x": 1130, + "y": 150, + "width": 200, + "height": 18, + "text": "[ Actions ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-actions-content", + "type": "text", + "x": 1130, + "y": 175, + "width": 300, + "height": 85, + "text": "[r] Regenerate with same context\n[R] Regenerate with new prompt\n[e] Edit source code\n[x] Explore variants (5)\n[d] Delete from cache\n[c] Copy to clipboard", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-box", + "type": "rectangle", + "x": 780, + "y": 280, + "width": 660, + "height": 260, + "strokeColor": "#666666", + "backgroundColor": "#1a1a2e", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-title", + "type": "text", + "x": 790, + "y": 285, + "width": 200, + "height": 18, + "text": "[ Source Code ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ffff55", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line1", + "type": "text", + "x": 790, + "y": 310, + "width": 640, + "height": 18, + "text": " 1 β”‚ import re", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ff79c6", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line2", + "type": "text", + "x": 790, + "y": 328, + "width": 640, + "height": 18, + "text": " 2 β”‚ from typing import List", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ff79c6", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line3", + "type": "text", + "x": 790, + "y": 346, + "width": 640, + "height": 18, + "text": " 3 β”‚", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#666666", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line4", + "type": "text", + "x": 790, + "y": 364, + "width": 640, + "height": 18, + "text": " 4 β”‚ def extract_emails(text: str) -> List[str]:", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#8be9fd", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line5", + "type": "text", + "x": 790, + "y": 382, + "width": 640, + "height": 18, + "text": " 5 β”‚ \"\"\"Extract all email addresses from the given text.\"\"\"", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#6272a4", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line6", + "type": "text", + "x": 790, + "y": 400, + "width": 640, + "height": 18, + "text": " 6 β”‚ pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}'", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#f1fa8c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line7", + "type": "text", + "x": 790, + "y": 418, + "width": 640, + "height": 18, + "text": " 7 β”‚ return re.findall(pattern, text)", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line8", + "type": "text", + "x": 790, + "y": 436, + "width": 640, + "height": 18, + "text": " 8 β”‚", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#666666", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line9", + "type": "text", + "x": 790, + "y": 454, + "width": 640, + "height": 18, + "text": " 9 β”‚ def count_words(text: str) -> int:", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#8be9fd", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-line10", + "type": "text", + "x": 790, + "y": 472, + "width": 640, + "height": 18, + "text": " 10 β”‚ \"\"\"Count the number of words in text.\"\"\"", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#6272a4", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-source-more", + "type": "text", + "x": 790, + "y": 495, + "width": 640, + "height": 18, + "text": " β–Ό [12 more lines - scroll to view]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#666666", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen2-footer", + "type": "text", + "x": 780, + "y": 555, + "width": 660, + "height": 18, + "text": "[↑↓] Scroll Code [PgUp/Dn] Page [y] Yank Line [Esc] Back [q] Quit", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen2"], + "boundElements": null + }, + { + "id": "screen3-label", + "type": "text", + "x": 20, + "y": 640, + "width": 220, + "height": 25, + "text": "Screen 3: Type Registry", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "screen3-frame", + "type": "rectangle", + "x": 20, + "y": 670, + "width": 700, + "height": 440, + "strokeColor": "#444444", + "backgroundColor": "#1e1e1e", + "fillStyle": "solid", + "strokeWidth": 3, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen3-titlebar", + "type": "rectangle", + "x": 20, + "y": 670, + "width": 700, + "height": 30, + "strokeColor": "#444444", + "backgroundColor": "#2d2d2d", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "screen3-title-text", + "type": "text", + "x": 40, + "y": 675, + "width": 500, + "height": 20, + "text": "Type Registry [Esc] Back [?] Help", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-box", + "type": "rectangle", + "x": 40, + "y": 715, + "width": 660, + "height": 180, + "strokeColor": "#666666", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-title", + "type": "text", + "x": 50, + "y": 720, + "width": 200, + "height": 18, + "text": "[ Registered Types ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#ffff55", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-header", + "type": "text", + "x": 50, + "y": 745, + "width": 640, + "height": 18, + "text": " Type Name Kind Fields Used In", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-divider", + "type": "text", + "x": 50, + "y": 763, + "width": 640, + "height": 18, + "text": " ────────────────────────────────────────────────────────────────", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#444444", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row1-bg", + "type": "rectangle", + "x": 45, + "y": 778, + "width": 650, + "height": 20, + "strokeColor": "transparent", + "backgroundColor": "#3a3a5a", + "fillStyle": "solid", + "strokeWidth": 0, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row1", + "type": "text", + "x": 50, + "y": 780, + "width": 640, + "height": 18, + "text": "> User Pydantic 5 3 modules", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row2", + "type": "text", + "x": 50, + "y": 800, + "width": 640, + "height": 18, + "text": " Product Pydantic 8 2 modules", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row3", + "type": "text", + "x": 50, + "y": 820, + "width": 640, + "height": 18, + "text": " OrderItem TypedDict 4 1 module", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row4", + "type": "text", + "x": 50, + "y": 840, + "width": 640, + "height": 18, + "text": " Config dataclass 6 5 modules", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-types-row5", + "type": "text", + "x": 50, + "y": 860, + "width": 640, + "height": 18, + "text": " APIResponse Pydantic 3 4 modules", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-detail-box", + "type": "rectangle", + "x": 40, + "y": 910, + "width": 660, + "height": 140, + "strokeColor": "#00ff00", + "backgroundColor": "#1a1a2e", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-detail-title", + "type": "text", + "x": 50, + "y": 915, + "width": 200, + "height": 18, + "text": "[ Type Definition: User ]", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-detail-code", + "type": "text", + "x": 50, + "y": 940, + "width": 640, + "height": 100, + "text": "class User(BaseModel):\n id: int\n name: str\n email: EmailStr\n created_at: datetime\n is_active: bool = True", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#8be9fd", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "screen3-footer", + "type": "text", + "x": 40, + "y": 1065, + "width": 660, + "height": 18, + "text": "[↑↓] Navigate [Enter] View Detail [n] New Type [d] Delete [q] Quit", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["screen3"], + "boundElements": null + }, + { + "id": "flow-arrow-1", + "type": "arrow", + "x": 720, + "y": 360, + "width": 40, + "height": 0, + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null, + "points": [[0, 0], [40, 0]], + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "flow-label-1", + "type": "text", + "x": 725, + "y": 340, + "width": 60, + "height": 18, + "text": "[Enter]", + "fontSize": 11, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "flow-arrow-2", + "type": "arrow", + "x": 370, + "y": 600, + "width": 0, + "height": 70, + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null, + "points": [[0, 0], [0, 70]], + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "flow-label-2", + "type": "text", + "x": 380, + "y": 625, + "width": 30, + "height": 18, + "text": "[t]", + "fontSize": 11, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ffff", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": [], + "boundElements": null + }, + { + "id": "legend-bg", + "type": "rectangle", + "x": 760, + "y": 670, + "width": 700, + "height": 130, + "strokeColor": "#444444", + "backgroundColor": "#252525", + "fillStyle": "solid", + "strokeWidth": 2, + "roughness": 0, + "opacity": 100, + "groupIds": ["legend"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "legend-title", + "type": "text", + "x": 780, + "y": 680, + "width": 200, + "height": 25, + "text": "UI Design Notes", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["legend"], + "boundElements": null + }, + { + "id": "legend-content", + "type": "text", + "x": 780, + "y": 710, + "width": 660, + "height": 80, + "text": "Color Scheme (Dracula-inspired):\n Primary Text: #e0e0e0 Success/OK: #00ff00 Error: #ff5555\n Highlight: #00ffff Warning/Dynamic: #ffff55 Comment: #6272a4\n \nFramework: Rich (Python) or Textual for TUI implementation", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#888888", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["legend"], + "boundElements": null + }, + { + "id": "features-bg", + "type": "rectangle", + "x": 760, + "y": 820, + "width": 700, + "height": 290, + "strokeColor": "#444444", + "backgroundColor": "#1e1e1e", + "fillStyle": "solid", + "strokeWidth": 2, + "roughness": 0, + "opacity": 100, + "groupIds": ["features"], + "boundElements": null, + "roundness": {"type": 3, "value": 8} + }, + { + "id": "features-title", + "type": "text", + "x": 780, + "y": 830, + "width": 200, + "height": 25, + "text": "Feature Checklist", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#e0e0e0", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["features"], + "boundElements": null + }, + { + "id": "features-content", + "type": "text", + "x": 780, + "y": 860, + "width": 660, + "height": 240, + "text": "Dashboard:\n [x] Cache statistics overview\n [x] Quick action shortcuts\n [x] Module list with status indicators\n [x] Keyboard navigation\n [x] Search functionality (planned)\n\nModule Detail:\n [x] Full metadata display\n [x] Syntax-highlighted source view\n [x] Regeneration options\n [x] Edit capabilities\n\nType Registry:\n [x] List all registered types\n [x] Type definition preview\n [x] Usage tracking across modules", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "strokeColor": "#00ff00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "roughness": 0, + "opacity": 100, + "groupIds": ["features"], + "boundElements": null + } + ] +} diff --git a/examples/14_evolve.py b/examples/14_evolve.py new file mode 100644 index 0000000..031d107 --- /dev/null +++ b/examples/14_evolve.py @@ -0,0 +1,113 @@ +"""Example 14: Evolve a function with scored mutations. + +Run with a real LLM: + uv run python examples/14_evolve.py + +Run offline with deterministic fake mutations: + WISHFUL_FAKE_LLM=1 uv run python examples/14_evolve.py +""" + +from __future__ import annotations + +import importlib +import os + +import wishful + + +def normalize_scores(values): + """Baseline implementation: correct, but intentionally a little clunky.""" + numbers = [float(value) for value in values] + if not numbers: + return [] + minimum = min(numbers) + maximum = max(numbers) + if maximum == minimum: + return [0.0 for _ in numbers] + return [(number - minimum) / (maximum - minimum) for number in numbers] + + +normalize_scores.__wishful_source__ = """ +def normalize_scores(values): + numbers = [float(value) for value in values] + if not numbers: + return [] + minimum = min(numbers) + maximum = max(numbers) + if maximum == minimum: + return [0.0 for _ in numbers] + return [(number - minimum) / (maximum - minimum) for number in numbers] +""".strip() + + +def is_correct(fn) -> bool: + """Correctness gate: outputs stay normalized and stable on edge cases.""" + cases = [ + ([10, 20, 30], [0.0, 0.5, 1.0]), + (["2", "2", "2"], [0.0, 0.0, 0.0]), + ([], []), + ] + return all(fn(values) == expected for values, expected in cases) + + +def source_quality(fn) -> float: + """Toy fitness: reward correct, concise source for a quick demo.""" + if not is_correct(fn): + return 0.0 + source = getattr(fn, "__wishful_source__", "") + return 1_000.0 - len(source) + + +def install_fake_mutations_if_needed() -> None: + """Make the example deterministic under WISHFUL_FAKE_LLM=1.""" + if os.getenv("WISHFUL_FAKE_LLM") != "1": + return + + candidates = iter( + [ + # Fails correctness because equal inputs divide by zero. + """ +def normalize_scores(values): + values = [float(value) for value in values] + minimum, maximum = min(values), max(values) + return [(value - minimum) / (maximum - minimum) for value in values] +""".strip(), + # Correct and shorter than the baseline. + """ +def normalize_scores(values): + values = [float(value) for value in values] + if not values: + return [] + span = max(values) - min(values) + return [0.0 if span == 0 else (value - min(values)) / span for value in values] +""".strip(), + ] + ) + + evolver_module = importlib.import_module("wishful.evolve.evolver") + evolver_module.mutate_with_llm = lambda **kwargs: next(candidates) + + +def main() -> None: + install_fake_mutations_if_needed() + + evolved = wishful.evolve( + normalize_scores, + fitness=source_quality, + test=is_correct, + generations=1, + variants=2, + mutation_prompt="Keep the behavior identical but make the code concise.", + verbose=False, + ) + + print("Original fitness:", source_quality(normalize_scores)) + print("Evolved fitness:", evolved.__wishful_evolution__["final_fitness"]) + print("Improvement:", evolved.__wishful_evolution__["improvement"]) + print("Result:", evolved([10, 20, 30])) + print("\nWinning source:\n") + print(evolved.__wishful_source__) + + +if __name__ == "__main__": + main() diff --git a/src/wishful/__init__.py b/src/wishful/__init__.py index 39c9afc..1a8aba0 100644 --- a/src/wishful/__init__.py +++ b/src/wishful/__init__.py @@ -10,6 +10,7 @@ from wishful.config import configure, reset_defaults, settings from wishful.core.discovery import set_context_radius as _set_context_radius from wishful.core.finder import install as install_finder +from wishful.evolve import EvolutionError, evolve from wishful.explore import ExplorationError, explore from wishful.llm.client import GenerationError from wishful.safety.validator import SecurityError @@ -30,7 +31,9 @@ "SecurityError", "GenerationError", "ExplorationError", + "EvolutionError", "explore", + "evolve", "type", ] @@ -57,7 +60,7 @@ def inspect_cache() -> List[str]: def regenerate(module_name: str) -> None: """Force regeneration of a module on next import. - + Accepts module names with or without the wishful.static prefix. Example: regenerate('users') or regenerate('wishful.static.users') """ @@ -65,7 +68,7 @@ def regenerate(module_name: str) -> None: if not module_name.startswith("wishful"): # Default to static namespace for backward compatibility module_name = f"wishful.static.{module_name}" - + cache.delete_cached(module_name) sys.modules.pop(module_name, None) importlib.invalidate_caches() @@ -78,23 +81,23 @@ def set_context_radius(radius: int) -> None: def reimport(module_path: str): """Force a fresh import by clearing the module from cache. - + This is especially useful for wishful.dynamic.* imports in loops, where you want the LLM to regenerate with fresh context on each iteration. - + Args: module_path: The full module path (e.g., 'wishful.dynamic.story') - + Returns: The freshly imported module - + Example: >>> story = wishful.reimport('wishful.dynamic.story') >>> next_line = story.cosmic_horror_next_sentence(current_text) """ # Clear from Python's module cache sys.modules.pop(module_path, None) - + # Import fresh (this triggers wishful's import hook if it's a wishful.* module) return importlib.import_module(module_path) diff --git a/src/wishful/evolve/__init__.py b/src/wishful/evolve/__init__.py new file mode 100644 index 0000000..d047811 --- /dev/null +++ b/src/wishful/evolve/__init__.py @@ -0,0 +1,13 @@ +"""wishful.evolve - AlphaEvolve-style evolutionary improvement.""" + +from wishful.evolve.evolver import evolve +from wishful.evolve.exceptions import EvolutionError +from wishful.evolve.history import EvolutionHistory, GenerationRecord, VariantRecord + +__all__ = [ + "EvolutionError", + "EvolutionHistory", + "GenerationRecord", + "VariantRecord", + "evolve", +] diff --git a/src/wishful/evolve/evolver.py b/src/wishful/evolve/evolver.py new file mode 100644 index 0000000..12b4211 --- /dev/null +++ b/src/wishful/evolve/evolver.py @@ -0,0 +1,218 @@ +"""Public evolution loop for improving Python functions.""" + +from __future__ import annotations + +import textwrap +from collections.abc import Callable +from typing import Any + +from wishful.config import settings +from wishful.evolve.exceptions import EvolutionError +from wishful.evolve.history import EvolutionHistory, GenerationRecord +from wishful.evolve.mutation import get_function_source, mutate_with_llm +from wishful.safety.validator import validate_code + + +def evolve( + fn: Callable[..., Any], + *, + fitness: Callable[[Callable[..., Any]], float], + generations: int = 5, + variants: int = 3, + test: Callable[[Callable[..., Any]], bool] | None = None, + mutation_prompt: str = "", + keep_history: bool = True, + history_limit: int = 10, + timeout_per_variant: float = 30.0, + verbose: bool = True, +) -> Callable[..., Any]: + """Improve a function by mutating it and selecting higher-fitness variants. + + Args: + fn: Function to evolve. + fitness: Scoring function. Higher scores are better. + generations: Number of mutation rounds to run. + variants: Number of candidate variants to try per generation. + test: Optional correctness filter. Variants that return false are rejected. + mutation_prompt: Human guidance included in the mutation prompt. + keep_history: Whether prior attempts should be passed to the LLM. + history_limit: Maximum number of prior attempts to include in mutation context. + timeout_per_variant: Reserved for future timeout enforcement. + verbose: Reserved for future progress output. + + Returns: + The best passing function, annotated with ``__wishful_source__`` and + ``__wishful_evolution__``. + + Raises: + EvolutionError: If the original function and all variants fail the test. + """ + _validate_evolve_args(generations, variants, history_limit, timeout_per_variant) + + function_name = fn.__name__ + current_source = _normalized_function_source(fn) + original_passed, original_error = _passes_test(fn, test) + original_fitness = _score_variant(fn, fitness) if original_passed else 0.0 + + history = EvolutionHistory( + original_fitness=original_fitness, + final_fitness=original_fitness, + generations=0, + total_variants_tried=0, + ) + history.add_variant( + current_source, + fitness=original_fitness if original_passed else None, + failed=not original_passed, + error_message=original_error, + ) + + best_fn = fn if original_passed else None + best_source = current_source + best_fitness = original_fitness if original_passed else float("-inf") + total_attempts = 0 + + for generation in range(1, generations + 1): + variants_tried = 0 + + for _ in range(variants): + variants_tried += 1 + total_attempts += 1 + + mutation_history = ( + history.get_context_for_llm(limit=history_limit) if keep_history else [] + ) + try: + candidate_source = mutate_with_llm( + source=best_source, + mutation_prompt=mutation_prompt, + function_name=function_name, + history=mutation_history, + ) + except Exception as exc: + history.add_variant( + "", + failed=True, + error_message=f"{type(exc).__name__}: {exc}", + ) + continue + + try: + candidate = _compile_function(candidate_source, function_name) + passed, error_message = _passes_test(candidate, test) + if not passed: + history.add_variant( + candidate_source, + failed=True, + error_message=error_message, + ) + continue + + candidate_fitness = _score_variant(candidate, fitness) + history.add_variant(candidate_source, fitness=candidate_fitness) + + if best_fn is None or candidate_fitness > best_fitness: + best_fn = candidate + best_source = textwrap.dedent(candidate_source).strip() + best_fitness = candidate_fitness + + except Exception as exc: + history.add_variant( + candidate_source, + failed=True, + error_message=f"{type(exc).__name__}: {exc}", + ) + + recorded_fitness = best_fitness if best_fn is not None else original_fitness + history.history.append( + GenerationRecord( + generation=generation, + best_fitness=recorded_fitness, + variants_tried=variants_tried, + best_source=best_source if best_fn is not None else None, + ) + ) + history.generations = generation + history.total_variants_tried = total_attempts + + if best_fn is None: + raise EvolutionError( + "No variant satisfied the evolution test", + best_variant=None, + best_fitness=None, + original_fitness=original_fitness, + generations_completed=generations, + total_attempts=total_attempts, + ) + + history.final_fitness = best_fitness + history.generations = generations + history.total_variants_tried = total_attempts + _attach_evolution_metadata(best_fn, best_source, history) + return best_fn + + +def _validate_evolve_args( + generations: int, + variants: int, + history_limit: int, + timeout_per_variant: float, +) -> None: + if generations < 0: + raise ValueError("generations must be >= 0") + if variants < 1: + raise ValueError("variants must be >= 1") + if history_limit < 0: + raise ValueError("history_limit must be >= 0") + if timeout_per_variant <= 0: + raise ValueError("timeout_per_variant must be > 0") + + +def _normalized_function_source(fn: Callable[..., Any]) -> str: + return textwrap.dedent(get_function_source(fn)).strip() + + +def _passes_test( + fn: Callable[..., Any], + test: Callable[[Callable[..., Any]], bool] | None, +) -> tuple[bool, str | None]: + if test is None: + return True, None + try: + if test(fn): + return True, None + return False, "test returned False" + except Exception as exc: + return False, f"{type(exc).__name__}: {exc}" + + +def _score_variant( + fn: Callable[..., Any], + fitness: Callable[[Callable[..., Any]], float], +) -> float: + return float(fitness(fn)) + + +def _compile_function(source: str, function_name: str) -> Callable[..., Any]: + normalized_source = textwrap.dedent(source).strip() + validate_code(normalized_source, allow_unsafe=settings.allow_unsafe) + namespace: dict[str, Any] = {} + exec(compile(normalized_source, "", "exec"), namespace) + + candidate = namespace.get(function_name) + if not callable(candidate): + raise EvolutionError( + f"Generated source did not define callable {function_name!r}" + ) + + setattr(candidate, "__wishful_source__", normalized_source) + return candidate + + +def _attach_evolution_metadata( + fn: Callable[..., Any], + source: str, + history: EvolutionHistory, +) -> None: + setattr(fn, "__wishful_source__", source) + setattr(fn, "__wishful_evolution__", history.to_dict()) diff --git a/src/wishful/evolve/exceptions.py b/src/wishful/evolve/exceptions.py new file mode 100644 index 0000000..e58f4c1 --- /dev/null +++ b/src/wishful/evolve/exceptions.py @@ -0,0 +1,23 @@ +"""Exceptions for wishful.evolve.""" + +from typing import Callable, Optional + + +class EvolutionError(Exception): + """Raised when evolution fails to produce valid results.""" + + def __init__( + self, + message: str, + best_variant: Optional[Callable] = None, + best_fitness: Optional[float] = None, + original_fitness: Optional[float] = None, + generations_completed: int = 0, + total_attempts: int = 0, + ): + super().__init__(message) + self.best_variant = best_variant + self.best_fitness = best_fitness + self.original_fitness = original_fitness + self.generations_completed = generations_completed + self.total_attempts = total_attempts diff --git a/src/wishful/evolve/history.py b/src/wishful/evolve/history.py new file mode 100644 index 0000000..3723602 --- /dev/null +++ b/src/wishful/evolve/history.py @@ -0,0 +1,125 @@ +"""Evolution history tracking for AlphaEvolve-style context passing.""" + +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class VariantRecord: + """Record of a single variant attempt.""" + + source: str + fitness: Optional[float] = None + failed: bool = False + error_message: Optional[str] = None + + +@dataclass +class GenerationRecord: + """Record of a single generation.""" + + generation: int + best_fitness: float + variants_tried: int + best_source: Optional[str] = None + + +@dataclass +class EvolutionHistory: + """Complete evolution history with context for LLM.""" + + original_fitness: float + final_fitness: float + generations: int + total_variants_tried: int + history: List[GenerationRecord] = field(default_factory=list) + + # All variant attempts (for context passing) + all_variants: List[VariantRecord] = field(default_factory=list) + + @property + def improvement(self) -> str: + """Return improvement as percentage string.""" + if self.original_fitness == 0: + return "N/A" + pct = ( + (self.final_fitness - self.original_fitness) + / abs(self.original_fitness) + * 100 + ) + sign = "+" if pct >= 0 else "" + return f"{sign}{pct:.1f}%" + + def get_context_for_llm(self, limit: int = 10) -> List[dict]: + """ + Get history formatted for LLM context. + + Returns top `limit` variants sorted by fitness (best first). + Failed variants are included to help LLM learn what doesn't work. + + This is THE KEY AlphaEvolve mechanism - passing history to LLM. + """ + # Sort by fitness (handle None as worst) + sorted_variants = sorted( + self.all_variants, + key=lambda v: v.fitness if v.fitness is not None else float("-inf"), + reverse=True, + ) + + # Take top N + top_variants = sorted_variants[:limit] + + # Format for LLM + return [ + { + "source": v.source, + "fitness": v.fitness, + "failed": v.failed, + "error": v.error_message, + } + for v in top_variants + ] + + def add_variant( + self, + source: str, + fitness: Optional[float] = None, + failed: bool = False, + error_message: Optional[str] = None, + ): + """Add a variant attempt to history.""" + self.all_variants.append( + VariantRecord( + source=source, + fitness=fitness, + failed=failed, + error_message=error_message, + ) + ) + + def to_dict(self) -> dict: + """Convert to dictionary for __wishful_evolution__.""" + return { + "original_fitness": self.original_fitness, + "final_fitness": self.final_fitness, + "improvement": self.improvement, + "generations": self.generations, + "total_variants_tried": self.total_variants_tried, + "history": [ + { + "generation": r.generation, + "best_fitness": r.best_fitness, + "variants_tried": r.variants_tried, + } + for r in self.history + ], + "variants": [ + { + "source": v.source, + "fitness": v.fitness, + "failed": v.failed, + "error": v.error_message, + } + for v in self.all_variants + ], + } diff --git a/src/wishful/evolve/mutation.py b/src/wishful/evolve/mutation.py new file mode 100644 index 0000000..511f37e --- /dev/null +++ b/src/wishful/evolve/mutation.py @@ -0,0 +1,163 @@ +"""LLM-based code mutation with history context for AlphaEvolve-style evolution. + +This module contains the core AlphaEvolve-inspired component: mutation functions +that receive and use evolutionary history to make informed code improvements. +""" + +from typing import Callable, List +import inspect + +from wishful.llm.client import generate_module_code + + +def mutate_with_llm( + source: str, mutation_prompt: str, function_name: str, history: List[dict] +) -> str: + """ + Ask the LLM to create a mutation informed by evolutionary history. + + This is the key AlphaEvolve insight: the LLM doesn't just see the current + codeβ€”it sees what approaches worked and what didn't, enabling it to make + increasingly informed mutations. + + Args: + source: The current best source code + mutation_prompt: User hint about how to mutate + function_name: Name of the function being evolved + history: List of previous attempts with scores: + [{"source": "...", "fitness": 51.0, "failed": False, "error": None}, ...] + + Returns: + New source code with mutation applied + """ + context = _build_evolution_context(source, mutation_prompt, function_name, history) + + # Use existing LLM infrastructure + return generate_module_code( + module="wishful.evolve._mutation", functions=[function_name], context=context + ) + + +def _build_evolution_context( + source: str, mutation_prompt: str, function_name: str, history: List[dict] +) -> str: + """ + Build rich context string for LLM mutation. + + The context includes: + 1. The current best implementation + 2. User guidance (mutation_prompt) + 3. Full history of attempts sorted by fitness + + This is THE KEY to AlphaEvolve: giving the LLM enough context to make + informed mutations rather than random changes. + """ + parts = [ + "=" * 60, + "EVOLUTION TASK: Improve this Python function", + "=" * 60, + "", + "CURRENT BEST IMPLEMENTATION:", + "```python", + source, + "```", + "", + ] + + # Add history context (the AlphaEvolve secret sauce) + if history: + parts.extend( + [ + "-" * 60, + "EVOLUTION HISTORY (sorted by fitness, best first):", + "-" * 60, + "", + "Learn from these previous attempts. Higher fitness = better.", + "", + ] + ) + + for i, entry in enumerate(history): + fitness = entry.get("fitness") + failed = entry.get("failed", False) + error = entry.get("error") + variant_source = entry.get("source", "") + + if failed: + status = f"FAILED: {error}" if error else "FAILED" + parts.append(f"Attempt {i + 1}: {status}") + else: + parts.append(f"Attempt {i + 1}: Fitness = {fitness:.2f}") + + # Include truncated source for context + source_preview = _truncate_source(variant_source, max_lines=10) + parts.extend(["```python", source_preview, "```", ""]) + + # Add user guidance (only if provided) + if mutation_prompt: + parts.extend(["-" * 60, f"USER GUIDANCE: {mutation_prompt}", "-" * 60, ""]) + + # Instructions for the LLM + parts.extend( + [ + "YOUR TASK:", + "1. Analyze what made high-scoring attempts successful", + "2. Understand why low-scoring attempts performed poorly", + "3. Create an IMPROVED version that should score higher", + "4. Keep the same function name and signature", + "5. Return ONLY the Python code, no explanations", + "", + ] + ) + + return "\n".join(parts) + + +def _truncate_source(source: str, max_lines: int = 10) -> str: + """ + Truncate source code to max_lines for context. + + Long source code can overflow context windows. This helper ensures + we include enough to understand the approach without overwhelming + the LLM's context. + """ + lines = source.strip().split("\n") + if len(lines) <= max_lines: + return source + return ( + "\n".join(lines[:max_lines]) + + f"\n # ... ({len(lines) - max_lines} more lines)" + ) + + +def get_function_source(fn: Callable) -> str: + """ + Get source code of a function. + + Tries multiple strategies: + 1. Check for __wishful_source__ attribute (wishful-generated functions) + 2. Use inspect.getsource() for regular functions + + Args: + fn: The function to get source code from + + Returns: + The source code as a string + + Raises: + ValueError: If source code cannot be obtained + """ + # First try wishful's attached source + if hasattr(fn, "__wishful_source__"): + return fn.__wishful_source__ + + # Try inspect + try: + return inspect.getsource(fn) + except (OSError, TypeError): + pass + + raise ValueError( + f"Cannot get source code for {fn}. " + "Function must have __wishful_source__ attribute or be inspectable." + ) diff --git a/tests/test_evolve.py b/tests/test_evolve.py new file mode 100644 index 0000000..2f7a4a3 --- /dev/null +++ b/tests/test_evolve.py @@ -0,0 +1,851 @@ +"""Tests for wishful.evolve functionality.""" + +import importlib + +import pytest + +from wishful.evolve.exceptions import EvolutionError +from wishful.evolve.history import EvolutionHistory, GenerationRecord, VariantRecord + + +class TestEvolutionErrorUnit: + """Unit tests for EvolutionError exception.""" + + def test_evolution_error_message(self): + """EvolutionError should store message.""" + err = EvolutionError("test error") + assert str(err) == "test error" + + def test_evolution_error_attributes(self): + """EvolutionError should store all attributes.""" + + def dummy(): + pass + + err = EvolutionError( + message="evolution failed", + best_variant=dummy, + best_fitness=42.5, + original_fitness=10.0, + generations_completed=5, + total_attempts=25, + ) + + assert err.best_variant == dummy + assert err.best_fitness == 42.5 + assert err.original_fitness == 10.0 + assert err.generations_completed == 5 + assert err.total_attempts == 25 + + def test_evolution_error_defaults(self): + """EvolutionError should have sensible defaults.""" + err = EvolutionError("test") + assert err.best_variant is None + assert err.best_fitness is None + assert err.original_fitness is None + assert err.generations_completed == 0 + assert err.total_attempts == 0 + + +class TestVariantRecord: + """Unit tests for VariantRecord dataclass.""" + + def test_variant_record_creation(self): + """VariantRecord should store variant data.""" + record = VariantRecord( + source="def fn(): pass", fitness=42.0, failed=False, error_message=None + ) + assert record.source == "def fn(): pass" + assert record.fitness == 42.0 + assert record.failed is False + assert record.error_message is None + + def test_variant_record_failed(self): + """VariantRecord should handle failed variants.""" + record = VariantRecord( + source="def fn(): syntax error", + fitness=None, + failed=True, + error_message="SyntaxError", + ) + assert record.failed is True + assert record.error_message == "SyntaxError" + + +class TestGenerationRecord: + """Unit tests for GenerationRecord dataclass.""" + + def test_generation_record_creation(self): + """GenerationRecord should store generation data.""" + record = GenerationRecord( + generation=1, + best_fitness=55.0, + variants_tried=5, + best_source="def fn(): return 1", + ) + assert record.generation == 1 + assert record.best_fitness == 55.0 + assert record.variants_tried == 5 + assert record.best_source == "def fn(): return 1" + + +class TestEvolutionHistory: + """Unit tests for EvolutionHistory - the core tracking class.""" + + def test_history_creation(self): + """EvolutionHistory should initialize correctly.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=50.0, + generations=5, + total_variants_tried=25, + ) + assert history.original_fitness == 10.0 + assert history.final_fitness == 50.0 + assert history.generations == 5 + assert history.total_variants_tried == 25 + assert history.history == [] + assert history.all_variants == [] + + def test_improvement_calculation(self): + """improvement property should calculate percentage correctly.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=15.0, + generations=1, + total_variants_tried=1, + ) + assert history.improvement == "+50.0%" + + def test_improvement_negative(self): + """improvement should handle negative changes.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=5.0, + generations=1, + total_variants_tried=1, + ) + assert history.improvement == "-50.0%" + + def test_improvement_zero_original(self): + """improvement should handle zero original fitness.""" + history = EvolutionHistory( + original_fitness=0.0, + final_fitness=10.0, + generations=1, + total_variants_tried=1, + ) + assert history.improvement == "N/A" + + def test_add_variant(self): + """add_variant should append to all_variants.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + history.add_variant("def fn(): return 1", fitness=20.0) + history.add_variant("def fn(): return 2", fitness=30.0) + + assert len(history.all_variants) == 2 + assert history.all_variants[0].fitness == 20.0 + assert history.all_variants[1].fitness == 30.0 + + def test_add_variant_failed(self): + """add_variant should handle failed variants.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + history.add_variant( + "def fn(): syntax error", failed=True, error_message="SyntaxError" + ) + + assert history.all_variants[0].failed is True + assert history.all_variants[0].error_message == "SyntaxError" + + def test_get_context_for_llm_sorted(self): + """get_context_for_llm should return variants sorted by fitness (best first).""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + history.add_variant("def fn(): return 1", fitness=10.0) + history.add_variant("def fn(): return 2", fitness=50.0) # Best + history.add_variant("def fn(): return 3", fitness=30.0) + + context = history.get_context_for_llm() + + assert len(context) == 3 + assert context[0]["fitness"] == 50.0 # Best first + assert context[1]["fitness"] == 30.0 + assert context[2]["fitness"] == 10.0 + + def test_get_context_for_llm_limit(self): + """get_context_for_llm should respect limit parameter.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + for i in range(10): + history.add_variant(f"def fn(): return {i}", fitness=float(i)) + + context = history.get_context_for_llm(limit=3) + + assert len(context) == 3 + # Should be top 3 by fitness: 9, 8, 7 + assert context[0]["fitness"] == 9.0 + assert context[1]["fitness"] == 8.0 + assert context[2]["fitness"] == 7.0 + + def test_get_context_for_llm_includes_failed(self): + """get_context_for_llm should include failed variants (they help LLM learn).""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + history.add_variant("def fn(): return 1", fitness=50.0) + history.add_variant( + "def fn(): syntax", failed=True, error_message="SyntaxError" + ) + + context = history.get_context_for_llm() + + assert len(context) == 2 + failed_entry = next(c for c in context if c["failed"]) + assert failed_entry["error"] == "SyntaxError" + + def test_get_context_for_llm_handles_none_fitness(self): + """get_context_for_llm should handle None fitness (treat as worst).""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=10.0, + generations=0, + total_variants_tried=0, + ) + + history.add_variant("def fn(): fail", fitness=None, failed=True) + history.add_variant("def fn(): return 1", fitness=10.0) + + context = history.get_context_for_llm() + + # None fitness should be sorted last + assert context[0]["fitness"] == 10.0 + assert context[1]["fitness"] is None + + def test_to_dict(self): + """to_dict should produce correct dictionary structure.""" + history = EvolutionHistory( + original_fitness=10.0, + final_fitness=50.0, + generations=2, + total_variants_tried=10, + ) + history.history.append( + GenerationRecord(generation=1, best_fitness=30.0, variants_tried=5) + ) + history.history.append( + GenerationRecord(generation=2, best_fitness=50.0, variants_tried=5) + ) + + d = history.to_dict() + + assert d["original_fitness"] == 10.0 + assert d["final_fitness"] == 50.0 + assert d["improvement"] == "+400.0%" + assert d["generations"] == 2 + assert d["total_variants_tried"] == 10 + assert len(d["history"]) == 2 + assert d["history"][0]["generation"] == 1 + assert d["history"][0]["best_fitness"] == 30.0 + + +# ============================================================================= +# Phase 2: Mutation Module Tests +# ============================================================================= + + +class TestBuildEvolutionContext: + """Tests for _build_evolution_context() - the AlphaEvolve context builder.""" + + def test_context_includes_current_source(self): + """Context should include the current best implementation.""" + from wishful.evolve.mutation import _build_evolution_context + + context = _build_evolution_context( + source="def fn(x):\n return x * 2", + mutation_prompt="", + function_name="fn", + history=[], + ) + + assert "def fn(x):" in context + assert "return x * 2" in context + assert "CURRENT BEST IMPLEMENTATION" in context + + def test_context_includes_mutation_prompt(self): + """Context should include user guidance when provided.""" + from wishful.evolve.mutation import _build_evolution_context + + context = _build_evolution_context( + source="def fn(x): return x", + mutation_prompt="make it faster using numpy", + function_name="fn", + history=[], + ) + + assert "make it faster using numpy" in context + assert "USER GUIDANCE" in context + + def test_context_excludes_prompt_when_empty(self): + """Context should not have USER GUIDANCE section when prompt is empty.""" + from wishful.evolve.mutation import _build_evolution_context + + context = _build_evolution_context( + source="def fn(x): return x", + mutation_prompt="", + function_name="fn", + history=[], + ) + + assert "USER GUIDANCE" not in context + + def test_context_includes_history(self): + """Context should include evolution history when provided.""" + from wishful.evolve.mutation import _build_evolution_context + + history = [ + { + "source": "def fn(x): return x + 1", + "fitness": 50.0, + "failed": False, + "error": None, + }, + { + "source": "def fn(x): return x * 2", + "fitness": 30.0, + "failed": False, + "error": None, + }, + ] + + context = _build_evolution_context( + source="def fn(x): return x + 1", + mutation_prompt="", + function_name="fn", + history=history, + ) + + assert "EVOLUTION HISTORY" in context + assert "sorted by fitness" in context.lower() + assert "Fitness = 50.00" in context + assert "Fitness = 30.00" in context + + def test_context_includes_failed_variants(self): + """Context should include failed variants with error info.""" + from wishful.evolve.mutation import _build_evolution_context + + history = [ + { + "source": "def fn(x): return x", + "fitness": 50.0, + "failed": False, + "error": None, + }, + { + "source": "def fn(x): syntax error", + "fitness": None, + "failed": True, + "error": "SyntaxError", + }, + ] + + context = _build_evolution_context( + source="def fn(x): return x", + mutation_prompt="", + function_name="fn", + history=history, + ) + + assert "FAILED" in context + assert "SyntaxError" in context + + def test_context_has_task_instructions(self): + """Context should include clear instructions for the LLM.""" + from wishful.evolve.mutation import _build_evolution_context + + context = _build_evolution_context( + source="def fn(x): return x", + mutation_prompt="", + function_name="fn", + history=[], + ) + + assert "YOUR TASK" in context + assert "IMPROVED" in context or "improved" in context + + def test_context_empty_history(self): + """Context should work with empty history.""" + from wishful.evolve.mutation import _build_evolution_context + + context = _build_evolution_context( + source="def fn(x): return x", + mutation_prompt="", + function_name="fn", + history=[], + ) + + # Should not include history section when empty + assert "EVOLUTION HISTORY" not in context + + +class TestTruncateSource: + """Tests for _truncate_source() utility.""" + + def test_short_source_unchanged(self): + """Short source code should be returned unchanged.""" + from wishful.evolve.mutation import _truncate_source + + source = "def fn():\n return 1" + result = _truncate_source(source, max_lines=10) + + assert result == source + + def test_long_source_truncated(self): + """Long source code should be truncated with indicator.""" + from wishful.evolve.mutation import _truncate_source + + lines = [f" line{i}" for i in range(20)] + source = "def fn():\n" + "\n".join(lines) + + result = _truncate_source(source, max_lines=5) + + # Should have max_lines lines + result_lines = result.strip().split("\n") + assert len(result_lines) == 6 # 5 lines + truncation indicator + + # Should have truncation indicator + assert "more lines" in result + + def test_truncate_exact_limit(self): + """Source at exactly max_lines should not be truncated.""" + from wishful.evolve.mutation import _truncate_source + + lines = ["def fn():"] + [f" line{i}" for i in range(9)] + source = "\n".join(lines) + + result = _truncate_source(source, max_lines=10) + + assert result == source + assert "more lines" not in result + + +class TestGetFunctionSource: + """Tests for get_function_source() utility.""" + + def test_get_source_from_wishful_attribute(self): + """Should extract source from __wishful_source__ attribute.""" + from wishful.evolve.mutation import get_function_source + + def dummy(): + pass + + dummy.__wishful_source__ = "def dummy():\n return 42" + + result = get_function_source(dummy) + + assert result == "def dummy():\n return 42" + + def test_get_source_from_inspect(self): + """Should fall back to inspect.getsource() for regular functions.""" + from wishful.evolve.mutation import get_function_source + + def regular_function(): + return 123 + + result = get_function_source(regular_function) + + assert "def regular_function" in result + assert "return 123" in result + + def test_get_source_raises_for_unavailable(self): + """Should raise ValueError when source is unavailable.""" + from wishful.evolve.mutation import get_function_source + + # Built-in function has no source + with pytest.raises(ValueError, match="source"): + get_function_source(len) + + +class TestMutateWithLLM: + """Tests for mutate_with_llm() - the core LLM mutation function.""" + + def test_mutate_calls_llm(self, monkeypatch): + """mutate_with_llm should call generate_module_code.""" + called_with = {"context": None, "functions": None} + + def fake_generate(module, functions, context, **kwargs): + called_with["context"] = context + called_with["functions"] = functions + return "def fn(x):\n return x + 1" + + mutation_module = importlib.import_module("wishful.evolve.mutation") + monkeypatch.setattr(mutation_module, "generate_module_code", fake_generate) + + from wishful.evolve.mutation import mutate_with_llm + + result = mutate_with_llm( + source="def fn(x):\n return x", + mutation_prompt="improve it", + function_name="fn", + history=[], + ) + + assert called_with["functions"] == ["fn"] + assert "def fn(x)" in called_with["context"] + assert "improve it" in called_with["context"] + assert "def fn(x):" in result + + def test_mutate_passes_history_to_context(self, monkeypatch): + """mutate_with_llm should include history in LLM context.""" + called_with = {"context": None} + + def fake_generate(module, functions, context, **kwargs): + called_with["context"] = context + return "def fn(x):\n return x + 1" + + mutation_module = importlib.import_module("wishful.evolve.mutation") + monkeypatch.setattr(mutation_module, "generate_module_code", fake_generate) + + from wishful.evolve.mutation import mutate_with_llm + + history = [ + { + "source": "def fn(x): return x * 2", + "fitness": 50.0, + "failed": False, + "error": None, + }, + ] + + mutate_with_llm( + source="def fn(x):\n return x", + mutation_prompt="", + function_name="fn", + history=history, + ) + + assert "Fitness = 50.00" in called_with["context"] + assert "def fn(x): return x * 2" in called_with["context"] + + def test_mutate_returns_generated_code(self, monkeypatch): + """mutate_with_llm should return the LLM-generated code.""" + + def fake_generate(module, functions, context, **kwargs): + return "def fn(x):\n return x * 100" + + mutation_module = importlib.import_module("wishful.evolve.mutation") + monkeypatch.setattr(mutation_module, "generate_module_code", fake_generate) + + from wishful.evolve.mutation import mutate_with_llm + + result = mutate_with_llm( + source="def fn(x):\n return x", + mutation_prompt="", + function_name="fn", + history=[], + ) + + assert result == "def fn(x):\n return x * 100" + + +# ============================================================================= +# Phase 3: Public evolve() Loop Tests +# ============================================================================= + + +class TestEvolveBasic: + """Tests for the public evolve() loop.""" + + def test_evolve_returns_better_variant(self, monkeypatch): + """evolve should return the highest-fitness generated variant.""" + from wishful.evolve import evolve + + def score(fn): + return float(fn(10)) + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + variants = iter( + [ + "def transform(x):\n return x + 5", + "def transform(x):\n return x + 20", + ] + ) + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr( + evolver_module, "mutate_with_llm", lambda **kwargs: next(variants) + ) + + evolved = evolve( + transform, + fitness=score, + generations=1, + variants=2, + verbose=False, + ) + + assert evolved(10) == 30 + assert evolved.__wishful_source__ == "def transform(x):\n return x + 20" + assert evolved.__wishful_evolution__["original_fitness"] == 10.0 + assert evolved.__wishful_evolution__["final_fitness"] == 30.0 + assert evolved.__wishful_evolution__["improvement"] == "+200.0%" + + def test_evolve_uses_history_for_later_mutations(self, monkeypatch): + """evolve should pass scored variant history into subsequent mutations.""" + from wishful.evolve import evolve + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + seen_history = [] + variants = iter( + [ + "def transform(x):\n return x + 1", + "def transform(x):\n return x + 2", + ] + ) + + def fake_mutate(**kwargs): + seen_history.append(kwargs["history"]) + return next(variants) + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr(evolver_module, "mutate_with_llm", fake_mutate) + + evolve( + transform, + fitness=lambda fn: float(fn(10)), + generations=1, + variants=2, + history_limit=5, + verbose=False, + ) + + assert seen_history[0][0]["fitness"] == 10.0 + assert any(entry["fitness"] == 11.0 for entry in seen_history[1]) + + def test_evolve_can_disable_history_context(self, monkeypatch): + """keep_history=False should call mutations with an empty history.""" + from wishful.evolve import evolve + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + seen_history = [] + + def fake_mutate(**kwargs): + seen_history.append(kwargs["history"]) + return "def transform(x):\n return x + 1" + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr(evolver_module, "mutate_with_llm", fake_mutate) + + evolve( + transform, + fitness=lambda fn: float(fn(10)), + generations=1, + variants=1, + keep_history=False, + verbose=False, + ) + + assert seen_history == [[]] + + +class TestEvolveWithTest: + """Tests for correctness filters during evolution.""" + + def test_evolve_skips_variants_that_fail_test(self, monkeypatch): + """A high-scoring variant must not win if the test rejects it.""" + from wishful.evolve import evolve + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + variants = iter( + [ + "def transform(x):\n return 999", + "def transform(x):\n return x + 5", + ] + ) + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr( + evolver_module, "mutate_with_llm", lambda **kwargs: next(variants) + ) + + evolved = evolve( + transform, + fitness=lambda fn: float(fn(10)), + test=lambda fn: fn(10) < 100, + generations=1, + variants=2, + verbose=False, + ) + + assert evolved(10) == 15 + failures = [ + entry + for entry in evolved.__wishful_evolution__["variants"] + if entry["failed"] + ] + assert failures + assert "test returned False" in failures[0]["error"] + + def test_evolve_does_not_score_original_that_fails_test(self, monkeypatch): + """A rejected baseline should not crash fitness before mutations run.""" + from wishful.evolve import evolve + + def transform(x): + return 999 + + transform.__wishful_source__ = "def transform(x):\n return 999" + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr( + evolver_module, + "mutate_with_llm", + lambda **kwargs: "def transform(x):\n return x + 3", + ) + + def score(fn): + value = fn(10) + if value >= 100: + raise AssertionError("fitness should only score passing functions") + return float(value) + + evolved = evolve( + transform, + fitness=score, + test=lambda fn: fn(10) < 100, + generations=1, + variants=1, + verbose=False, + ) + + assert evolved(10) == 13 + evolution = evolved.__wishful_evolution__ + assert evolution["original_fitness"] == 0.0 + assert evolution["variants"][0]["failed"] is True + assert "test returned False" in evolution["variants"][0]["error"] + + def test_evolve_raises_when_original_and_all_variants_fail(self, monkeypatch): + """evolve should raise with best-known context when nothing satisfies test.""" + from wishful.evolve import evolve + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr( + evolver_module, + "mutate_with_llm", + lambda **kwargs: "def transform(x):\n return x + 1", + ) + + with pytest.raises(EvolutionError) as exc: + evolve( + transform, + fitness=lambda fn: float(fn(10)), + test=lambda fn: False, + generations=1, + variants=1, + verbose=False, + ) + + assert exc.value.generations_completed == 1 + assert exc.value.total_attempts == 1 + + def test_evolve_records_mutation_errors_and_keeps_trying(self, monkeypatch): + """A failed mutation call should not abort the whole evolution run.""" + from wishful.evolve import evolve + + def transform(x): + return x + + transform.__wishful_source__ = "def transform(x):\n return x" + + calls = 0 + + def fake_mutate(**kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("provider hiccup") + return "def transform(x):\n return x + 3" + + evolver_module = importlib.import_module("wishful.evolve.evolver") + monkeypatch.setattr(evolver_module, "mutate_with_llm", fake_mutate) + + evolved = evolve( + transform, + fitness=lambda fn: float(fn(10)), + generations=1, + variants=2, + verbose=False, + ) + + assert evolved(10) == 13 + failures = [ + entry + for entry in evolved.__wishful_evolution__["variants"] + if entry["failed"] + ] + assert "provider hiccup" in failures[0]["error"] + + +class TestEvolveIntegration: + """Tests for evolve package exports.""" + + def test_evolve_importable_from_package(self): + """evolve and EvolutionError should be importable from wishful.evolve.""" + from wishful.evolve import EvolutionError as PackageEvolutionError + from wishful.evolve import evolve + + assert callable(evolve) + assert PackageEvolutionError is EvolutionError + + def test_evolve_importable_from_wishful(self): + """evolve and EvolutionError should be importable from root package.""" + from wishful import EvolutionError as RootEvolutionError + from wishful import evolve + + assert callable(evolve) + assert RootEvolutionError is EvolutionError diff --git a/uv.lock b/uv.lock index 3f0bcb7..d5509fd 100644 --- a/uv.lock +++ b/uv.lock @@ -1725,7 +1725,7 @@ wheels = [ [[package]] name = "wishful" -version = "0.2.3" +version = "0.2.4" source = { editable = "." } dependencies = [ { name = "litellm" },