From 7a35d94464d74f1d9889dd6d1682d47bb0e18e20 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 19:33:30 -0500 Subject: [PATCH 01/17] current plan --- plans/API.md | 98 ++++++++++++++++++++++++++++++++++++++ plans/ARCH.md | 58 +++++++++++++++++++++++ plans/CHECKLIST.md | 23 +++++++++ plans/DATA.md | 33 +++++++++++++ plans/DEVLOG_CONTRACT.md | 14 ++++++ plans/FEEDBACK.md | 1 + plans/FLOWS.md | 89 ++++++++++++++++++++++++++++++++++ plans/IMPLEMENTER.md | 53 +++++++++++++++++++++ plans/PLAN.md | 100 +++++++++++++++++++++++++++++++++++++++ plans/RISKS.md | 32 +++++++++++++ plans/TESTPLAN.md | 66 ++++++++++++++++++++++++++ plans/VERIFIER.md | 16 +++++++ 12 files changed, 583 insertions(+) create mode 100644 plans/API.md create mode 100644 plans/ARCH.md create mode 100644 plans/CHECKLIST.md create mode 100644 plans/DATA.md create mode 100644 plans/DEVLOG_CONTRACT.md create mode 100644 plans/FEEDBACK.md create mode 100644 plans/FLOWS.md create mode 100644 plans/IMPLEMENTER.md create mode 100644 plans/PLAN.md create mode 100644 plans/RISKS.md create mode 100644 plans/TESTPLAN.md create mode 100644 plans/VERIFIER.md diff --git a/plans/API.md b/plans/API.md new file mode 100644 index 0000000..901ed9e --- /dev/null +++ b/plans/API.md @@ -0,0 +1,98 @@ +# API + +1. Tool: call_subagent + 1. Signature: call_subagent(prompt: str, system_prompt: str | None = None, tool_subset: list[str] | None = None) -> str + 2. Behavior: spawns subprocess; parses SSE; returns concatenated output_text. + 3. Error behavior: returns "Error: " string on validation or subprocess failure. +2. Subprocess input contract (stdin JSON) + 1. Fields: model, provider, url, api_key_name, prompt, system_prompt, tool_subset, think, temperature, max_tokens. + 2. Example payload: + { + "model": "", + "provider": "openai-chat|anthropic|vertexai-anthropic", + "url": "", + "api_key_name": "", + "prompt": "...", + "system_prompt": "...", + "tool_subset": ["search_web", "scrape_webpage"], + "think": false, + "temperature": 0.8, + "max_tokens": 8192 + } +3. SSE parsing contract + 1. Only consume events with "event: output_text". + 2. Ignore tool_call, tool_result, reasoning_content, block_end, response_end. + +Available tool names +1. search_web +2. scrape_webpage +3. bash_find +4. bash_ripgrep +5. bash_read +6. python_execute + +## OpenAI Responses API Contract + +1. Request args (make_openai_responses_request_args) + 1. data.model = opts.model + 2. data.input = [ + {"role": "user", "content": [{"type": "input_text", "text": prompt}]} + ] + 3. data.instructions = system_prompt when provided + 4. data.stream = True, data.store = False + 5. data.temperature = opts.temperature + 6. data.max_output_tokens = opts.max_tokens when provided + 7. data.reasoning = {"effort": "high", "summary": "detailed"} if opts.think else {"effort": "none"} + 8. if opts.tools: data.tools = opts.tools and data.tool_choice = "auto" + +2. Streaming event mapping (handle_openai_responses_stream_response) + 1. response.created -> StreamResponse(type=response_start, id=response.id) + 2. response.output_text.delta -> StreamResponse(type=output_text, delta) + 3. response.reasoning_summary_text.delta -> StreamResponse(type=reasoning_content, delta) + 4. response.output_item.added (item.type=tool_call) -> create ToolCall(id=item.id, name=item.name) + 5. response.tool_call_arguments.delta -> append ToolCall.arguments; emit StreamResponse(type=tool_call, delta) + 6. response.output_item.done (item.type=tool_call) -> set ToolCall.arguments from item.arguments if present + 7. response.completed -> StreamResponse(type=block_end) + +3. Tool response continuation (tool_call_response_to_openai_responses_messages) + 1. request_args.data["previous_response_id"] = last_openai_response_id + 2. request_args.data["input"] = [ + {"type": "tool_result", "tool_call_id": tc.id, "content": str(tc.result)} + ] for each tool call + 3. tool_result items replace prior input for the follow-up request (no message history replay). + +4. invoke_llm routing contract + 1. If request_args.data contains "messages", use that list and write back to "messages". + 2. Else if request_args.data contains "input", use that list and write back to "input". + +## Structured Outputs API Contract + +1. LLMOptions additions (internal) + 1. output_mode: "json_schema" | "json_object" | None + 2. output_schema: dict | None +2. OpenAI chat request args + 1. response_format json_schema: + {"type":"json_schema","json_schema":{"name":"","schema":,"strict":true}} + 2. response_format json_object: + {"type":"json_object"} +3. Anthropic/Vertex request args + 1. output_config: {"format":{"type":"json_schema","schema":}} +4. Output to stdout + 1. Non-SSE prints JSON once at end, then DONE marker. + 2. SSE emits one output_structured event with JSON string. + +Non-SSE example +``` +{"capital":"Paris","country":"France"} + +=== [ DONE ] === +``` + +SSE example +``` +event: output_structured +data: {"id":"","delta":"{\"capital\":\"Paris\",\"country\":\"France\"}","type":"output_structured"} + +event: response_end +data: {"id":"","delta":"","type":"response_end"} +``` diff --git a/plans/ARCH.md b/plans/ARCH.md new file mode 100644 index 0000000..64b2d1d --- /dev/null +++ b/plans/ARCH.md @@ -0,0 +1,58 @@ +# Architecture + +1. New modules + 1. src/buzzllm/subagent.py: call_subagent tool + SSE parsing. + 2. src/buzzllm/subagent_runner.py: subprocess entrypoint; reads JSON; runs invoke_llm SSE. + 3. src/buzzllm/tools/catalog.py: tool name to callable/desc mapping. +2. Existing modules touched + 1. src/buzzllm/tools/utils.py: schema builder helpers for tool subsets. + 2. src/buzzllm/main.py: include call_subagent in tool list when tools enabled. + +```mermaid +graph TD + A[Main invoke_llm] --> B[call_subagent tool] + B --> C[Subprocess: subagent_runner] + C --> D[invoke_llm SSE] + D --> E[SSE stdout] + E --> B + C --> F[Tool registry (subset only)] +``` + +Acceptance +1. Subprocess has isolated TOOL_CALLS and tool registry. +2. Main agent receives only subagent output_text. + +## OpenAI Responses Tooling + +1. Existing modules touched + 1. src/buzzllm/llm.py: make_openai_responses_request_args, handle_openai_responses_stream_response, tool_call_response_to_openai_responses_messages, invoke_llm input/messages routing. + 2. tests/unit/test_llm_request_args.py, tests/unit/test_llm_stream_handlers.py, tests/unit/test_llm_tool_messages.py. + 3. tests/integration/test_openai_responses_integration.py (new). +2. In-memory state + 1. last_openai_response_id (string) stored during streaming for previous_response_id. + 2. input list buffer for tool_result continuation requests. +3. No new modules; reuse existing ToolCall structure and TOOL_CALLS registry. + +```mermaid +graph TD + A[invoke_llm] --> B[make_openai_responses_request_args] + B --> C[OpenAI Responses SSE] + C --> D[handle_openai_responses_stream_response] + D --> E[TOOL_CALLS populated] + E --> F[run_tools] + F --> G[tool_call_response_to_openai_responses_messages] + G --> B +``` + +Acceptance +1. Responses tool loop uses previous_response_id without affecting chat completions. +2. invoke_llm supports messages and input modes without branching elsewhere. + +## Structured Outputs Architecture + +1. Existing modules touched + 1. src/buzzllm/llm.py: LLMOptions structured fields, request args (openai-chat, anthropic, vertex), invoke_llm buffering, print_to_stdout output_structured. +2. In-memory state + 1. structured_output_buffer (string) used only when structured output is enabled. +3. StreamResponse type + 1. Add output_structured to StreamResponse.type for SSE. diff --git a/plans/CHECKLIST.md b/plans/CHECKLIST.md new file mode 100644 index 0000000..a4f4eda --- /dev/null +++ b/plans/CHECKLIST.md @@ -0,0 +1,23 @@ +# Definition of Done + +1. call_subagent tool available when tools enabled. +2. Subagent tool_subset allowlist enforced. +3. SSE parser returns only output_text. +4. uv run pytest passes. +5. devlogs.md updated for final iteration. +6. No new dependencies added. + +## OpenAI Responses Definition of Done + +1. make_openai_responses_request_args builds input list and tools when present. +2. handle_openai_responses_stream_response emits tool_call + output_text events. +3. Tool results are sent with previous_response_id and tool_result input items. +4. uv run pytest passes (integration skipped without OPENAI_API_KEY). +5. devlogs.md updated for final iteration. + +## Structured Outputs Definition of Done + +1. OpenAI chat uses response_format when output_mode is set. +2. Anthropic/Vertex uses output_config.format when output_mode is set. +3. Non-SSE prints JSON once; SSE emits output_structured. +4. uv run pytest passes. diff --git a/plans/DATA.md b/plans/DATA.md new file mode 100644 index 0000000..1236120 --- /dev/null +++ b/plans/DATA.md @@ -0,0 +1,33 @@ +# Data + +1. Storage changes: none. +2. Migrations: none. +3. In-memory data: + 1. SSE output_text buffer in src/buzzllm/subagent.py. + 2. tool_subset list (validated against catalog). + +Acceptance +1. No new files under data/ or migrations/. +2. No persistent state introduced. + +## OpenAI Responses Data + +1. Storage changes: none. +2. Migrations: none. +3. In-memory data: + 1. last_openai_response_id (string) for previous_response_id. + 2. request_args.data["input"] list reused between tool turns. + +Acceptance +1. No new persistence is introduced for responses. +2. previous_response_id is only stored in-memory per invocation. + +## Structured Outputs Data + +1. Storage changes: none. +2. Migrations: none. +3. In-memory data: + 1. structured_output_buffer (string) for output_text accumulation. + +Acceptance +1. No new persistence is introduced for structured outputs. diff --git a/plans/DEVLOG_CONTRACT.md b/plans/DEVLOG_CONTRACT.md new file mode 100644 index 0000000..20867aa --- /dev/null +++ b/plans/DEVLOG_CONTRACT.md @@ -0,0 +1,14 @@ +# Devlog Contract + +1. Update devlogs.md at least once per iteration and after each verifier run. +2. Use this template: + 1. Timestamp: + 2. Goal: + 3. Changes: + 4. Commands+Results: + 5. Decisions: + 6. Next: + 7. Checkpoint: +3. Checkpoint hash must be updated after each milestone. + +4. Applies to all plan sections, including OpenAI Responses Tooling milestones. diff --git a/plans/FEEDBACK.md b/plans/FEEDBACK.md new file mode 100644 index 0000000..340d7f8 --- /dev/null +++ b/plans/FEEDBACK.md @@ -0,0 +1 @@ +# Feedback diff --git a/plans/FLOWS.md b/plans/FLOWS.md new file mode 100644 index 0000000..9f0fca6 --- /dev/null +++ b/plans/FLOWS.md @@ -0,0 +1,89 @@ +# Flows + +1. Main to subagent flow + 1. Main agent calls call_subagent(prompt, system_prompt, tool_subset). + 2. call_subagent builds JSON payload with parent model/provider/url/api_key_name. + 3. call_subagent spawns subprocess runner and parses SSE output_text. + 4. call_subagent returns concatenated output_text to main agent. +2. Tool subset validation flow + 1. tool_subset is validated against catalog names. + 2. If unknown tools exist, return error string without spawning subprocess. +3. Failure flow + 1. If subprocess exits non-zero, return error string containing exit code. + +```mermaid +sequenceDiagram + participant M as Main agent + participant T as call_subagent tool + participant P as Subprocess runner + participant L as invoke_llm + + M->>T: tool call (prompt, system_prompt, tool_subset) + T->>P: spawn + send JSON + P->>L: invoke_llm (SSE) + L-->>P: SSE events + P-->>T: SSE stdout + T-->>M: output_text (concatenated) +``` + +Acceptance +1. call_subagent returns only output_text concatenation. +2. Unknown tool_subset names do not spawn subprocess. + +## OpenAI Responses Flows + +1. Initial request flow + 1. Build input list with user input_text and optional instructions. + 2. Send responses request with tools + stream enabled. + 3. Stream output_text deltas and reasoning summary (if present). +2. Tool call flow + 1. On response.output_item.added with tool_call, create ToolCall and capture id/name. + 2. On response.tool_call_arguments.delta, append arguments and emit tool_call deltas. + 3. After tool execution, send follow-up with previous_response_id and tool_result input items. +3. Failure flow + 1. If tool execution raises, invoke_llm prints error block_end and exits. + 2. If responses stream emits invalid JSON, ignore that line and continue. + +```mermaid +sequenceDiagram + participant U as User + participant L as invoke_llm + participant O as OpenAI Responses + participant T as Tools + + U->>L: prompt + system + L->>O: /v1/responses (input + tools) + O-->>L: output_text.delta + O-->>L: output_item.added (tool_call) + O-->>L: tool_call_arguments.delta + L->>T: execute tool calls + T-->>L: results + L->>O: /v1/responses (previous_response_id + tool_result) + O-->>L: output_text.delta +``` + +Acceptance +1. Tool call flow works without constructing chat messages. +2. Follow-up request uses previous_response_id from response.created. + +## Structured Outputs Flow + +1. OpenAI chat flow + 1. Set response_format json_schema or json_object in request args. + 2. Buffer output_text deltas; do not print partial JSON. + 3. On block_end with no tool calls, emit output_structured event or print JSON once. +2. Anthropic/Vertex flow + 1. Set output_config.format json_schema in request args. + 2. Buffer output_text deltas; emit output_structured at end. + +```mermaid +sequenceDiagram + participant U as User + participant L as invoke_llm + participant P as Provider + + U->>L: prompt + structured schema + L->>P: request with response_format/output_config + P-->>L: output_text deltas (buffered) + L-->>U: output_structured (SSE) or JSON once (stdout) +``` diff --git a/plans/IMPLEMENTER.md b/plans/IMPLEMENTER.md new file mode 100644 index 0000000..d55cf59 --- /dev/null +++ b/plans/IMPLEMENTER.md @@ -0,0 +1,53 @@ +# Implementer Guide + +Allowed tools/commands +1. rg, uv run pytest, python -m buzzllm.subagent_runner (local), git status, git diff. + +Repo conventions +1. Keep changes minimal and localized. +2. Keep tool names in a single catalog module. +3. Do not add new dependencies. + +Do not list +1. Do not refactor unrelated code. +2. Do not rename widely used symbols. +3. Do not add try/except unless explicitly requested. +4. Do not change CLI behavior beyond adding call_subagent tool availability. + +Milestones +1. M1: Add tool catalog + helper builder; tests for catalog pass. +2. M2: Add subagent runner + SSE parser; unit tests pass. +3. M3: Add call_subagent tool + main wiring; integration tests pass. +4. M4: Full test run passes; update devlogs.md. + +Devlog + checkpoints +1. Update devlogs.md every iteration and after each verifier run. +2. Commit checkpoint after each milestone with message: ralph(iter N): . +3. If stuck for 3 iterations, propose/perform rollback and log it. + +## OpenAI Responses Implementer Milestones + +Allowed tools/commands +1. rg, uv run pytest, python -m buzzllm.main (local), git status, git diff. + +Do not list +1. Do not change openai-chat or anthropic behavior. +2. Do not weaken or delete tests; update expectations instead. +3. Do not add new dependencies. + +Milestones +1. M1: Update make_openai_responses_request_args to build input list + tools (file: src/buzzllm/llm.py; tests: tests/unit/test_llm_request_args.py). +2. M2: Parse tool call events in handle_openai_responses_stream_response (file: src/buzzllm/llm.py; tests: tests/unit/test_llm_stream_handlers.py). +3. M3: Implement tool_call_response_to_openai_responses_messages and invoke_llm input routing (file: src/buzzllm/llm.py; tests: tests/unit/test_llm_tool_messages.py). +4. M4: Add integration test for responses API and run full pytest (file: tests/integration/test_openai_responses_integration.py). + +Devlog + checkpoints +1. Update devlogs.md every iteration and after each verifier run. +2. Commit checkpoint after each milestone with message: ralph(iter N): . +3. If stuck for 3 iterations, propose/perform rollback and log it. + +## Structured Outputs Implementer Notes + +1. Do not modify system prompts for structured output. +2. When structured output is enabled, buffer output_text and emit output_structured once. +3. Non-SSE should print JSON once, then DONE marker. diff --git a/plans/PLAN.md b/plans/PLAN.md new file mode 100644 index 0000000..4efb87e --- /dev/null +++ b/plans/PLAN.md @@ -0,0 +1,100 @@ +# Plan: Subagent Tool Runner (SSE subprocess) + +1. Feature name: Subagent Tool Runner (SSE subprocess) +2. Objective: Allow main agent to spawn subagents with tool subsets using the same model/provider, returning only concatenated output_text. +3. Context/problem: current system is single-agent; tool wiring is tied to system prompts; globals make in-process nesting unsafe. +4. Non-goals: openai-responses tool support, streaming subagent output to user, persistent subagent state, new UI/CLI flags. +5. Success criteria: call_subagent tool available when tools are enabled; tool_subset allowlist enforced; output is output_text only; tests pass; no regression in existing flows. +6. Constraints: no new deps; no new try/except unless explicitly requested; ASCII only; minimal diff; subprocess uses SSE. + +Links +1. ARCH.md +2. FLOWS.md +3. API.md +4. DATA.md +5. TESTPLAN.md +6. VERIFIER.md +7. IMPLEMENTER.md +8. DEVLOG_CONTRACT.md +9. RISKS.md +10. CHECKLIST.md +11. FEEDBACK.md + +Milestones +1. M1: Tool catalog + schema builder for named tool subsets (files: src/buzzllm/tools/catalog.py, src/buzzllm/tools/utils.py). +2. M2: Subagent runner reads JSON from stdin, runs invoke_llm SSE (files: src/buzzllm/subagent_runner.py). +3. M3: call_subagent tool + wiring in main tool set (files: src/buzzllm/subagent.py, src/buzzllm/main.py). +4. M4: Tests + fixtures (files: tests/unit/test_subagent_*.py, tests/integration/test_subagent_runner.py). + +Acceptance +1. call_subagent returns concatenated output_text string. +2. Subagent tool_subset is enforced; unknown tool names return an error string. +3. SSE parser ignores tool_call/tool_result/reasoning_content events. +4. uv run pytest passes. + +# Plan: OpenAI Responses Tooling + Streaming Parity + +1. Feature name: OpenAI Responses Tooling + Streaming Parity +2. Objective: Enable tools and multi-turn tool loop for openai-responses with streaming parity to openai-chat. +3. Context/problem: openai-responses request args ignore tools, tool loop is messages-only, and stream handler does not parse tool calls. +4. Non-goals: new CLI flags, new providers, refactoring beyond input/messages routing, new dependencies. +5. Success criteria: tools execute end-to-end via openai-responses; tool calls parsed and emitted; tool results appended via previous_response_id; unit tests pass; integration test passes when API key is present. +6. Constraints: no new deps; minimal diff; no new try/except; ASCII only; keep openai-chat/anthropic behavior unchanged; keep requests streaming. + +Links +1. ARCH.md +2. FLOWS.md +3. API.md +4. DATA.md +5. TESTPLAN.md +6. VERIFIER.md +7. IMPLEMENTER.md +8. DEVLOG_CONTRACT.md +9. RISKS.md +10. CHECKLIST.md +11. FEEDBACK.md + +Milestones +1. M1: Request args + input contract updated for openai-responses (files: src/buzzllm/llm.py, tests/unit/test_llm_request_args.py). +2. M2: Stream handler parses tool-call events + reasoning summary (files: src/buzzllm/llm.py, tests/unit/test_llm_stream_handlers.py). +3. M3: Tool response wiring + input/messages routing in invoke_llm (files: src/buzzllm/llm.py, tests/unit/test_llm_tool_messages.py). +4. M4: Integration test (skipped without API key) + full pytest (files: tests/integration/test_openai_responses_integration.py). + +Acceptance +1. openai-responses can call tools and continue via previous_response_id. +2. Streaming emits output_text and tool_call events for responses API. +3. Request args honor think/temperature/max_tokens for responses. +4. uv run pytest passes; integration test skipped without OPENAI_API_KEY. + +# Plan: Structured Outputs (OpenAI response_format + Anthropic output_config) + +1. Feature name: Structured Outputs (OpenAI response_format + Anthropic output_config) +2. Objective: Add JSON-structured output via provider-native API parameters with no system prompt changes. +3. Context/problem: output is free-form text; no schema validation or JSON-only response mode. +4. Non-goals: response repair loop, new CLI flags, changes to openai-responses flow, prompt edits. +5. Success criteria: openai-chat uses response_format; anthropic/vertex uses output_config.format; output_structured event emitted; JSON printed once in non-SSE; tests pass. +6. Constraints: no new deps; no new try/except; ASCII only; keep streaming logic intact. + +Links +1. ARCH.md +2. FLOWS.md +3. API.md +4. DATA.md +5. TESTPLAN.md +6. VERIFIER.md +7. IMPLEMENTER.md +8. DEVLOG_CONTRACT.md +9. RISKS.md +10. CHECKLIST.md +11. FEEDBACK.md + +Milestones +1. M1: Add structured output config to LLMOptions and request args (file: src/buzzllm/llm.py; tests: tests/unit/test_llm_request_args.py). +2. M2: Collect output_text into buffer and emit output_structured at end (file: src/buzzllm/llm.py; tests: tests/unit/test_llm_stream_handlers.py). +3. M3: Add structured-output unit tests (files: tests/unit/test_llm_structured_output.py). + +Acceptance +1. Structured output uses response_format (OpenAI) and output_config.format (Anthropic/Vertex). +2. No system prompt changes when structured output is enabled. +3. SSE includes output_structured; non-SSE prints JSON once at end. +4. uv run pytest passes. diff --git a/plans/RISKS.md b/plans/RISKS.md new file mode 100644 index 0000000..4522e5a --- /dev/null +++ b/plans/RISKS.md @@ -0,0 +1,32 @@ +# Risks + +1. SSE parsing misses content due to event ordering. + 1. Mitigation: parse by event type; unit tests cover mixed events. +2. Tool name mismatch between catalog and schema. + 1. Mitigation: single catalog; validate tool_subset; unit tests. +3. Subprocess overhead and flaky IO. + 1. Mitigation: minimal stdout parsing; integration test with mock stdout. +4. Recursive subagents create infinite loops. + 1. Mitigation: optional max depth guard in call_subagent. + +Acceptance +1. Tests cover SSE parsing and allowlist enforcement. + +## OpenAI Responses Risks + +1. Responses streaming event schema mismatch with parser. + 1. Mitigation: add unit fixtures for output_item.added/tool_call_arguments.delta/output_item.done; adjust parser to exact fields. +2. previous_response_id not propagated, causing tool loop to stall. + 1. Mitigation: store last response id on response.created and assert in unit tests. +3. Tool result input shape rejected by API. + 1. Mitigation: keep tool_result items minimal; integration test validates end-to-end. + +Acceptance +1. Tool-call parsing covered by unit tests and integration test (when API key is set). + +## Structured Outputs Risks + +1. Partial JSON streamed to stdout and parsed by callers. + 1. Mitigation: buffer output_text; print JSON once or output_structured only. +2. Schema mismatch between OpenAI and Anthropic formats. + 1. Mitigation: keep provider-specific mapping in request args; unit tests for both. diff --git a/plans/TESTPLAN.md b/plans/TESTPLAN.md new file mode 100644 index 0000000..1308919 --- /dev/null +++ b/plans/TESTPLAN.md @@ -0,0 +1,66 @@ +# Test Plan + +1. Unit tests + 1. tests/unit/test_subagent_sse_parser.py + 1. Parses SSE with mixed events; concatenates only output_text. + 2. Handles multiple output_text chunks. + 2. tests/unit/test_tool_catalog.py + 1. Accepts valid tool_subset names. + 2. Rejects unknown tool names with error string. + 3. tests/unit/test_subagent_payload.py + 1. Payload builder preserves model/provider/url/api_key_name/think/temperature/max_tokens. +2. Integration tests + 1. tests/integration/test_subagent_runner.py + 1. Mock subprocess stdout to SSE fixture; raw_call_subagent returns expected output. + +Fixtures/mocks +1. SSE fixture string with output_text, tool_call, reasoning_content events. +2. Mock subprocess.Popen with controllable stdout/returncode. + +Acceptance +1. New tests pass without API keys. +2. No new integration tests require external services. + +## OpenAI Responses Test Plan + +1. Unit tests + 1. tests/unit/test_llm_request_args.py + 1. input list uses role=user + input_text content. + 2. tools + tool_choice=auto are included when opts.tools is set. + 3. reasoning effort toggles with opts.think. + 4. max_output_tokens + temperature set from opts. + 2. tests/unit/test_llm_stream_handlers.py + 1. response.output_item.added(tool_call) creates ToolCall with id/name. + 2. response.tool_call_arguments.delta appends arguments. + 3. response.output_item.done(tool_call) fills arguments if missing. + 3. tests/unit/test_llm_tool_messages.py + 1. tool_call_response_to_openai_responses_messages sets previous_response_id. + 2. tool_result input items are constructed correctly. +2. Integration tests (skipped without OPENAI_API_KEY) + 1. tests/integration/test_openai_responses_integration.py + 1. simple responses output_text flow. + 2. tool call + tool result continuation (single tool) using a tiny stub tool. + +Fixtures/mocks +1. JSON lines for response.created, output_item.added(tool_call), tool_call_arguments.delta, output_text.delta. +2. Stub tool function for tool_result mapping. + +Acceptance +1. Unit tests cover tool-call parsing and continuation request shape. +2. Integration test is skipped without OPENAI_API_KEY. + +## Structured Outputs Test Plan + +1. Unit tests + 1. tests/unit/test_llm_request_args.py + 1. openai-chat response_format json_schema/json_object set when output_mode is enabled. + 2. anthropic/vertex output_config.format set when output_mode is enabled. + 2. tests/unit/test_llm_stream_handlers.py + 1. output_text is buffered and not printed when structured output is enabled. + 2. output_structured emitted once on block_end. + 3. tests/unit/test_llm_structured_output.py + 1. Non-SSE stdout contains JSON once and DONE marker. + 2. SSE includes output_structured event only. + +Acceptance +1. Structured output tests pass without API keys. diff --git a/plans/VERIFIER.md b/plans/VERIFIER.md new file mode 100644 index 0000000..dd3b277 --- /dev/null +++ b/plans/VERIFIER.md @@ -0,0 +1,16 @@ +# Verifier + +1. Command chain: uv run pytest +2. Pass criteria: exit code 0; no failing tests. +3. Anti-thrash rule: if the same failure persists for 3 iterations, rollback to a prior checkpoint or adjust plan; record in devlogs.md. + +## OpenAI Responses Verifier + +1. Command chain: uv run pytest +2. Pass criteria: exit code 0; integration tests are skipped without OPENAI_API_KEY. +3. Anti-thrash rule: if the same failure persists for 3 iterations, rollback or adjust plan; record in devlogs.md. + +## Structured Outputs Verifier Notes + +1. Expected stdout (non-SSE) includes one JSON blob and DONE marker. +2. Expected SSE includes output_structured event and response_end; no output_text deltas. From 933e63094c966990221b4ac32b5e0fefaaf4db13 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:31:12 -0500 Subject: [PATCH 02/17] new deps --- uv.lock | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 40a2c81..34cc4b9 100644 --- a/uv.lock +++ b/uv.lock @@ -279,12 +279,15 @@ wheels = [ [[package]] name = "buzzllm" -version = "0.2.3" +version = "0.2.6" source = { virtual = "." } dependencies = [ + { name = "beautifulsoup4" }, { name = "crawl4ai" }, { name = "docker" }, + { name = "httpx" }, { name = "loguru" }, + { name = "pydantic" }, { name = "requests" }, { name = "tenacity" }, ] @@ -307,9 +310,12 @@ dev = [ [package.metadata] requires-dist = [ + { name = "beautifulsoup4", specifier = ">=4.0.0" }, { name = "crawl4ai" }, { name = "docker" }, + { name = "httpx", specifier = ">=0.28.0" }, { name = "loguru" }, + { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.24.0" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.1.0" }, From 4be24f7c4b4d6829da53025cc8a24235b2e844e3 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:31:21 -0500 Subject: [PATCH 03/17] updated test plan --- plans/TESTPLAN.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plans/TESTPLAN.md b/plans/TESTPLAN.md index 1308919..0d42ed6 100644 --- a/plans/TESTPLAN.md +++ b/plans/TESTPLAN.md @@ -40,6 +40,8 @@ Acceptance 1. tests/integration/test_openai_responses_integration.py 1. simple responses output_text flow. 2. tool call + tool result continuation (single tool) using a tiny stub tool. + 2. Manual real-API smoke test (requires OPENAI_API_KEY) + 1. Run openai-responses and openai-chat against live OpenAI APIs using OPENAI_API_KEY. Fixtures/mocks 1. JSON lines for response.created, output_item.added(tool_call), tool_call_arguments.delta, output_text.delta. @@ -48,6 +50,7 @@ Fixtures/mocks Acceptance 1. Unit tests cover tool-call parsing and continuation request shape. 2. Integration test is skipped without OPENAI_API_KEY. +3. Real-API smoke test uses OPENAI_API_KEY when available. ## Structured Outputs Test Plan From 2930a758334d36d78bb33a429d26c2c1aedfa579 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:31:41 -0500 Subject: [PATCH 04/17] ralph wiggum loop, will nnot go in final project --- ralphy.py | 883 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 883 insertions(+) create mode 100644 ralphy.py diff --git a/ralphy.py b/ralphy.py new file mode 100644 index 0000000..0145148 --- /dev/null +++ b/ralphy.py @@ -0,0 +1,883 @@ +#!/usr/bin/env python3 +""" +Ralph Wiggum dual-session loop: +- Each iteration spawns TWO *fresh* Claude Code headless runs: + 1) Implementer: applies plans -> edits repo -> updates devlogs.md -> commits + 2) Verifier: blind-to-implementer narrative; reviews artifacts -> returns structured verdict + +Inputs: +- plans/ directory: PLAN.md, IMPLEMENTER.md, VERIFIER.md, TESTPLAN.md, etc. +- devlogs.md at repo root (or specify path) +- (optional) a deterministic verifier shell command you want the driver to run + (verifier runs it directly via Bash tool; recommended). + +Outputs: +- plans/FEEDBACK.md appended each iteration (verdict + logs summary) +- logs/ralph.jsonl (session ids, verdict, timings) +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import textwrap +import time +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional, Tuple, Any, Dict + +from loguru import logger + + +MAX_PLAN_CHARS = 120_000 +MAX_DIFF_CHARS = 80_000 +MAX_DEVLOG_CHARS = 30_000 +MAX_FEEDBACK_CHARS = 25_000 + + +# ------------------------- +# Utilities +# ------------------------- + + +def utc_ts() -> str: + return datetime.now(timezone.utc).isoformat() + + +def run( + cmd: list[str], + cwd: Path, + timeout_s: Optional[int] = None, + env: Optional[dict] = None, +) -> Tuple[int, str, str]: + if timeout_s is not None and timeout_s <= 0: + timeout_s = None + cmd_name = cmd[0] if cmd else "process" + p = subprocess.Popen( + cmd, + cwd=str(cwd), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + bufsize=1, + ) + stdout_lines: list[str] = [] + stderr_lines: list[str] = [] + + def _reader(stream, lines: list[str], stream_name: str) -> None: + for line in iter(stream.readline, ""): + lines.append(line) + logger.info("{} {} | {}", cmd_name, stream_name, line.rstrip("\n")) + stream.close() + + t_out = threading.Thread( + target=_reader, + args=(p.stdout, stdout_lines, "stdout"), + daemon=True, + ) + t_err = threading.Thread( + target=_reader, + args=(p.stderr, stderr_lines, "stderr"), + daemon=True, + ) + t_out.start() + t_err.start() + + timed_out = threading.Event() + timer = None + if timeout_s is not None: + + def _timeout() -> None: + timed_out.set() + p.kill() + + timer = threading.Timer(timeout_s, _timeout) + timer.start() + + rc = p.wait() + if timer: + timer.cancel() + + t_out.join() + t_err.join() + + if timed_out.is_set(): + assert timeout_s is not None + raise subprocess.TimeoutExpired(cmd, float(timeout_s)) + + return rc, "".join(stdout_lines), "".join(stderr_lines) + + +def configure_logger(log_path: Path) -> None: + level = os.getenv("LOGGING_LEVEL", "INFO").upper() + log_path.parent.mkdir(parents=True, exist_ok=True) + logger.remove() + logger.add( + sys.stdout, + level=level, + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | {message}", + ) + logger.add( + log_path, + level=level, + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | {message}", + ) + + +def clamp(s: str, max_chars: int) -> str: + if max_chars <= 0: + return s + if len(s) <= max_chars: + return s + return s[:max_chars] + f"\n\n...[truncated {len(s) - max_chars} chars]..." + + +def read_text(p: Path) -> str: + try: + return p.read_text(encoding="utf-8") + except Exception: + return "" + + +def write_jsonl(path: Path, rec: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + + +def append_md(path: Path, block: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(block) + + +def sha_short(sha: str) -> str: + return sha.strip()[:10] if sha else "" + + +# ------------------------- +# Git / Repo artifact capture +# ------------------------- + + +@dataclass +class RepoArtifacts: + branch: str + head: str + status_porcelain: str + diff_stat: str + diff: str + recent_log: str + devlog_tail: str + + +def git(repo: Path, args: list[str], timeout_s: int) -> Tuple[int, str, str]: + return run(["git"] + args, cwd=repo, timeout_s=timeout_s) + + +def capture_repo_artifacts( + repo: Path, + devlog_path: Path, + timeout_s: int, + max_diff_chars: int, + max_devlog_chars: int, +) -> RepoArtifacts: + _, branch, _ = git(repo, ["rev-parse", "--abbrev-ref", "HEAD"], timeout_s) + _, head, _ = git(repo, ["rev-parse", "HEAD"], timeout_s) + _, status, _ = git(repo, ["status", "--porcelain"], timeout_s) + _, diff_stat, _ = git(repo, ["diff", "--stat"], timeout_s) + _, diff, _ = git(repo, ["diff"], timeout_s) + _, recent_log, _ = git( + repo, ["log", "-n", "25", "--oneline", "--decorate"], timeout_s + ) + + devlog_txt = read_text(devlog_path) if devlog_path.exists() else "" + devlog_tail = clamp(devlog_txt[-max_devlog_chars:], max_devlog_chars) + + return RepoArtifacts( + branch=branch.strip(), + head=head.strip(), + status_porcelain=status.rstrip(), + diff_stat=diff_stat.rstrip(), + diff=clamp(diff.rstrip(), max_diff_chars), + recent_log=recent_log.rstrip(), + devlog_tail=devlog_tail.rstrip(), + ) + + +# ------------------------- +# Plans bundle +# ------------------------- + + +def read_plan_bundle(plans_dir: Path, include_feedback: bool, max_chars: int) -> str: + md_files = sorted(plans_dir.glob("*.md")) + if not include_feedback: + md_files = [p for p in md_files if p.name != "FEEDBACK.md"] + + parts = [] + for p in md_files: + parts.append(f"# FILE: {p.name}\n\n{read_text(p).strip()}\n") + bundle = "\n\n---\n\n".join(parts).strip() + "\n" + return clamp(bundle, max_chars) + + +def tail_feedback(plans_dir: Path, max_chars: int) -> str: + p = plans_dir / "FEEDBACK.md" + if not p.exists(): + return "" + t = read_text(p) + return clamp(t[-max_chars:], max_chars) + + +# ------------------------- +# Claude Code headless invocation +# ------------------------- + + +def claude_headless( + claude_bin: str, + prompt: str, + repo: Path, + timeout_s: int, + model: Optional[str], +) -> Dict[str, Any]: + """ + Uses Claude Code headless: `claude -p "...prompt..."` + Returns parsed JSON when possible; otherwise raw text in "result". + """ + # cmd = [claude_bin, "-p", prompt] + cmd = ["opencode", "run", prompt] + if model: + cmd += ["--model", model] + + rc, out, err = run(cmd, cwd=repo, timeout_s=timeout_s) + data = {"result": out, "_claude_rc": rc, "_stderr": err} + if rc != 0 and not out.strip(): + data["_driver_error"] = f"claude rc={rc}" + data["_stdout"] = out + return data + + try: + parsed = json.loads(out) + except Exception: + return data + if isinstance(parsed, dict): + parsed.setdefault("result", out) + parsed["_claude_rc"] = rc + parsed["_stderr"] = err + return parsed + return data + + +# ------------------------- +# Verifier schema + parsing +# ------------------------- + +VERDICT_SCHEMA = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["PASS", "FAIL", "ROLLBACK_RECOMMENDED"]}, + "blockers": {"type": "array", "items": {"type": "string"}}, + "risky_changes": {"type": "array", "items": {"type": "string"}}, + "next_actions": {"type": "array", "items": {"type": "string"}}, + "rollback_target": {"type": ["string", "null"]}, + "notes": {"type": "string"}, + }, + "required": [ + "verdict", + "blockers", + "risky_changes", + "next_actions", + "rollback_target", + "notes", + ], +} + + +def extract_structured_output(text: str) -> Optional[Dict[str, Any]]: + decoder = json.JSONDecoder() + s = text.strip() + if s: + try: + obj, _ = decoder.raw_decode(s) + if isinstance(obj, dict): + return obj + except json.JSONDecodeError: + pass + for match in re.finditer(r"\{", text): + try: + obj, _ = decoder.raw_decode(text[match.start() :]) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + return obj + return None + + +# ------------------------- +# Prompts +# ------------------------- + + +def build_implementer_prompt( + plan_bundle: str, + feedback_tail: str, + artifacts: RepoArtifacts, + iteration: int, + devlog_path: Path, +) -> str: + return textwrap.dedent(f"""\ + You are the IMPLEMENTER. + + Your job: implement the feature(s) exactly as specified in the PLAN BUNDLE. + You must follow the latest VERIFIER VERDICT in FEEDBACK (if present). + + Hard rules: + - Update {devlog_path.name} at least once this iteration and after any verifier/test run you perform. + - Create meaningful checkpoint commits regularly with message format: `ralph(iter {iteration}): `. + - Do not weaken tests to "make them pass". Fix the code. + - Keep changes focused to the plan. Avoid unrelated refactors. + + What you have: + - PLAN BUNDLE (authoritative spec) + - FEEDBACK tail (verifier verdicts) + - Repo artifacts (git status/diff/log + devlog tail) + + Iteration: {iteration} + + === FEEDBACK (tail) === + {feedback_tail or "(empty)"} + + === REPO ARTIFACTS === + Branch: {artifacts.branch} + HEAD: {sha_short(artifacts.head)} + + git status --porcelain: + {artifacts.status_porcelain or "(clean)"} + + git diff --stat: + {artifacts.diff_stat or "(no diff)"} + + Recent commits: + {artifacts.recent_log or "(none)"} + + devlogs tail: + {artifacts.devlog_tail or "(missing/empty)"} + + === PLAN BUNDLE === + {plan_bundle} + + Now implement. Use the repo tools as needed. When you think you're done, ensure verifier criteria in plans/VERIFIER.md are satisfied. + YOU HAVE MY FULL APPROVAL FOR ALL THE TOOLS YOU WANNA USE AND ALL THE THINGS THAT YOU WANT TO DO. + """).strip() + + +def build_verifier_prompt( + plan_bundle: str, + artifacts: RepoArtifacts, + driver_verify_cmd: Optional[str], + iteration: int, + bias_appendix: Optional[str] = None, +) -> str: + # Verifier is explicitly "blind" to implementer narrative. + appendix_block = "" + if bias_appendix: + appendix_block = ( + f"\n\n=== APPENDIX (may be biased; optional) ===\n{bias_appendix}\n" + ) + + verifier_block = "(no verifier command provided)" + if driver_verify_cmd: + verifier_block = textwrap.dedent(f"""\ + Command to run (verifier must execute via Bash tool): + {driver_verify_cmd} + """).strip() + + return textwrap.dedent(f"""\ + You are the VERIFIER. + + Goal: Decide whether the current repo satisfies the PLAN BUNDLE. + IMPORTANT: Do NOT rely on the implementer's explanation. Base your judgment on artifacts: + - plans/* + - git diff / changed files + - recent commits + - devlogs + - test/lint logs (run the command below if provided) + + If a verifier command is provided below, you MUST run it via the Bash tool and + use its results in your verdict. Include the exit code and any failing output in notes. + + Output MUST be a single JSON object that matches the schema below. + Do NOT wrap the JSON in code fences or add extra text. + - verdict: PASS | FAIL | ROLLBACK_RECOMMENDED + - blockers: bullet strings tied to specific plan sections or failing evidence + - risky_changes: bullet strings of what needs careful review + - next_actions: ordered list (max 5), concrete + - rollback_target: commit hash or null + - notes: brief (include how you decided) + + === OUTPUT JSON SCHEMA === + {json.dumps(VERDICT_SCHEMA, indent=2)} + + Iteration: {iteration} + + === VERIFIER COMMAND (run via Bash tool) === + {verifier_block} + + === REPO ARTIFACTS === + Branch: {artifacts.branch} + HEAD: {sha_short(artifacts.head)} + + git status --porcelain: + {artifacts.status_porcelain or "(clean)"} + + git diff --stat: + {artifacts.diff_stat or "(no diff)"} + + git diff (truncated if large): + {artifacts.diff or "(no diff)"} + + Recent commits: + {artifacts.recent_log or "(none)"} + + devlogs tail: + {artifacts.devlog_tail or "(missing/empty)"} + + === PLAN BUNDLE (authoritative) === + {plan_bundle} + {appendix_block} + + + YOU HAVE MY FULL APPROVAL FOR ALL THE TOOLS YOU WANNA USE AND ALL THE THINGS THAT YOU WANT TO DO. + """).strip() + + +# ------------------------- +# Feedback + loop control +# ------------------------- + + +def feedback_block( + iteration: int, + impl_session: Optional[str], + ver_session: Optional[str], + verdict: Dict[str, Any], +) -> str: + return ( + textwrap.dedent(f"""\ + \n\n## Iteration {iteration} ({utc_ts()}) + - implementer_session: {impl_session or "unknown"} + - verifier_session: {ver_session or "unknown"} + + ### Verifier verdict (structured) + ```json + {json.dumps(verdict, indent=2, ensure_ascii=False)} + ``` + """).rstrip() + + "\n" + ) + + +def maybe_auto_rollback( + repo: Path, rollback_target: Optional[str], timeout_s: int +) -> Tuple[bool, str]: + if not rollback_target: + return False, "no rollback_target" + # Basic safety: accept hex-ish commit ids only + if not re.fullmatch(r"[0-9a-fA-F]{7,40}", rollback_target.strip()): + return False, f"invalid rollback_target format: {rollback_target}" + rc, out, err = git(repo, ["reset", "--hard", rollback_target.strip()], timeout_s) + ok = rc == 0 + msg = (out + "\n" + err).strip() + return ok, msg or ("rollback ok" if ok else "rollback failed") + + +# ------------------------- +# Main +# ------------------------- + + +@dataclass +class Args: + repo: Path + plans: Path + claude_bin: str + model_impl: Optional[str] + model_ver: Optional[str] + max_iters: int + timeout_s: int + devlog_path: Path + driver_verify_cmd: Optional[str] + auto_rollback: bool + logs_jsonl: Path + sleep_s: float + include_feedback_in_plan_bundle: bool + bias_appendix_mode: str # "none" | "append_implementer_result" + + +def parse_args() -> Args: + ap = argparse.ArgumentParser( + description="Ralph dual-session loop: implementer Claude + verifier Claude (fresh sessions)." + ) + + ap.add_argument("--repo", type=Path, default=Path.cwd()) + ap.add_argument("--plans", type=Path, default=Path("plans")) + ap.add_argument("--claude-bin", type=str, default="claude") + + ap.add_argument("--model-impl", type=str, default=None) + ap.add_argument("--model-ver", type=str, default=None) + + ap.add_argument("--max-iters", type=int, default=25) + ap.add_argument( + "--timeout-s", + type=int, + default=1800, + help="Command timeout in seconds (0 disables timeout)", + ) + + ap.add_argument("--devlog", type=Path, default=Path("devlogs.md")) + + ap.add_argument( + "--driver-verify-cmd", + type=str, + default=None, + help='Optional deterministic verifier command run by the VERIFIER. Example: "ruff check . && mypy . && pytest -q"', + ) + + ap.add_argument( + "--auto-rollback", + action="store_true", + help="If set, script will `git reset --hard ` when verifier recommends rollback.", + ) + + ap.add_argument("--logs-jsonl", type=Path, default=Path("logs/ralph.jsonl")) + + ap.add_argument("--sleep-s", type=float, default=1.0) + + ap.add_argument( + "--include-feedback-in-plan-bundle", + action="store_true", + help="If set, plan bundle includes plans/FEEDBACK.md. Usually keep false to avoid bloat.", + ) + ap.add_argument( + "--bias-appendix-mode", + type=str, + default="none", + choices=["none", "append_implementer_result"], + help="Whether to include implementer's natural language result as optional appendix to verifier.", + ) + + ns = ap.parse_args() + + repo = ns.repo.resolve() + plans = (repo / ns.plans).resolve() + devlog = (repo / ns.devlog).resolve() + logs_jsonl = (repo / ns.logs_jsonl).resolve() + + return Args( + repo=repo, + plans=plans, + claude_bin=ns.claude_bin, + model_impl=ns.model_impl, + model_ver=ns.model_ver, + max_iters=ns.max_iters, + timeout_s=ns.timeout_s, + devlog_path=devlog, + driver_verify_cmd=ns.driver_verify_cmd, + auto_rollback=ns.auto_rollback, + logs_jsonl=logs_jsonl, + sleep_s=ns.sleep_s, + include_feedback_in_plan_bundle=ns.include_feedback_in_plan_bundle, + bias_appendix_mode=ns.bias_appendix_mode, + ) + + +def main() -> int: + args = parse_args() + log_path = args.logs_jsonl.with_suffix(".log") + configure_logger(log_path) + logger.info( + "ralphy start repo={} plans={} max_iters={} timeout_s={} verifier_cmd={}", + args.repo, + args.plans, + args.max_iters, + args.timeout_s, + args.driver_verify_cmd or "(none)", + ) + + if not args.plans.exists(): + logger.error("plans dir not found: {}", args.plans) + raise SystemExit(f"plans dir not found: {args.plans} (use --plans)") + + # Ensure FEEDBACK.md exists + feedback_file = args.plans / "FEEDBACK.md" + feedback_file.parent.mkdir(parents=True, exist_ok=True) + if not feedback_file.exists(): + feedback_file.write_text("# FEEDBACK\n\n", encoding="utf-8") + logger.info("created feedback file {}", feedback_file) + + # Ensure devlogs.md exists + args.devlog_path.parent.mkdir(parents=True, exist_ok=True) + if not args.devlog_path.exists(): + args.devlog_path.write_text("# devlogs\n\n", encoding="utf-8") + logger.info("created devlog file {}", args.devlog_path) + + last_head = "" + for i in range(1, args.max_iters + 1): + logger.info("iter {} start", i) + t_iter0 = utc_ts() + + # Capture artifacts BEFORE implementer + pre_art = capture_repo_artifacts( + repo=args.repo, + devlog_path=args.devlog_path, + timeout_s=args.timeout_s, + max_diff_chars=MAX_DIFF_CHARS, + max_devlog_chars=MAX_DEVLOG_CHARS, + ) + pre_status_lines = ( + len(pre_art.status_porcelain.splitlines()) + if pre_art.status_porcelain + else 0 + ) + logger.debug( + "pre artifacts branch={} head={} status_lines={} diff_stat={}", + pre_art.branch, + sha_short(pre_art.head), + pre_status_lines, + pre_art.diff_stat or "(no diff)", + ) + + plan_bundle_impl = read_plan_bundle( + args.plans, + include_feedback=args.include_feedback_in_plan_bundle, + max_chars=MAX_PLAN_CHARS, + ) + feedback_tail_txt = tail_feedback(args.plans, max_chars=MAX_FEEDBACK_CHARS) + logger.debug( + "plan bundle chars={} feedback chars={}", + len(plan_bundle_impl), + len(feedback_tail_txt), + ) + + impl_prompt = build_implementer_prompt( + plan_bundle=plan_bundle_impl, + feedback_tail=feedback_tail_txt, + artifacts=pre_art, + iteration=i, + devlog_path=args.devlog_path, + ) + + logger.info("implementer run start model={}", args.model_impl or "(default)") + impl_json = claude_headless( + claude_bin=args.claude_bin, + prompt=impl_prompt, + repo=args.repo, + timeout_s=args.timeout_s, + model=args.model_impl, + ) + impl_session = impl_json.get("session_id") or impl_json.get("session") or None + impl_result_text = impl_json.get("result", "") + logger.info( + "implementer run done rc={} session={} out_chars={}", + impl_json.get("_claude_rc"), + impl_session or "(none)", + len(impl_result_text), + ) + if impl_json.get("_claude_rc") != 0: + logger.error( + "implementer nonzero rc={} stderr_chars={}", + impl_json.get("_claude_rc"), + len(impl_json.get("_stderr", "")), + ) + if impl_json.get("_driver_error"): + logger.error("implementer driver error: {}", impl_json.get("_driver_error")) + + # Verifier runs deterministic command (optional) + drv_rc = None + if args.driver_verify_cmd: + logger.info( + "verifier command configured (verifier must run): {}", + args.driver_verify_cmd, + ) + else: + logger.debug("verifier command not configured") + + # Capture artifacts AFTER implementer (this is what verifier will judge) + post_art = capture_repo_artifacts( + repo=args.repo, + devlog_path=args.devlog_path, + timeout_s=args.timeout_s, + max_diff_chars=MAX_DIFF_CHARS, + max_devlog_chars=MAX_DEVLOG_CHARS, + ) + post_status_lines = ( + len(post_art.status_porcelain.splitlines()) + if post_art.status_porcelain + else 0 + ) + logger.debug( + "post artifacts branch={} head={} status_lines={} diff_stat={}", + post_art.branch, + sha_short(post_art.head), + post_status_lines, + post_art.diff_stat or "(no diff)", + ) + + # Build verifier prompt (blind to implementer narrative by default) + plan_bundle_ver = read_plan_bundle( + args.plans, include_feedback=False, max_chars=MAX_PLAN_CHARS + ) + appendix = ( + impl_result_text + if args.bias_appendix_mode == "append_implementer_result" + else None + ) + + ver_prompt = build_verifier_prompt( + plan_bundle=plan_bundle_ver, + artifacts=post_art, + driver_verify_cmd=args.driver_verify_cmd, + iteration=i, + bias_appendix=appendix, + ) + + logger.info("verifier run start model={}", args.model_ver or "(default)") + ver_json = claude_headless( + claude_bin=args.claude_bin, + prompt=ver_prompt, + repo=args.repo, + timeout_s=args.timeout_s, + model=args.model_ver, + ) + ver_session = ver_json.get("session_id") or ver_json.get("session") or None + logger.info( + "verifier run done rc={} out_chars={} err_chars={}", + ver_json.get("_claude_rc"), + len(ver_json.get("result", "")), + len(ver_json.get("_stderr", "")), + ) + if ver_json.get("_claude_rc") != 0: + logger.error("verifier nonzero rc={}", ver_json.get("_claude_rc")) + + verdict = extract_structured_output(ver_json.get("result", "")) + if verdict is None: + raw_error = ( + ver_json.get("result", "") or ver_json.get("_stderr", "") + ).strip() + error_text = clamp(raw_error, 500) if raw_error else "" + if ver_json.get("_claude_rc") != 0 and error_text: + logger.error("verifier run failed; using synthesized verdict") + verdict = { + "verdict": "FAIL", + "blockers": [f"Verifier run failed: {error_text}"], + "risky_changes": [], + "next_actions": [ + "Fix verifier runner/model configuration, then rerun." + ], + "rollback_target": None, + "notes": "Driver synthesized verdict from verifier error output.", + } + else: + logger.error("verifier output invalid JSON; using fallback verdict") + verdict = { + "verdict": "FAIL", + "blockers": [ + "Verifier did not return valid JSON; check logs/ralph.jsonl" + ], + "risky_changes": [], + "next_actions": ["Fix verifier output format/schema, then rerun."], + "rollback_target": None, + "notes": "Driver fallback verdict.", + } + + # Append verdict to plans/FEEDBACK.md (this is the ONLY narrative implementer should read next) + append_md(feedback_file, feedback_block(i, impl_session, ver_session, verdict)) + logger.debug("feedback appended") + + # Optional auto rollback + rollback_did = False + rollback_msg = "" + if args.auto_rollback and verdict.get("verdict") == "ROLLBACK_RECOMMENDED": + logger.warning("auto rollback recommended") + ok, msg = maybe_auto_rollback( + args.repo, verdict.get("rollback_target"), args.timeout_s + ) + rollback_did = ok + rollback_msg = msg + append_md( + feedback_file, f"\n- auto_rollback: {rollback_did} ({rollback_msg})\n" + ) + if rollback_did: + logger.info("auto rollback succeeded: {}", rollback_msg) + else: + logger.error("auto rollback failed: {}", rollback_msg) + + # Decide done + done_by_verdict = verdict.get("verdict") == "PASS" + done = done_by_verdict + + t_iter1 = utc_ts() + + # Log jsonl + head_now = post_art.head + rec = { + "ts_start": t_iter0, + "ts_end": t_iter1, + "iter": i, + "branch": post_art.branch, + "head": head_now, + "impl_session": impl_session, + "ver_session": ver_session, + "impl_rc": impl_json.get("_claude_rc"), + "ver_rc": ver_json.get("_claude_rc"), + "driver_verify_cmd": args.driver_verify_cmd, + "driver_verify_rc": drv_rc, + "verdict": verdict, + "auto_rollback": rollback_did, + "auto_rollback_msg": rollback_msg, + } + write_jsonl(args.logs_jsonl, rec) + logger.debug("wrote jsonl log record") + logger.info( + "iter {} verdict={} head={} driver_rc={} done={}", + i, + verdict.get("verdict"), + sha_short(head_now), + drv_rc, + done, + ) + + if done: + logger.info("done PASS stopping") + return 0 + + # If no repo progress for many iters, the verifier should recommend rollback, + # but we can at least detect "stuck head". + if last_head and head_now == last_head: + logger.warning( + "HEAD unchanged since last iteration ({}), might be stuck", + sha_short(head_now), + ) + last_head = head_now + + time.sleep(args.sleep_s) + + logger.warning("max iterations hit") + return 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception: + logger.exception("unhandled exception") + raise From e4e83a4e5cbc02d8bd1e6ac9724a327d7f496acc Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:31:56 -0500 Subject: [PATCH 05/17] updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7cba0ac..2e34f41 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ wheels/ dev_scratch .dingllm/files.txt llm.md +.llm.md From 82170a530a04446498e44aa88c717f420ccd8e16 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:34:41 -0500 Subject: [PATCH 06/17] ralph(iter 1): M1 tool catalog --- src/buzzllm/tools/catalog.py | 32 ++++++++++++++++++++++++++++++++ src/buzzllm/tools/utils.py | 22 ++++++++++++++++++++++ tests/unit/test_tool_catalog.py | 20 ++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 src/buzzllm/tools/catalog.py create mode 100644 tests/unit/test_tool_catalog.py diff --git a/src/buzzllm/tools/catalog.py b/src/buzzllm/tools/catalog.py new file mode 100644 index 0000000..a0d91a5 --- /dev/null +++ b/src/buzzllm/tools/catalog.py @@ -0,0 +1,32 @@ +from . import websearch, codesearch, pythonexec + + +TOOL_CATALOG = { + "search_web": { + "callable": websearch.search_web, + "desc": "", + }, + "scrape_webpage": { + "callable": websearch.scrape_webpage, + "desc": "", + }, + "bash_find": { + "callable": codesearch.bash_find, + "desc": codesearch.bash_find_tool_desc, + }, + "bash_ripgrep": { + "callable": codesearch.bash_ripgrep, + "desc": codesearch.bash_ripgrep_tool_desc, + }, + "bash_read": { + "callable": codesearch.bash_read, + "desc": "", + }, + "python_execute": { + "callable": pythonexec.python_execute, + "desc": "", + }, +} + + +TOOL_NAMES = set(TOOL_CATALOG.keys()) diff --git a/src/buzzllm/tools/utils.py b/src/buzzllm/tools/utils.py index c90fdcf..578fa5f 100644 --- a/src/buzzllm/tools/utils.py +++ b/src/buzzllm/tools/utils.py @@ -8,6 +8,28 @@ def add_tool(func: Callable): AVAILABLE_TOOLS[func.__name__] = func +def build_tool_schemas( + tool_subset: list[str], callable_to_schema: Callable +) -> list[dict]: + from .catalog import TOOL_CATALOG + + schemas = [] + for tool_name in tool_subset: + if tool_name not in TOOL_CATALOG: + raise ValueError(f"Unknown tool name: {tool_name}") + tool_entry = TOOL_CATALOG[tool_name] + tool_func = tool_entry["callable"] + tool_desc = tool_entry.get("desc", "") + + add_tool(tool_func) + if tool_desc: + schemas.append(callable_to_schema(tool_func, tool_desc)) + else: + schemas.append(callable_to_schema(tool_func)) + + return schemas + + def callable_to_openai_schema(func, desc: str = ""): name = func.__name__ description = desc or inspect.getdoc(func) diff --git a/tests/unit/test_tool_catalog.py b/tests/unit/test_tool_catalog.py new file mode 100644 index 0000000..9534353 --- /dev/null +++ b/tests/unit/test_tool_catalog.py @@ -0,0 +1,20 @@ +import pytest + +from buzzllm.tools import utils + + +class TestBuildToolSchemas: + def test_build_tool_schemas_with_valid_subset(self): + schemas = utils.build_tool_schemas( + ["search_web", "bash_find"], utils.callable_to_openai_schema + ) + + assert len(schemas) == 2 + assert schemas[0]["function"]["name"] == "search_web" + assert schemas[1]["function"]["name"] == "bash_find" + assert "search_web" in utils.AVAILABLE_TOOLS + assert "bash_find" in utils.AVAILABLE_TOOLS + + def test_build_tool_schemas_rejects_unknown_tool(self): + with pytest.raises(ValueError, match="Unknown tool name"): + utils.build_tool_schemas(["unknown_tool"], utils.callable_to_openai_schema) From 6d49c1a3d76633357e241256eb95810afc37a4fb Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:34:56 -0500 Subject: [PATCH 07/17] ralph(iter 1): M1 devlog --- devlogs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devlogs.md b/devlogs.md index da1c133..3f8a4d1 100644 --- a/devlogs.md +++ b/devlogs.md @@ -53,3 +53,13 @@ Add `--brief/-b` flag to hide intermediate tool calls/results, showing only fina ### Use Case Useful when using buzzllm as a Claude Code skill - prevents flooding context with intermediate steps. + +## 2026-02-15: Iter 1 M1 Tool Catalog + +Timestamp: 2026-02-15 +Goal: Add tool catalog + helper for tool subset schemas. +Changes: Added tool catalog mapping and schema builder; added unit tests for tool subset validation. +Commands+Results: (none) +Decisions: Keep tool catalog as a single mapping with per-tool descriptions. +Next: Implement subagent runner + SSE parser. +Checkpoint: 82170a5 From a2db6f5fbd16a291cf8ad3bc6414152eae8493f8 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:35:45 -0500 Subject: [PATCH 08/17] ralph(iter 1): M2 subagent runner --- src/buzzllm/subagent.py | 23 +++++++ src/buzzllm/subagent_runner.py | 93 ++++++++++++++++++++++++++ tests/unit/test_subagent_sse_parser.py | 22 ++++++ 3 files changed, 138 insertions(+) create mode 100644 src/buzzllm/subagent.py create mode 100644 src/buzzllm/subagent_runner.py create mode 100644 tests/unit/test_subagent_sse_parser.py diff --git a/src/buzzllm/subagent.py b/src/buzzllm/subagent.py new file mode 100644 index 0000000..c84e836 --- /dev/null +++ b/src/buzzllm/subagent.py @@ -0,0 +1,23 @@ +import json +from typing import Iterable + + +def parse_sse_output_text(lines: Iterable[str]) -> str: + output_chunks = [] + current_event = "" + + for raw_line in lines: + line = raw_line.strip() + if not line: + continue + if line.startswith("event: "): + current_event = line[len("event: ") :].strip() + continue + if line.startswith("data: ") and current_event == "output_text": + data_content = line[len("data: ") :] + payload = json.loads(data_content) + delta = payload.get("delta", "") + if delta: + output_chunks.append(delta) + + return "".join(output_chunks) diff --git a/src/buzzllm/subagent_runner.py b/src/buzzllm/subagent_runner.py new file mode 100644 index 0000000..e1d1cb2 --- /dev/null +++ b/src/buzzllm/subagent_runner.py @@ -0,0 +1,93 @@ +import asyncio +import json +import sys + +from .llm import ( + LLMOptions, + invoke_llm, + make_openai_request_args, + handle_openai_stream_response, + tool_call_response_to_openai_messages, + make_openai_responses_request_args, + handle_openai_responses_stream_response, + tool_call_response_to_openai_responses_messages, + make_anthropic_request_args, + handle_anthropic_stream_response, + tool_call_response_to_anthropic_messages, + make_vertexai_anthropic_request_args, +) +from .tools import utils + + +def main() -> None: + payload = json.loads(sys.stdin.read()) + + provider = payload["provider"] + tool_subset = payload.get("tool_subset") or [] + + provider_map = { + "openai-chat": ( + make_openai_request_args, + handle_openai_stream_response, + utils.callable_to_openai_schema, + tool_call_response_to_openai_messages, + ), + "openai-responses": ( + make_openai_responses_request_args, + handle_openai_responses_stream_response, + utils.callable_to_openai_schema, + tool_call_response_to_openai_responses_messages, + ), + "anthropic": ( + make_anthropic_request_args, + handle_anthropic_stream_response, + utils.callable_to_anthropic_schema, + tool_call_response_to_anthropic_messages, + ), + "vertexai-anthropic": ( + make_vertexai_anthropic_request_args, + handle_anthropic_stream_response, + utils.callable_to_anthropic_schema, + tool_call_response_to_anthropic_messages, + ), + } + + ( + make_request_args_fn, + handle_stream_response_fn, + callable_to_schema, + add_tool_response, + ) = provider_map[provider] + + tools = None + if tool_subset: + tools = utils.build_tool_schemas(tool_subset, callable_to_schema) + + opts = LLMOptions( + model=payload["model"], + url=payload["url"], + api_key_name=payload.get("api_key_name"), + max_tokens=payload.get("max_tokens"), + temperature=payload.get("temperature", 0.8), + think=payload.get("think", False), + tools=tools, + ) + + system_prompt = payload.get("system_prompt") or "" + + asyncio.run( + invoke_llm( + opts, + payload["prompt"], + system_prompt, + make_request_args_fn, + handle_stream_response_fn, + add_tool_response, + sse=True, + brief=False, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_subagent_sse_parser.py b/tests/unit/test_subagent_sse_parser.py new file mode 100644 index 0000000..91cf724 --- /dev/null +++ b/tests/unit/test_subagent_sse_parser.py @@ -0,0 +1,22 @@ +from buzzllm.subagent import parse_sse_output_text + + +class TestSubagentSseParser: + def test_parses_only_output_text(self): + sse_lines = [ + "event: output_text", + 'data: {"id":"","delta":"Hello ","type":"output_text"}', + "", + "event: tool_call", + 'data: {"id":"","delta":"ignored","type":"tool_call"}', + "", + "event: output_text", + 'data: {"id":"","delta":"world","type":"output_text"}', + "", + "event: reasoning_content", + 'data: {"id":"","delta":"ignored","type":"reasoning_content"}', + ] + + output = parse_sse_output_text(sse_lines) + + assert output == "Hello world" From 9def8d74d8a1758846cbe2b8e0d91e003e65f3a3 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:35:54 -0500 Subject: [PATCH 09/17] ralph(iter 1): M2 devlog --- devlogs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devlogs.md b/devlogs.md index 3f8a4d1..f348ce6 100644 --- a/devlogs.md +++ b/devlogs.md @@ -63,3 +63,13 @@ Commands+Results: (none) Decisions: Keep tool catalog as a single mapping with per-tool descriptions. Next: Implement subagent runner + SSE parser. Checkpoint: 82170a5 + +## 2026-02-15: Iter 1 M2 Subagent Runner + +Timestamp: 2026-02-15 +Goal: Add subagent runner + SSE output parser. +Changes: Added subagent runner entrypoint and SSE output_text parser; added unit test for mixed SSE events. +Commands+Results: (none) +Decisions: Keep SSE parsing minimal and output_text-only for subagent output. +Next: Add call_subagent tool and main wiring. +Checkpoint: a2db6f5 From 6587e1f2a6fadb95a559a4f2f14643a55dcf5a14 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:37:02 -0500 Subject: [PATCH 10/17] ralph(iter 1): M3 call_subagent tool --- src/buzzllm/main.py | 19 ++++- src/buzzllm/subagent.py | 84 ++++++++++++++++++++++- tests/integration/test_subagent_runner.py | 51 ++++++++++++++ tests/unit/test_subagent_payload.py | 54 +++++++++++++++ 4 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_subagent_runner.py create mode 100644 tests/unit/test_subagent_payload.py diff --git a/src/buzzllm/main.py b/src/buzzllm/main.py index de663ba..905763c 100644 --- a/src/buzzllm/main.py +++ b/src/buzzllm/main.py @@ -16,7 +16,7 @@ ) from .prompts import prompts from .tools import utils, websearch, codesearch, pythonexec - +from . import subagent def parse_args(): @@ -52,7 +52,10 @@ def parse_args(): "-S", "--sse", action="store_true", help="Enable SSE mode for printing" ) parser.add_argument( - "-b", "--brief", action="store_true", help="Only print final output, hide tool calls and results" + "-b", + "--brief", + action="store_true", + help="Only print final output, hide tool calls and results", ) return parser.parse_args() @@ -142,6 +145,18 @@ async def chat( callable_to_schema(utils.AVAILABLE_TOOLS["python_execute"]), ] + if tools is not None: + subagent.configure_subagent_context( + model=model, + provider=provider, + url=url, + api_key_name=api_key_name, + think=think, + temperature=temperature, + max_tokens=max_tokens, + ) + utils.add_tool(subagent.call_subagent) + tools.append(callable_to_schema(utils.AVAILABLE_TOOLS["call_subagent"])) # Create LLM options opts = LLMOptions( diff --git a/src/buzzllm/subagent.py b/src/buzzllm/subagent.py index c84e836..4c1af91 100644 --- a/src/buzzllm/subagent.py +++ b/src/buzzllm/subagent.py @@ -1,5 +1,9 @@ import json -from typing import Iterable +import subprocess +import sys +from typing import Iterable, Optional + +from .tools.catalog import TOOL_NAMES def parse_sse_output_text(lines: Iterable[str]) -> str: @@ -21,3 +25,81 @@ def parse_sse_output_text(lines: Iterable[str]) -> str: output_chunks.append(delta) return "".join(output_chunks) + + +_SUBAGENT_CONTEXT: dict[str, object] = {} + + +def configure_subagent_context( + model: str, + provider: str, + url: str, + api_key_name: Optional[str], + think: bool, + temperature: float, + max_tokens: Optional[int], +) -> None: + _SUBAGENT_CONTEXT.clear() + _SUBAGENT_CONTEXT.update( + { + "model": model, + "provider": provider, + "url": url, + "api_key_name": api_key_name, + "think": think, + "temperature": temperature, + "max_tokens": max_tokens, + } + ) + + +def build_subagent_payload( + prompt: str, + system_prompt: Optional[str], + tool_subset: Optional[list[str]], +) -> dict: + payload = { + "model": _SUBAGENT_CONTEXT["model"], + "provider": _SUBAGENT_CONTEXT["provider"], + "url": _SUBAGENT_CONTEXT["url"], + "api_key_name": _SUBAGENT_CONTEXT["api_key_name"], + "prompt": prompt, + "system_prompt": system_prompt or "", + "tool_subset": tool_subset or [], + "think": _SUBAGENT_CONTEXT["think"], + "temperature": _SUBAGENT_CONTEXT["temperature"], + "max_tokens": _SUBAGENT_CONTEXT["max_tokens"], + } + return payload + + +def raw_call_subagent(payload: dict) -> str: + result = subprocess.run( + [sys.executable, "-m", "buzzllm.subagent_runner"], + input=json.dumps(payload), + text=True, + capture_output=True, + ) + + if result.returncode != 0: + return f"Error: Subagent exited with code {result.returncode}" + + return parse_sse_output_text(result.stdout.splitlines()) + + +def call_subagent( + prompt: str, + system_prompt: Optional[str] = None, + tool_subset: Optional[list[str]] = None, +) -> str: + """Run a subagent with an optional tool subset.""" + if not _SUBAGENT_CONTEXT: + return "Error: Subagent context is not configured" + + requested_tools = tool_subset or [] + unknown_tools = sorted(set(requested_tools) - TOOL_NAMES) + if unknown_tools: + return f"Error: Unknown tool names: {', '.join(unknown_tools)}" + + payload = build_subagent_payload(prompt, system_prompt, requested_tools) + return raw_call_subagent(payload) diff --git a/tests/integration/test_subagent_runner.py b/tests/integration/test_subagent_runner.py new file mode 100644 index 0000000..820f1be --- /dev/null +++ b/tests/integration/test_subagent_runner.py @@ -0,0 +1,51 @@ +import json + +from buzzllm import subagent + + +class TestSubagentRunner: + def test_call_subagent_parses_output_text(self, monkeypatch): + subagent.configure_subagent_context( + model="gpt-4o-mini", + provider="openai-chat", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + think=False, + temperature=0.8, + max_tokens=1024, + ) + + sse_text = "\n".join( + [ + "event: output_text", + 'data: {\\"id\\":\\"\\",\\"delta\\":\\"Hello \\",\\"type\\":\\"output_text\\"}', + "", + "event: output_text", + 'data: {\\"id\\":\\"\\",\\"delta\\":\\"world\\",\\"type\\":\\"output_text\\"}', + "", + ] + ) + + captured = {} + + def fake_run(cmd, input, text, capture_output): + captured["input"] = input + + class Result: + returncode = 0 + stdout = sse_text + stderr = "" + + return Result() + + monkeypatch.setattr(subagent.subprocess, "run", fake_run) + + output = subagent.call_subagent( + prompt="Hi", + system_prompt="System", + tool_subset=["search_web"], + ) + + payload = json.loads(captured["input"]) + assert payload["tool_subset"] == ["search_web"] + assert output == "Hello world" diff --git a/tests/unit/test_subagent_payload.py b/tests/unit/test_subagent_payload.py new file mode 100644 index 0000000..6d91b68 --- /dev/null +++ b/tests/unit/test_subagent_payload.py @@ -0,0 +1,54 @@ +from unittest.mock import patch + +from buzzllm import subagent + + +class TestSubagentPayload: + def test_payload_includes_context_fields(self): + subagent.configure_subagent_context( + model="gpt-4o-mini", + provider="openai-chat", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + think=True, + temperature=0.4, + max_tokens=1234, + ) + + payload = subagent.build_subagent_payload( + prompt="Hello", + system_prompt="System", + tool_subset=["search_web"], + ) + + assert payload["model"] == "gpt-4o-mini" + assert payload["provider"] == "openai-chat" + assert payload["url"] == "https://api.openai.com/v1/chat/completions" + assert payload["api_key_name"] == "OPENAI_API_KEY" + assert payload["prompt"] == "Hello" + assert payload["system_prompt"] == "System" + assert payload["tool_subset"] == ["search_web"] + assert payload["think"] is True + assert payload["temperature"] == 0.4 + assert payload["max_tokens"] == 1234 + + def test_unknown_tool_subset_returns_error(self): + subagent.configure_subagent_context( + model="gpt-4o-mini", + provider="openai-chat", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + think=False, + temperature=0.8, + max_tokens=1024, + ) + + with patch("buzzllm.subagent.subprocess.run") as mock_run: + result = subagent.call_subagent( + prompt="Hello", + system_prompt="System", + tool_subset=["unknown_tool"], + ) + + assert result.startswith("Error: Unknown tool names") + mock_run.assert_not_called() From 101be192d642482306082e69445c208cbb443cfd Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:37:12 -0500 Subject: [PATCH 11/17] ralph(iter 1): M3 devlog --- devlogs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devlogs.md b/devlogs.md index f348ce6..1891eab 100644 --- a/devlogs.md +++ b/devlogs.md @@ -73,3 +73,13 @@ Commands+Results: (none) Decisions: Keep SSE parsing minimal and output_text-only for subagent output. Next: Add call_subagent tool and main wiring. Checkpoint: a2db6f5 + +## 2026-02-15: Iter 1 M3 call_subagent Tool + +Timestamp: 2026-02-15 +Goal: Add call_subagent tool and wire into main tool list. +Changes: Added subagent context/config + payload builder, call_subagent execution, and integration/unit tests. +Commands+Results: (none) +Decisions: Validate tool_subset against catalog before spawning subprocess. +Next: Implement openai-responses tooling updates. +Checkpoint: 6587e1f From d9e7781f28c73e5817b4f5bd08dbd7afc57d7bd8 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:49:17 -0500 Subject: [PATCH 12/17] ralph(iter 1): responses + structured outputs --- src/buzzllm/llm.py | 159 +++++++++++++++--- tests/conftest.py | 30 +++- .../test_openai_responses_integration.py | 91 ++++++++++ tests/unit/test_llm_dataclasses.py | 10 ++ tests/unit/test_llm_request_args.py | 94 ++++++++++- tests/unit/test_llm_stream_handlers.py | 46 ++++- tests/unit/test_llm_structured_output.py | 96 +++++++++++ tests/unit/test_llm_tool_messages.py | 57 ++++++- 8 files changed, 534 insertions(+), 49 deletions(-) create mode 100644 tests/integration/test_openai_responses_integration.py create mode 100644 tests/unit/test_llm_structured_output.py diff --git a/src/buzzllm/llm.py b/src/buzzllm/llm.py index c444d8c..bbc70ea 100644 --- a/src/buzzllm/llm.py +++ b/src/buzzllm/llm.py @@ -21,6 +21,8 @@ class LLMOptions: think: bool = False tools: Optional[list[dict]] = None max_infer_iters: int = 10 + output_mode: Optional[Literal["json_schema", "json_object"]] = None + output_schema: Optional[dict] = None @dataclass @@ -41,9 +43,9 @@ class StreamResponse: "tool_result", "block_end", "response_end", + "output_structured", ] - def to_json(self): return json.dumps(dataclasses.asdict(self)) @@ -68,6 +70,7 @@ async def execute(self, func: Callable): TOOL_CALLS: dict[str, ToolCall] = {} current_tool_call_id: str = "" +last_openai_response_id: str = "" async def run_tools(): @@ -93,11 +96,13 @@ async def invoke_llm( """Invoke LLM with streaming response, printing StreamResponse objects to stdout as JSON""" request_args = make_request_args(opts, prompt, system_prompt) - messages = request_args.data.get("messages", []) + messages = request_args.data.get("messages") + inputs = request_args.data.get("input") try: while True: message_started = False + structured_output_buffer = "" if opts.output_mode else None # Make streaming request response = requests.post( opts.url, @@ -120,10 +125,24 @@ async def invoke_llm( continue if stream_response.type == "response_start": message_started = True + if opts.output_mode and stream_response.type == "output_text": + structured_output_buffer = ( + structured_output_buffer or "" + ) + stream_response.delta + continue + if opts.output_mode and stream_response.type == "block_end": + continue print_to_stdout(stream_response, sse, brief) # Perform tool calls if not TOOL_CALLS: + if opts.output_mode: + structured_response = StreamResponse( + id="", + delta=structured_output_buffer or "", + type="output_structured", + ) + print_to_stdout(structured_response, sse, brief) return await run_tools() @@ -134,19 +153,21 @@ async def invoke_llm( result_response = StreamResponse( id=tc.id, delta=f"\n\nTool Result ({tc.name}):\n{str(tc.result)}\n", - type="tool_result" + type="tool_result", ) print_to_stdout(result_response, sse, brief) # Add tool call and response messages - add_tool_response(messages, TOOL_CALLS) - # tool_call_response_to_openai_messages(messages, TOOL_CALLS) - request_args.data["messages"] = messages + if messages is not None: + add_tool_response(messages, TOOL_CALLS) + request_args.data["messages"] = messages + elif inputs is not None: + add_tool_response(request_args, TOOL_CALLS) + inputs = request_args.data.get("input") # Clear tool calls for next iteration TOOL_CALLS.clear() - except Exception as e: print(e) # Print error as StreamResponse @@ -158,11 +179,13 @@ async def invoke_llm( # Cleanup any running containers try: from .tools import pythonexec + pythonexec.cleanup_python_exec() except ImportError: pass - print_to_stdout(StreamResponse(id="", delta="", type="response_end"), sse, brief) - + print_to_stdout( + StreamResponse(id="", delta="", type="response_end"), sse, brief + ) def print_to_stdout(data: StreamResponse, sse: bool, brief: bool = False) -> None: @@ -192,7 +215,6 @@ def print_to_stdout(data: StreamResponse, sse: bool, brief: bool = False) -> Non print(data.delta, end="", flush=True) - # LLM Specific funcs @@ -202,7 +224,16 @@ def print_to_stdout(data: StreamResponse, sse: bool, brief: bool = False) -> Non def make_openai_request_args( opts: LLMOptions, prompt: str, system_prompt: str ) -> RequestArgs: - OPENAI_REASONING_MODELS=['gpt-5.2', 'gpt-5.1', 'gpt-5', 'gpt-5-mini', 'o4-mini', 'o3', 'o3-pro', 'gpt-5-pro'] + OPENAI_REASONING_MODELS = [ + "gpt-5.2", + "gpt-5.1", + "gpt-5", + "gpt-5-mini", + "o4-mini", + "o3", + "o3-pro", + "gpt-5-pro", + ] # json body data = { "messages": [ @@ -214,13 +245,27 @@ def make_openai_request_args( } if opts.model in OPENAI_REASONING_MODELS: - data['messages'][0]['role'] = 'developer' + data["messages"][0]["role"] = "developer" data["response_format"] = {"type": "text"} - data["reasoning_effort"] = "none" if opts.model == 'gpt-5.1' and not opts.think else "high" + data["reasoning_effort"] = ( + "none" if opts.model == "gpt-5.1" and not opts.think else "high" + ) else: data["temperature"] = opts.temperature data["max_tokens"] = opts.max_tokens + if opts.output_mode == "json_schema" and opts.output_schema: + data["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "schema": opts.output_schema, + "strict": True, + }, + } + elif opts.output_mode == "json_object": + data["response_format"] = {"type": "json_object"} + if opts.tools: data["tools"] = opts.tools @@ -237,7 +282,6 @@ def make_openai_request_args( def handle_openai_stream_response( line: str, message_started: bool ) -> Generator[StreamResponse | None, None, None]: - if not line.startswith("data: "): yield None data_content = line[len("data: ") :] # Remove 'data: ' prefix @@ -265,7 +309,9 @@ def handle_openai_stream_response( yield StreamResponse(id="", delta=delta["content"], type="output_text") # Handle reasoning content (for o1 models) - if ("reasoning" in delta and delta["reasoning"]) or ('reasoning_content' in delta and delta['reasoning_content']): + if ("reasoning" in delta and delta["reasoning"]) or ( + "reasoning_content" in delta and delta["reasoning_content"] + ): yield StreamResponse( id="", delta=delta["reasoning"], type="reasoning_content" ) @@ -357,6 +403,11 @@ def make_anthropic_request_args( if opts.tools: data["tools"] = opts.tools + if opts.output_mode and opts.output_schema: + data["output_config"] = { + "format": {"type": "json_schema", "schema": opts.output_schema} + } + headers = {"Content-Type": "application/json"} if opts.api_key_name: api_key = os.environ.get(opts.api_key_name, None) @@ -476,20 +527,30 @@ def make_openai_responses_request_args( ) -> RequestArgs: data = { "model": opts.model, - "input": prompt, + "input": [ + { + "role": "user", + "content": [{"type": "input_text", "text": prompt}], + } + ], "stream": True, "store": False, "reasoning": { - "effort": "high", - "summary": "detailed" - } + "effort": "high" if opts.think else "none", + "summary": "detailed" if opts.think else "none", + }, + "temperature": opts.temperature, } + if opts.max_tokens: + data["max_output_tokens"] = opts.max_tokens + if system_prompt: data["instructions"] = system_prompt if opts.tools: - raise NotImplementedError("Tools with OpenAI Responses API has not yet been implemented") + data["tools"] = opts.tools + data["tool_choice"] = "auto" headers = {"Content-Type": "application/json"} if opts.api_key_name: @@ -507,7 +568,7 @@ def handle_openai_responses_stream_response( yield None return - data_content = line[len("data: "):] + data_content = line[len("data: ") :] try: chunk_data = json.loads(data_content) @@ -516,6 +577,8 @@ def handle_openai_responses_stream_response( # Handle response start if event_type == "response.created": response_id = chunk_data.get("response", {}).get("id", "") + global last_openai_response_id + last_openai_response_id = response_id yield StreamResponse(id=response_id, type="response_start", delta="") # Handle regular text content @@ -528,6 +591,31 @@ def handle_openai_responses_stream_response( delta_text = chunk_data.get("delta", "") yield StreamResponse(id="", delta=delta_text, type="reasoning_content") + elif event_type == "response.output_item.added": + item = chunk_data.get("item", {}) + if item.get("type") == "tool_call": + tool_id = item.get("id", "") + tool_name = item.get("name", "") + TOOL_CALLS[tool_id] = ToolCall( + id=tool_id, name=tool_name, arguments="", executed=False + ) + + elif event_type == "response.tool_call_arguments.delta": + tool_call_id = chunk_data.get("tool_call_id", "") + delta_text = chunk_data.get("delta", "") + if tool_call_id in TOOL_CALLS: + TOOL_CALLS[tool_call_id].arguments += delta_text + if delta_text: + yield StreamResponse(id="", delta=delta_text, type="tool_call") + + elif event_type == "response.output_item.done": + item = chunk_data.get("item", {}) + if item.get("type") == "tool_call": + tool_id = item.get("id", "") + arguments = item.get("arguments") + if tool_id in TOOL_CALLS and arguments: + TOOL_CALLS[tool_id].arguments = arguments + # Handle response completion elif event_type == "response.completed": yield StreamResponse(id="", delta="", type="block_end") @@ -537,11 +625,23 @@ def handle_openai_responses_stream_response( def tool_call_response_to_openai_responses_messages( - messages: list, tool_calls: dict[str, ToolCall] + request_args: RequestArgs, tool_calls: dict[str, ToolCall] ): - # For now, reuse the same logic as OpenAI chat completions - # This may need adjustment based on how Responses API handles conversation state - tool_call_response_to_openai_messages(messages, tool_calls) + if not tool_calls: + return + + request_args.data["previous_response_id"] = last_openai_response_id + tool_results = [] + for tc in tool_calls.values(): + tool_results.append( + { + "type": "tool_result", + "tool_call_id": tc.id, + "content": str(tc.result), + } + ) + + request_args.data["input"] = tool_results # === @@ -568,7 +668,14 @@ def make_vertexai_anthropic_request_args( if opts.tools: data["tools"] = opts.tools + if opts.output_mode and opts.output_schema: + data["output_config"] = { + "format": {"type": "json_schema", "schema": opts.output_schema} + } + headers = {"Content-Type": "application/json"} - api_key = subprocess.run(['gcloud', 'auth', 'print-access-token'], capture_output=True, text=True) + api_key = subprocess.run( + ["gcloud", "auth", "print-access-token"], capture_output=True, text=True + ) headers["Authorization"] = f"Bearer {api_key.stdout.strip()}" return RequestArgs(data=data, headers=headers) diff --git a/tests/conftest.py b/tests/conftest.py index 2db3768..0530256 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,12 @@ # === Skip conditions === + def docker_available() -> bool: """Check if Docker daemon is running and accessible""" try: import docker + client = docker.from_env() client.ping() return True @@ -24,23 +26,21 @@ def anthropic_api_available() -> bool: skip_if_no_docker = pytest.mark.skipif( - not docker_available(), - reason="Docker daemon not available" + not docker_available(), reason="Docker daemon not available" ) skip_if_no_openai = pytest.mark.skipif( - not openai_api_available(), - reason="OPENAI_API_KEY not set" + not openai_api_available(), reason="OPENAI_API_KEY not set" ) skip_if_no_anthropic = pytest.mark.skipif( - not anthropic_api_available(), - reason="ANTHROPIC_API_KEY not set" + not anthropic_api_available(), reason="ANTHROPIC_API_KEY not set" ) # === Reset global state === + @pytest.fixture(autouse=True) def reset_tool_state(): """Reset global tool state between tests""" @@ -50,20 +50,24 @@ def reset_tool_state(): utils.AVAILABLE_TOOLS.clear() llm.TOOL_CALLS.clear() llm.current_tool_call_id = "" + llm.last_openai_response_id = "" yield utils.AVAILABLE_TOOLS.clear() llm.TOOL_CALLS.clear() llm.current_tool_call_id = "" + llm.last_openai_response_id = "" # === LLM fixtures === + @pytest.fixture def sample_llm_options(): """Basic LLMOptions fixture""" from buzzllm.llm import LLMOptions + return LLMOptions( model="gpt-4o-mini", url="https://api.openai.com/v1/chat/completions", @@ -77,22 +81,25 @@ def sample_llm_options(): def sample_tool_calls(): """Sample ToolCall dict for testing message conversion""" from buzzllm.llm import ToolCall + return { "call_1": ToolCall( id="call_1", name="search_web", arguments='{"query": "python tutorial"}', executed=True, - result="Search results..." + result="Search results...", ), } # === Tool schema fixtures === + @pytest.fixture def sample_function_with_docstring(): """Sample function for schema conversion""" + def my_func(name: str, count: int = 10) -> str: """This is a sample function @@ -100,19 +107,23 @@ def my_func(name: str, count: int = 10) -> str: :param count: Number of items """ return f"{name}: {count}" + return my_func @pytest.fixture def sample_function_without_docstring(): """Function without docstring - should raise error""" + def no_doc_func(x: int): pass + return no_doc_func # === Codesearch fixtures === + @pytest.fixture def temp_cwd(tmp_path, monkeypatch): """Set up a temp directory as CWD for codesearch tests""" @@ -121,12 +132,14 @@ def temp_cwd(tmp_path, monkeypatch): (tmp_path / "subdir" / "nested.py").write_text("x = 1\n") import buzzllm.tools.codesearch as codesearch_module + monkeypatch.setattr(codesearch_module, "CWD", tmp_path) return tmp_path # === Websearch mock fixtures === + @pytest.fixture def mock_duckduckgo_html(): """Mock HTML response from DuckDuckGo lite""" @@ -150,7 +163,7 @@ def mock_brave_json(): { "title": "Example Title", "url": "https://example.com", - "description": "Example description" + "description": "Example description", } ] } @@ -159,6 +172,7 @@ def mock_brave_json(): # === Environment fixtures === + @pytest.fixture def env_with_api_keys(monkeypatch): """Set up environment with fake API keys for unit tests""" diff --git a/tests/integration/test_openai_responses_integration.py b/tests/integration/test_openai_responses_integration.py new file mode 100644 index 0000000..7122a0c --- /dev/null +++ b/tests/integration/test_openai_responses_integration.py @@ -0,0 +1,91 @@ +import pytest + +from tests.conftest import skip_if_no_openai + + +@pytest.mark.integration +class TestOpenAIResponsesIntegration: + @skip_if_no_openai + def test_simple_responses_output_text(self): + from buzzllm.llm import ( + LLMOptions, + make_openai_responses_request_args, + handle_openai_responses_stream_response, + ) + import requests + + opts = LLMOptions( + model="o3", + url="https://api.openai.com/v1/responses", + api_key_name="OPENAI_API_KEY", + max_tokens=50, + temperature=0.0, + ) + + request_args = make_openai_responses_request_args( + opts, "Say 'hello' and nothing else.", "You are helpful." + ) + + response = requests.post( + opts.url, + headers=request_args.headers, + json=request_args.data, + stream=True, + timeout=30, + ) + response.raise_for_status() + + collected_text = [] + for line in response.iter_lines(decode_unicode=True): + if not line: + continue + for stream_response in handle_openai_responses_stream_response(line, True): + if stream_response and stream_response.type == "output_text": + collected_text.append(stream_response.delta) + + full_text = "".join(collected_text).lower() + assert "hello" in full_text + + @skip_if_no_openai + @pytest.mark.asyncio + async def test_tool_calling_flow(self): + from buzzllm.llm import ( + LLMOptions, + invoke_llm, + make_openai_responses_request_args, + handle_openai_responses_stream_response, + tool_call_response_to_openai_responses_messages, + ) + from buzzllm.tools import utils + + tool_called = {"count": 0} + + def get_weather(city: str) -> str: + """Get weather for a city""" + tool_called["count"] += 1 + return f"Weather in {city}: Sunny" + + utils.add_tool(get_weather) + tools = [utils.callable_to_openai_schema(utils.AVAILABLE_TOOLS["get_weather"])] + + opts = LLMOptions( + model="o3", + url="https://api.openai.com/v1/responses", + api_key_name="OPENAI_API_KEY", + max_tokens=100, + temperature=0.0, + tools=tools, + ) + + await invoke_llm( + opts, + "You must call get_weather with city=Paris.", + "You are helpful. Use tools when needed.", + make_openai_responses_request_args, + handle_openai_responses_stream_response, + tool_call_response_to_openai_responses_messages, + sse=False, + brief=True, + ) + + assert tool_called["count"] >= 1 diff --git a/tests/unit/test_llm_dataclasses.py b/tests/unit/test_llm_dataclasses.py index 733d8b0..d010996 100644 --- a/tests/unit/test_llm_dataclasses.py +++ b/tests/unit/test_llm_dataclasses.py @@ -16,6 +16,8 @@ def test_default_values(self): assert opts.think is False assert opts.tools is None assert opts.max_infer_iters == 10 + assert opts.output_mode is None + assert opts.output_schema is None def test_custom_values(self): opts = LLMOptions( @@ -27,12 +29,16 @@ def test_custom_values(self): think=True, tools=[{"name": "tool1"}], max_infer_iters=5, + output_mode="json_schema", + output_schema={"type": "object"}, ) assert opts.max_tokens == 4096 assert opts.temperature == 0.5 assert opts.think is True assert opts.tools == [{"name": "tool1"}] assert opts.max_infer_iters == 5 + assert opts.output_mode == "json_schema" + assert opts.output_schema == {"type": "object"} class TestRequestArgs: @@ -60,6 +66,10 @@ def test_tool_call_type(self): resp = StreamResponse(id="", delta="search_web", type="tool_call") assert resp.type == "tool_call" + def test_output_structured_type(self): + resp = StreamResponse(id="", delta="{}", type="output_structured") + assert resp.type == "output_structured" + def test_to_json(self): resp = StreamResponse(id="abc", delta="test", type="output_text") json_str = resp.to_json() diff --git a/tests/unit/test_llm_request_args.py b/tests/unit/test_llm_request_args.py index 71ad730..77bd89d 100644 --- a/tests/unit/test_llm_request_args.py +++ b/tests/unit/test_llm_request_args.py @@ -107,6 +107,32 @@ def test_gpt5_1_with_think_mode(self, env_with_api_keys): assert args.data["reasoning_effort"] == "high" + def test_structured_output_json_schema(self, env_with_api_keys): + schema = {"type": "object", "properties": {"name": {"type": "string"}}} + opts = LLMOptions( + model="gpt-4o-mini", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + output_mode="json_schema", + output_schema=schema, + ) + args = make_openai_request_args(opts, "Hello", "System") + + response_format = args.data["response_format"] + assert response_format["type"] == "json_schema" + assert response_format["json_schema"]["schema"] == schema + + def test_structured_output_json_object(self, env_with_api_keys): + opts = LLMOptions( + model="gpt-4o-mini", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + output_mode="json_object", + ) + args = make_openai_request_args(opts, "Hello", "System") + + assert args.data["response_format"] == {"type": "json_object"} + class TestMakeAnthropicRequestArgs: def test_basic_request_structure(self, env_with_api_keys): @@ -170,6 +196,19 @@ def test_with_tools(self, env_with_api_keys): assert args.data["tools"] == tools + def test_structured_output_config(self, env_with_api_keys): + schema = {"type": "object", "properties": {"a": {"type": "integer"}}} + opts = LLMOptions( + model="claude-sonnet-4-20250514", + url="https://api.anthropic.com/v1/messages", + api_key_name="ANTHROPIC_API_KEY", + output_mode="json_schema", + output_schema=schema, + ) + args = make_anthropic_request_args(opts, "Hello", "System") + + assert args.data["output_config"]["format"]["schema"] == schema + class TestMakeVertexaiAnthropicRequestArgs: @patch("subprocess.run") @@ -215,6 +254,21 @@ def test_thinking_mode(self, mock_run, env_with_api_keys): assert args.data["max_tokens"] == 32000 assert args.data["thinking"]["type"] == "enabled" + @patch("subprocess.run") + def test_structured_output_config(self, mock_run, env_with_api_keys): + mock_run.return_value = MagicMock(stdout="token\n") + schema = {"type": "object", "properties": {"a": {"type": "integer"}}} + + opts = LLMOptions( + model="claude-3-5-sonnet@20240620", + url="https://us-central1-aiplatform.googleapis.com/...", + output_mode="json_schema", + output_schema=schema, + ) + args = make_vertexai_anthropic_request_args(opts, "Hello", "System") + + assert args.data["output_config"]["format"]["schema"] == schema + class TestMakeOpenaiResponsesRequestArgs: def test_basic_request_structure(self, env_with_api_keys): @@ -226,19 +280,49 @@ def test_basic_request_structure(self, env_with_api_keys): args = make_openai_responses_request_args(opts, "Hello", "Instructions") assert args.data["model"] == "o3" - assert args.data["input"] == "Hello" + assert args.data["input"][0]["role"] == "user" + assert args.data["input"][0]["content"][0]["type"] == "input_text" + assert args.data["input"][0]["content"][0]["text"] == "Hello" assert args.data["instructions"] == "Instructions" assert args.data["stream"] is True assert args.data["store"] is False + assert args.data["reasoning"]["effort"] == "none" + assert args.data["reasoning"]["summary"] == "none" + + def test_tools_are_included(self, env_with_api_keys): + tools = [{"type": "function", "function": {"name": "test"}}] + opts = LLMOptions( + model="o3", + url="https://api.openai.com/v1/responses", + api_key_name="OPENAI_API_KEY", + tools=tools, + ) + + args = make_openai_responses_request_args(opts, "Hello", "System") + assert args.data["tools"] == tools + assert args.data["tool_choice"] == "auto" + + def test_reasoning_mode(self, env_with_api_keys): + opts = LLMOptions( + model="o3", + url="https://api.openai.com/v1/responses", + api_key_name="OPENAI_API_KEY", + think=True, + ) + + args = make_openai_responses_request_args(opts, "Hello", "System") assert args.data["reasoning"]["effort"] == "high" + assert args.data["reasoning"]["summary"] == "detailed" - def test_tools_not_implemented(self, env_with_api_keys): + def test_max_output_tokens_and_temperature(self, env_with_api_keys): opts = LLMOptions( model="o3", url="https://api.openai.com/v1/responses", api_key_name="OPENAI_API_KEY", - tools=[{"name": "test"}], + max_tokens=2048, + temperature=0.3, ) - with pytest.raises(NotImplementedError): - make_openai_responses_request_args(opts, "Hello", "System") + args = make_openai_responses_request_args(opts, "Hello", "System") + assert args.data["max_output_tokens"] == 2048 + assert args.data["temperature"] == 0.3 diff --git a/tests/unit/test_llm_stream_handlers.py b/tests/unit/test_llm_stream_handlers.py index 16696dd..9faff3d 100644 --- a/tests/unit/test_llm_stream_handlers.py +++ b/tests/unit/test_llm_stream_handlers.py @@ -6,6 +6,7 @@ handle_anthropic_stream_response, handle_openai_responses_stream_response, TOOL_CALLS, + ToolCall, ) @@ -128,7 +129,10 @@ def test_content_block_delta_partial_json(self): list(handle_anthropic_stream_response(f"data: {json.dumps(start_data)}", True)) # Then send partial JSON - delta_data = {"type": "content_block_delta", "delta": {"partial_json": '{"x": 1}'}} + delta_data = { + "type": "content_block_delta", + "delta": {"partial_json": '{"x": 1}'}, + } results = list( handle_anthropic_stream_response(f"data: {json.dumps(delta_data)}", True) ) @@ -191,6 +195,46 @@ def test_response_completed(self): ends = [r for r in results if r and r.type == "block_end"] assert len(ends) == 1 + def test_tool_call_added(self): + data = { + "type": "response.output_item.added", + "item": {"type": "tool_call", "id": "call_1", "name": "search_web"}, + } + line = f"data: {json.dumps(data)}" + list(handle_openai_responses_stream_response(line, True)) + + assert "call_1" in TOOL_CALLS + assert TOOL_CALLS["call_1"].name == "search_web" + + def test_tool_call_arguments_delta(self): + TOOL_CALLS["call_2"] = ToolCall( + id="call_2", name="tool", arguments="", executed=False + ) + data = { + "type": "response.tool_call_arguments.delta", + "tool_call_id": "call_2", + "delta": '{"q": "test"}', + } + line = f"data: {json.dumps(data)}" + results = list(handle_openai_responses_stream_response(line, True)) + + assert TOOL_CALLS["call_2"].arguments == '{"q": "test"}' + tool_calls = [r for r in results if r and r.type == "tool_call"] + assert len(tool_calls) == 1 + + def test_tool_call_done_sets_arguments(self): + TOOL_CALLS["call_3"] = ToolCall( + id="call_3", name="tool", arguments="", executed=False + ) + data = { + "type": "response.output_item.done", + "item": {"type": "tool_call", "id": "call_3", "arguments": "{}"}, + } + line = f"data: {json.dumps(data)}" + list(handle_openai_responses_stream_response(line, True)) + + assert TOOL_CALLS["call_3"].arguments == "{}" + def test_invalid_json_yields_none(self): line = "data: broken{" results = list(handle_openai_responses_stream_response(line, True)) diff --git a/tests/unit/test_llm_structured_output.py b/tests/unit/test_llm_structured_output.py new file mode 100644 index 0000000..2f19a94 --- /dev/null +++ b/tests/unit/test_llm_structured_output.py @@ -0,0 +1,96 @@ +import pytest + +from buzzllm.llm import LLMOptions, RequestArgs, StreamResponse, invoke_llm + + +class FakeResponse: + def __init__(self, lines): + self._lines = lines + + def raise_for_status(self): + return None + + def iter_lines(self, decode_unicode=True): + return self._lines + + +@pytest.mark.asyncio +async def test_structured_output_non_sse(capsys, monkeypatch): + opts = LLMOptions( + model="gpt-4o-mini", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + output_mode="json_schema", + output_schema={"type": "object"}, + ) + + def make_request_args(_opts, _prompt, _system_prompt): + return RequestArgs(data={"messages": []}, headers={}) + + def handle_stream_response(_line, _started): + yield StreamResponse(id="", delta='{"ok": true}', type="output_text") + yield StreamResponse(id="", delta="", type="block_end") + + def add_tool_response(_messages, _tool_calls): + return None + + monkeypatch.setattr( + "buzzllm.llm.requests.post", + lambda *args, **kwargs: FakeResponse(["data: {}"]), + ) + + await invoke_llm( + opts, + "Hello", + "System", + make_request_args, + handle_stream_response, + add_tool_response, + sse=False, + brief=False, + ) + + captured = capsys.readouterr().out + assert '{"ok": true}' in captured + assert "=== [ DONE ] ===" in captured + + +@pytest.mark.asyncio +async def test_structured_output_sse(capsys, monkeypatch): + opts = LLMOptions( + model="gpt-4o-mini", + url="https://api.openai.com/v1/chat/completions", + api_key_name="OPENAI_API_KEY", + output_mode="json_schema", + output_schema={"type": "object"}, + ) + + def make_request_args(_opts, _prompt, _system_prompt): + return RequestArgs(data={"messages": []}, headers={}) + + def handle_stream_response(_line, _started): + yield StreamResponse(id="", delta='{"ok": true}', type="output_text") + yield StreamResponse(id="", delta="", type="block_end") + + def add_tool_response(_messages, _tool_calls): + return None + + monkeypatch.setattr( + "buzzllm.llm.requests.post", + lambda *args, **kwargs: FakeResponse(["data: {}"]), + ) + + await invoke_llm( + opts, + "Hello", + "System", + make_request_args, + handle_stream_response, + add_tool_response, + sse=True, + brief=False, + ) + + captured = capsys.readouterr().out + assert "event: output_structured" in captured + assert "event: output_text" not in captured diff --git a/tests/unit/test_llm_tool_messages.py b/tests/unit/test_llm_tool_messages.py index d00e12c..bbc34ad 100644 --- a/tests/unit/test_llm_tool_messages.py +++ b/tests/unit/test_llm_tool_messages.py @@ -1,20 +1,24 @@ import pytest +from typing import Any + from buzzllm.llm import ( ToolCall, tool_call_response_to_openai_messages, tool_call_response_to_anthropic_messages, + tool_call_response_to_openai_responses_messages, + RequestArgs, ) class TestToolCallResponseToOpenaiMessages: def test_empty_tool_calls_no_op(self): - messages = [{"role": "user", "content": "hello"}] + messages: list[dict[str, Any]] = [{"role": "user", "content": "hello"}] tool_call_response_to_openai_messages(messages, {}) assert len(messages) == 1 def test_single_tool_call(self): - messages = [{"role": "user", "content": "hello"}] + messages: list[dict[str, Any]] = [{"role": "user", "content": "hello"}] tool_calls = { "call_1": ToolCall( id="call_1", @@ -34,7 +38,9 @@ def test_single_tool_call(self): assert messages[1]["tool_calls"][0]["id"] == "call_1" assert messages[1]["tool_calls"][0]["type"] == "function" assert messages[1]["tool_calls"][0]["function"]["name"] == "search_web" - assert messages[1]["tool_calls"][0]["function"]["arguments"] == '{"query": "test"}' + assert ( + messages[1]["tool_calls"][0]["function"]["arguments"] == '{"query": "test"}' + ) # Tool result message assert messages[2]["role"] == "tool" @@ -42,7 +48,7 @@ def test_single_tool_call(self): assert messages[2]["content"] == "Found results" def test_multiple_tool_calls(self): - messages = [] + messages: list[dict[str, Any]] = [] tool_calls = { "call_1": ToolCall( id="call_1", name="func1", arguments="{}", executed=True, result="res1" @@ -61,7 +67,7 @@ def test_multiple_tool_calls(self): assert messages[2]["role"] == "tool" def test_preserves_existing_messages(self): - messages = [ + messages: list[dict[str, Any]] = [ {"role": "system", "content": "sys"}, {"role": "user", "content": "user"}, ] @@ -81,12 +87,12 @@ def test_preserves_existing_messages(self): class TestToolCallResponseToAnthropicMessages: def test_empty_tool_calls_no_op(self): - messages = [{"role": "user", "content": "hello"}] + messages: list[dict[str, Any]] = [{"role": "user", "content": "hello"}] tool_call_response_to_anthropic_messages(messages, {}) assert len(messages) == 1 def test_single_tool_call(self): - messages = [{"role": "user", "content": "hello"}] + messages: list[dict[str, Any]] = [{"role": "user", "content": "hello"}] tool_calls = { "tool_1": ToolCall( id="tool_1", @@ -116,7 +122,7 @@ def test_single_tool_call(self): assert messages[2]["content"][0]["content"] == "Search result" def test_multiple_tool_calls(self): - messages = [] + messages: list[dict[str, Any]] = [] tool_calls = { "t1": ToolCall( id="t1", name="f1", arguments='{"a": 1}', executed=True, result="r1" @@ -134,7 +140,7 @@ def test_multiple_tool_calls(self): assert len(messages[1]["content"]) == 2 # 2 tool_result blocks def test_arguments_parsed_to_dict(self): - messages = [] + messages: list[dict[str, Any]] = [] tool_calls = { "t1": ToolCall( id="t1", @@ -150,3 +156,36 @@ def test_arguments_parsed_to_dict(self): input_obj = messages[0]["content"][0]["input"] assert input_obj["nested"]["key"] == "value" assert input_obj["list"] == [1, 2, 3] + + +class TestToolCallResponseToOpenaiResponsesMessages: + def test_empty_tool_calls_no_op(self): + request_args = RequestArgs(data={"input": []}, headers={}) + tool_call_response_to_openai_responses_messages(request_args, {}) + assert request_args.data["input"] == [] + + def test_tool_results_payload(self): + request_args = RequestArgs(data={"input": []}, headers={}) + tool_calls = { + "call_1": ToolCall( + id="call_1", + name="search_web", + arguments='{"query": "test"}', + executed=True, + result="Result", + ) + } + + from buzzllm import llm as llm_module + + llm_module.last_openai_response_id = "resp_123" + + tool_call_response_to_openai_responses_messages(request_args, tool_calls) + assert request_args.data["previous_response_id"] == "resp_123" + assert request_args.data["input"] == [ + { + "type": "tool_result", + "tool_call_id": "call_1", + "content": "Result", + } + ] From 2ad2c37d8eebcdd82b94e531466047c31593dc21 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:49:29 -0500 Subject: [PATCH 13/17] ralph(iter 1): responses devlog --- devlogs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devlogs.md b/devlogs.md index 1891eab..a5b5a9f 100644 --- a/devlogs.md +++ b/devlogs.md @@ -83,3 +83,13 @@ Commands+Results: (none) Decisions: Validate tool_subset against catalog before spawning subprocess. Next: Implement openai-responses tooling updates. Checkpoint: 6587e1f + +## 2026-02-15: Iter 1 Responses + Structured Outputs + +Timestamp: 2026-02-15 +Goal: Add OpenAI Responses tool loop support and structured output handling. +Changes: Added responses input/tool handling, tool-call parsing, structured output buffering, and unit/integration tests. +Commands+Results: (none) +Decisions: Use response_format/output_config for structured outputs and previous_response_id for responses tool turns. +Next: Run full pytest verifier. +Checkpoint: d9e7781 From 2c0947076e2a0b44c88ba2156489d05419a89d4c Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:58:13 -0500 Subject: [PATCH 14/17] ralph(iter 1): responses fixes --- src/buzzllm/llm.py | 73 ++++++++++++++----- tests/conftest.py | 8 +- .../test_openai_responses_integration.py | 15 ++-- tests/integration/test_subagent_runner.py | 4 +- tests/unit/test_llm_request_args.py | 5 +- tests/unit/test_llm_stream_handlers.py | 41 +++++++++-- 6 files changed, 110 insertions(+), 36 deletions(-) diff --git a/src/buzzllm/llm.py b/src/buzzllm/llm.py index bbc70ea..4db47d0 100644 --- a/src/buzzllm/llm.py +++ b/src/buzzllm/llm.py @@ -71,6 +71,7 @@ async def execute(self, func: Callable): TOOL_CALLS: dict[str, ToolCall] = {} current_tool_call_id: str = "" last_openai_response_id: str = "" +openai_responses_item_id_to_call_id: dict[str, str] = {} async def run_tools(): @@ -167,6 +168,7 @@ async def invoke_llm( # Clear tool calls for next iteration TOOL_CALLS.clear() + openai_responses_item_id_to_call_id.clear() except Exception as e: print(e) @@ -522,6 +524,24 @@ def tool_call_response_to_anthropic_messages( # === +def _convert_openai_responses_tools(tools: list[dict]) -> list[dict]: + converted = [] + for tool in tools: + if tool.get("type") == "function" and "function" in tool: + function = tool.get("function", {}) + converted.append( + { + "type": "function", + "name": function.get("name"), + "description": function.get("description"), + "parameters": function.get("parameters"), + } + ) + else: + converted.append(tool) + return converted + + def make_openai_responses_request_args( opts: LLMOptions, prompt: str, system_prompt: str ) -> RequestArgs: @@ -535,13 +555,15 @@ def make_openai_responses_request_args( ], "stream": True, "store": False, - "reasoning": { - "effort": "high" if opts.think else "none", - "summary": "detailed" if opts.think else "none", - }, "temperature": opts.temperature, } + if opts.think: + data["reasoning"] = { + "effort": "high", + "summary": "detailed", + } + if opts.max_tokens: data["max_output_tokens"] = opts.max_tokens @@ -549,7 +571,7 @@ def make_openai_responses_request_args( data["instructions"] = system_prompt if opts.tools: - data["tools"] = opts.tools + data["tools"] = _convert_openai_responses_tools(opts.tools) data["tool_choice"] = "auto" headers = {"Content-Type": "application/json"} @@ -593,28 +615,45 @@ def handle_openai_responses_stream_response( elif event_type == "response.output_item.added": item = chunk_data.get("item", {}) - if item.get("type") == "tool_call": - tool_id = item.get("id", "") + item_type = item.get("type") + if item_type in ("tool_call", "function_call"): + item_id = item.get("id", "") + call_id = item.get("call_id") or item_id tool_name = item.get("name", "") - TOOL_CALLS[tool_id] = ToolCall( - id=tool_id, name=tool_name, arguments="", executed=False + if item_id: + openai_responses_item_id_to_call_id[item_id] = call_id + TOOL_CALLS[call_id] = ToolCall( + id=call_id, name=tool_name, arguments="", executed=False ) - elif event_type == "response.tool_call_arguments.delta": - tool_call_id = chunk_data.get("tool_call_id", "") + elif event_type in ( + "response.tool_call_arguments.delta", + "response.function_call_arguments.delta", + ): + item_id = ( + chunk_data.get("item_id") + or chunk_data.get("tool_call_id") + or chunk_data.get("call_id") + or "" + ) + call_id = openai_responses_item_id_to_call_id.get(item_id, item_id) delta_text = chunk_data.get("delta", "") - if tool_call_id in TOOL_CALLS: - TOOL_CALLS[tool_call_id].arguments += delta_text + if call_id in TOOL_CALLS: + TOOL_CALLS[call_id].arguments += delta_text if delta_text: yield StreamResponse(id="", delta=delta_text, type="tool_call") elif event_type == "response.output_item.done": item = chunk_data.get("item", {}) - if item.get("type") == "tool_call": - tool_id = item.get("id", "") + item_type = item.get("type") + if item_type in ("tool_call", "function_call"): + item_id = item.get("id", "") + call_id = item.get( + "call_id" + ) or openai_responses_item_id_to_call_id.get(item_id, item_id) arguments = item.get("arguments") - if tool_id in TOOL_CALLS and arguments: - TOOL_CALLS[tool_id].arguments = arguments + if call_id in TOOL_CALLS and arguments: + TOOL_CALLS[call_id].arguments = arguments # Handle response completion elif event_type == "response.completed": diff --git a/tests/conftest.py b/tests/conftest.py index 0530256..70fa286 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,11 @@ def docker_available() -> bool: client = docker.from_env() client.ping() - return True + images = client.images.list(name="buzz/python-exec") + for image in images: + if "buzz/python-exec:latest" in image.tags: + return True + return False except Exception: return False @@ -51,6 +55,7 @@ def reset_tool_state(): llm.TOOL_CALLS.clear() llm.current_tool_call_id = "" llm.last_openai_response_id = "" + llm.openai_responses_item_id_to_call_id.clear() yield @@ -58,6 +63,7 @@ def reset_tool_state(): llm.TOOL_CALLS.clear() llm.current_tool_call_id = "" llm.last_openai_response_id = "" + llm.openai_responses_item_id_to_call_id.clear() # === LLM fixtures === diff --git a/tests/integration/test_openai_responses_integration.py b/tests/integration/test_openai_responses_integration.py index 7122a0c..9df8c44 100644 --- a/tests/integration/test_openai_responses_integration.py +++ b/tests/integration/test_openai_responses_integration.py @@ -15,7 +15,7 @@ def test_simple_responses_output_text(self): import requests opts = LLMOptions( - model="o3", + model="gpt-4.1-mini", url="https://api.openai.com/v1/responses", api_key_name="OPENAI_API_KEY", max_tokens=50, @@ -69,7 +69,7 @@ def get_weather(city: str) -> str: tools = [utils.callable_to_openai_schema(utils.AVAILABLE_TOOLS["get_weather"])] opts = LLMOptions( - model="o3", + model="gpt-4.1-mini", url="https://api.openai.com/v1/responses", api_key_name="OPENAI_API_KEY", max_tokens=100, @@ -77,11 +77,16 @@ def get_weather(city: str) -> str: tools=tools, ) + def make_request_args_required(_opts, _prompt, _system_prompt): + args = make_openai_responses_request_args(_opts, _prompt, _system_prompt) + args.data["tool_choice"] = "required" + return args + await invoke_llm( opts, - "You must call get_weather with city=Paris.", - "You are helpful. Use tools when needed.", - make_openai_responses_request_args, + "Call get_weather with city=Paris before responding.", + "Always call tools when a tool is available.", + make_request_args_required, handle_openai_responses_stream_response, tool_call_response_to_openai_responses_messages, sse=False, diff --git a/tests/integration/test_subagent_runner.py b/tests/integration/test_subagent_runner.py index 820f1be..f15df32 100644 --- a/tests/integration/test_subagent_runner.py +++ b/tests/integration/test_subagent_runner.py @@ -18,10 +18,10 @@ def test_call_subagent_parses_output_text(self, monkeypatch): sse_text = "\n".join( [ "event: output_text", - 'data: {\\"id\\":\\"\\",\\"delta\\":\\"Hello \\",\\"type\\":\\"output_text\\"}', + 'data: {"id":"","delta":"Hello ","type":"output_text"}', "", "event: output_text", - 'data: {\\"id\\":\\"\\",\\"delta\\":\\"world\\",\\"type\\":\\"output_text\\"}', + 'data: {"id":"","delta":"world","type":"output_text"}', "", ] ) diff --git a/tests/unit/test_llm_request_args.py b/tests/unit/test_llm_request_args.py index 77bd89d..3b12b92 100644 --- a/tests/unit/test_llm_request_args.py +++ b/tests/unit/test_llm_request_args.py @@ -286,8 +286,7 @@ def test_basic_request_structure(self, env_with_api_keys): assert args.data["instructions"] == "Instructions" assert args.data["stream"] is True assert args.data["store"] is False - assert args.data["reasoning"]["effort"] == "none" - assert args.data["reasoning"]["summary"] == "none" + assert "reasoning" not in args.data def test_tools_are_included(self, env_with_api_keys): tools = [{"type": "function", "function": {"name": "test"}}] @@ -299,7 +298,7 @@ def test_tools_are_included(self, env_with_api_keys): ) args = make_openai_responses_request_args(opts, "Hello", "System") - assert args.data["tools"] == tools + assert args.data["tools"][0]["name"] == "test" assert args.data["tool_choice"] == "auto" def test_reasoning_mode(self, env_with_api_keys): diff --git a/tests/unit/test_llm_stream_handlers.py b/tests/unit/test_llm_stream_handlers.py index 9faff3d..83489e6 100644 --- a/tests/unit/test_llm_stream_handlers.py +++ b/tests/unit/test_llm_stream_handlers.py @@ -198,7 +198,12 @@ def test_response_completed(self): def test_tool_call_added(self): data = { "type": "response.output_item.added", - "item": {"type": "tool_call", "id": "call_1", "name": "search_web"}, + "item": { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "search_web", + }, } line = f"data: {json.dumps(data)}" list(handle_openai_responses_stream_response(line, True)) @@ -207,12 +212,15 @@ def test_tool_call_added(self): assert TOOL_CALLS["call_1"].name == "search_web" def test_tool_call_arguments_delta(self): - TOOL_CALLS["call_2"] = ToolCall( - id="call_2", name="tool", arguments="", executed=False + list( + handle_openai_responses_stream_response( + f"data: {json.dumps({'type': 'response.output_item.added', 'item': {'type': 'function_call', 'id': 'fc_2', 'call_id': 'call_2', 'name': 'tool'}})}", + True, + ) ) data = { - "type": "response.tool_call_arguments.delta", - "tool_call_id": "call_2", + "type": "response.function_call_arguments.delta", + "item_id": "fc_2", "delta": '{"q": "test"}', } line = f"data: {json.dumps(data)}" @@ -223,12 +231,29 @@ def test_tool_call_arguments_delta(self): assert len(tool_calls) == 1 def test_tool_call_done_sets_arguments(self): - TOOL_CALLS["call_3"] = ToolCall( - id="call_3", name="tool", arguments="", executed=False + added = { + "type": "response.output_item.added", + "item": { + "type": "function_call", + "id": "fc_3", + "call_id": "call_3", + "name": "tool", + }, + } + list( + handle_openai_responses_stream_response( + f"data: {json.dumps(added)}", + True, + ) ) data = { "type": "response.output_item.done", - "item": {"type": "tool_call", "id": "call_3", "arguments": "{}"}, + "item": { + "type": "function_call", + "id": "fc_3", + "call_id": "call_3", + "arguments": "{}", + }, } line = f"data: {json.dumps(data)}" list(handle_openai_responses_stream_response(line, True)) From e8b0d84892991144fde97b0e9a3bc59739a9693d Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 21:58:31 -0500 Subject: [PATCH 15/17] ralph(iter 1): test runs devlog --- devlogs.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/devlogs.md b/devlogs.md index a5b5a9f..a4b4aa3 100644 --- a/devlogs.md +++ b/devlogs.md @@ -93,3 +93,43 @@ Commands+Results: (none) Decisions: Use response_format/output_config for structured outputs and previous_response_id for responses tool turns. Next: Run full pytest verifier. Checkpoint: d9e7781 + +## 2026-02-15: Iter 1 Pytest Run 1 + +Timestamp: 2026-02-15 +Goal: Run full pytest verifier. +Changes: Ran full test suite. +Commands+Results: uv run pytest -> 5 failed (openai responses integration 400s, subagent SSE JSON decode, pythonexec docker image missing). +Decisions: Update responses reasoning payload, fix subagent SSE fixture JSON, skip docker tests when image missing. +Next: Apply fixes and rerun pytest. +Checkpoint: 2ad2c37 + +## 2026-02-15: Iter 1 Pytest Run 2 (Targeted) + +Timestamp: 2026-02-15 +Goal: Re-run failing tests after fixes. +Changes: Adjusted responses reasoning usage, tool schema conversion, tool-call parsing, docker image availability check, and subagent SSE fixture. +Commands+Results: uv run pytest (targeted) -> first run 2 failures (responses tool call), follow-up tool-call-only run passed. +Decisions: Parse function_call events for responses and force tool_choice=required in integration test. +Next: Run full pytest verifier. +Checkpoint: 2ad2c37 + +## 2026-02-15: Iter 1 Pytest Run 3 (Full) + +Timestamp: 2026-02-15 +Goal: Run full pytest after responses fixes. +Changes: Ran full test suite. +Commands+Results: uv run pytest -> 1 failed (responses output_item.done unit test missing added event). +Decisions: Update unit test to include output_item.added before output_item.done. +Next: Rerun full pytest. +Checkpoint: 2ad2c37 + +## 2026-02-15: Iter 1 Pytest Run 4 (Full) + +Timestamp: 2026-02-15 +Goal: Confirm full pytest passes. +Changes: Ran full test suite after last unit test fix. +Commands+Results: uv run pytest -> 191 passed, 3 skipped. +Decisions: None. +Next: Commit fixes and summarize. +Checkpoint: 2c09470 From 3b673aadc68f403d865e7cb008008db2200036be Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 22:04:52 -0500 Subject: [PATCH 16/17] ralph(iter 2): responses reasoning effort --- .gitignore | 1 + src/buzzllm/llm.py | 2 ++ tests/integration/test_openai_responses_integration.py | 6 +++--- tests/unit/test_llm_request_args.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2e34f41..1ef9fb6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dev_scratch .dingllm/files.txt llm.md .llm.md +logs/ diff --git a/src/buzzllm/llm.py b/src/buzzllm/llm.py index 4db47d0..ddea192 100644 --- a/src/buzzllm/llm.py +++ b/src/buzzllm/llm.py @@ -563,6 +563,8 @@ def make_openai_responses_request_args( "effort": "high", "summary": "detailed", } + else: + data["reasoning"] = {"effort": "none"} if opts.max_tokens: data["max_output_tokens"] = opts.max_tokens diff --git a/tests/integration/test_openai_responses_integration.py b/tests/integration/test_openai_responses_integration.py index 9df8c44..1041d46 100644 --- a/tests/integration/test_openai_responses_integration.py +++ b/tests/integration/test_openai_responses_integration.py @@ -15,7 +15,7 @@ def test_simple_responses_output_text(self): import requests opts = LLMOptions( - model="gpt-4.1-mini", + model="gpt-5.1", url="https://api.openai.com/v1/responses", api_key_name="OPENAI_API_KEY", max_tokens=50, @@ -33,7 +33,7 @@ def test_simple_responses_output_text(self): stream=True, timeout=30, ) - response.raise_for_status() + assert response.status_code == 200, response.text collected_text = [] for line in response.iter_lines(decode_unicode=True): @@ -69,7 +69,7 @@ def get_weather(city: str) -> str: tools = [utils.callable_to_openai_schema(utils.AVAILABLE_TOOLS["get_weather"])] opts = LLMOptions( - model="gpt-4.1-mini", + model="gpt-5.1", url="https://api.openai.com/v1/responses", api_key_name="OPENAI_API_KEY", max_tokens=100, diff --git a/tests/unit/test_llm_request_args.py b/tests/unit/test_llm_request_args.py index 3b12b92..e20c1c1 100644 --- a/tests/unit/test_llm_request_args.py +++ b/tests/unit/test_llm_request_args.py @@ -286,7 +286,7 @@ def test_basic_request_structure(self, env_with_api_keys): assert args.data["instructions"] == "Instructions" assert args.data["stream"] is True assert args.data["store"] is False - assert "reasoning" not in args.data + assert args.data["reasoning"]["effort"] == "none" def test_tools_are_included(self, env_with_api_keys): tools = [{"type": "function", "function": {"name": "test"}}] From 6067cafe234a0adfdd6037a0e62935dceb7a0339 Mon Sep 17 00:00:00 2001 From: Rohan Date: Sun, 15 Feb 2026 22:05:05 -0500 Subject: [PATCH 17/17] ralph(iter 2): devlog update --- devlogs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devlogs.md b/devlogs.md index a4b4aa3..b0666f2 100644 --- a/devlogs.md +++ b/devlogs.md @@ -133,3 +133,13 @@ Commands+Results: uv run pytest -> 191 passed, 3 skipped. Decisions: None. Next: Commit fixes and summarize. Checkpoint: 2c09470 + +## 2026-02-15: Iter 2 Responses Reasoning Effort + +Timestamp: 2026-02-15 +Goal: Align responses reasoning effort contract and keep integration passing. +Changes: Set responses reasoning effort to none when think is false; updated request-args unit test; moved responses integration model to gpt-5.1 and added response status assertion. +Commands+Results: uv run pytest -> 3 failed (responses integration 400s, request-args expectation); uv run pytest tests/integration/test_openai_responses_integration.py::TestOpenAIResponsesIntegration::test_simple_responses_output_text -s -> 400 reasoning.effort unsupported; uv run pytest tests/integration/test_openai_responses_integration.py::TestOpenAIResponsesIntegration::test_simple_responses_output_text -s -> 400 temperature unsupported; uv run pytest tests/integration/test_openai_responses_integration.py::TestOpenAIResponsesIntegration::test_simple_responses_output_text -s -> passed; uv run pytest -> 191 passed, 3 skipped. +Decisions: Use gpt-5.1 for responses integration to support reasoning effort none with temperature. +Next: None. +Checkpoint: 3b673aa