diff --git a/README.md b/README.md index fa20f66..677a12e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ AI coding assistant with persistent memory, powered by GitHub Copilot. -![Version](https://img.shields.io/badge/version-0.2-informational) +![Version](https://img.shields.io/badge/version-0.3-informational) [![CI](https://github.com/gdhanush27/co-claw/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/gdhanush27/co-claw/actions/workflows/ci.yml) ![VS Code](https://img.shields.io/badge/VS%20Code-1.93%2B-blue?logo=visual-studio-code) ![Copilot Required](https://img.shields.io/badge/Copilot-Required-orange?logo=github) diff --git a/docs/commands.md b/docs/commands.md index 1a77811..7419325 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -94,6 +94,8 @@ Runs your task through the multi-agent orchestrator. A Planner produces a JSON D Live progress is shown in the **Agents** sidebar (the CoClaw activity bar icon). Tune fan-out width with `CoClaw.agents.maxParallelCoders` (ceiling) and `CoClaw.agents.minParallelCoders` (floor — pads small tasks with generic lanes), and switch modes with `CoClaw.agents.mode` (`off`, `slash`, `always`). +Every `/agents` run also ends with an automatic **final reviewer** task (tagged `[final review]` in the plan) that depends on every coder and tester sibling, reads their shared-memory output, and produces a single consolidated review with an `APPROVED` / `CHANGES_REQUESTED` verdict. This is controlled by `CoClaw.agents.finalReviewer` (default `auto` — skipped when the planner already produced a covering reviewer; set to `always` to force one on every run, or `off` to disable). The final reviewer uses the **hard** tier, so it picks up whatever model you've assigned to `CoClaw.models.hard`. + ## Command Palette Commands Open the Command Palette with `Ctrl+Shift+P` and type the command name. diff --git a/docs/configuration.md b/docs/configuration.md index 0a01049..80e9f15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -54,6 +54,26 @@ All CoClaw settings are under the `CoClaw.*` namespace. Open them quickly with * - **Default:** `""` (empty — uses global preference) - **Description:** Override the Copilot model family at the workspace level. When set, this workspace uses the specified model regardless of the global selection. Leave empty to use the globally selected model. +### `CoClaw.models.light` + +- **Type:** string +- **Default:** `""` (empty — uses default model) +- **Description:** Model family to use for light-difficulty tasks (e.g. `gpt-4o-mini`). Light tasks are simple, fast operations such as formatting or renaming. Leave empty to fall back to the general model selection. + +### `CoClaw.models.medium` + +- **Type:** string +- **Default:** `""` (empty — uses default model) +- **Description:** Model family to use for medium-difficulty tasks (e.g. `gpt-4o`). Medium tasks are standard implementation or review work. Leave empty to fall back to the general model selection. + +### `CoClaw.models.hard` + +- **Type:** string +- **Default:** `""` (empty — uses default model) +- **Description:** Model family to use for hard-difficulty tasks (e.g. `claude-3.5-sonnet`). Hard tasks involve complex architecture, multi-file refactoring, or deep reasoning. Leave empty to fall back to the general model selection. + +> **Tip:** rather than editing the three `CoClaw.models.*` keys by hand, run **CoClaw: Select Tier Models (Light / Medium / Hard)** from the Command Palette, or send `/models` to the Telegram bot. Both surfaces write to the same settings. + ## Heartbeat Settings (`/open` mode) ### `CoClaw.heartbeat.enabled` @@ -159,6 +179,19 @@ All CoClaw settings are under the `CoClaw.*` namespace. Open them quickly with * - **Default:** `false` - **Description:** When `true`, every `/agents` run skips the per-task character cap entirely — equivalent to passing `--full` on every prompt. This overrides `CoClaw.agents.summaryMaxChars`. Useful when you regularly need long, untrimmed summaries; otherwise leave it off and pass `--full` ad-hoc. +### `CoClaw.agents.finalReviewer` + +- **Type:** string enum — `"auto"` / `"always"` / `"off"` +- **Default:** `"auto"` +- **Description:** Controls whether the orchestrator appends a **final reviewer** task at the end of every `/agents` run. The final reviewer is a `reviewer`-role agent that depends on every coder and tester task in the plan, reads their shared-memory output, inspects the actual files that changed, and ends its report with a single verdict line: `APPROVED` or `CHANGES_REQUESTED: `. +- **Modes:** + - `auto` *(default)* — inject a final reviewer **only if** the plan does not already contain a reviewer whose dependency graph transitively covers every coder + tester task. If the planner already produced a "review everything" task, no duplicate is added. + - `always` — append a final reviewer on every run, even if other reviewers already exist. Use this if you want a guaranteed, holistic final pass on top of any mid-DAG reviews. + - `off` — never inject. The plan's reviewers (if any) are used as-is. +- **Difficulty tier:** the auto-injected final reviewer is tagged with `difficulty: "hard"`, so it picks up whatever Copilot model you've assigned to the **hard** tier via `CoClaw.models.hard` (or the matching VS Code Settings dropdown / the `/model tier hard ` chat command). Pair `always` with a strong model on the hard tier (e.g. Claude Sonnet) for the best results. +- **What it produces:** the chat plan tags this task as `[final review]`, and the run summary contains the consolidated report — one short paragraph describing what was built, issues grouped by severity (Critical / Major / Minor / Nit) with concrete fixes, and the verdict line. +- **Why it exists:** with parallel coder fan-out, no single coder sees the whole change set. A final reviewer is the only agent guaranteed to look across every sibling at once. + #### Precedence The runtime cap is resolved in this order (first match wins): diff --git a/docs/model-switching.md b/docs/model-switching.md index c94d8e4..ae865b3 100644 --- a/docs/model-switching.md +++ b/docs/model-switching.md @@ -45,3 +45,75 @@ When set, this workspace-level preference overrides the global selection for tha The model's `maxInputTokens` determines how much memory context CoClaw can inject. With a larger context window model, more memories are included in each prompt. The token budget is controlled by `CoClaw.memory.tokenBudgetPercent` (default: 20%). **Example:** With a 128K-token model and 20% budget, up to ~25,600 tokens of memory are injected. + +## Tier-Based Model Selection + +CoClaw supports assigning different Copilot models to different task difficulty tiers: **light**, **medium**, and **hard**. This is most useful for multi-agent orchestration (`/agents`): the Planner tags each sub-task with a difficulty, and the Orchestrator routes that sub-task to the model you configured for that tier. + +| Tier | Intended workload | +|--------|----------------------------------------------------------------------------------| +| light | Trivial / formulaic work — typos, renames, version bumps, one-line patches. | +| medium | Typical implementation, review, and test work. The default when unsure. | +| hard | Multi-file refactors, architectural design, security-sensitive logic. | + +### Pick models via VS Code Quick Pick + +1. Press `Ctrl+Shift+P`. +2. Run **CoClaw: Select Tier Models (Light / Medium / Hard)**. +3. Pick the tier you want to configure. +4. Pick a Copilot model family from the list — or pick **Clear override** to reset that tier back to the default model. + +Repeat for the other tiers as needed. Selections persist as VS Code settings (`CoClaw.models.light` / `.medium` / `.hard`), so they roam with Settings Sync. + +### Pick models from Telegram + +There are two equivalent flows: + +- Send `/models` (alias `/m`) to jump straight into the **Model Tiers** panel. +- Or send `/settings` (alias `/s`) and tap the **Model Tiers** group. + +For each tier, the bot renders one inline button per available Copilot model family — tap to assign. Tap **🗑 Clear (use default)** to drop the override. + +### Pick models from chat with `/model` + +The `/model` chat command lets you inspect and change models without leaving the conversation. + +| Command | Effect | +|---|---| +| `/model` | Show the active model, all tier overrides, and one-click "Pick…" buttons. | +| `/model list` | List every Copilot model family currently available to you (live). | +| `/model gpt-4o-mini` | Switch the **general** model directly. Fuzzy match — `/model claude` will list every Claude family as buttons if more than one matches. | +| `/model tier hard` | Open the Quick Pick for the **hard** tier (skips the "which tier?" step). | +| `/model tier hard claude-opus-4` | Set the **hard** tier model directly. | +| `/model clear` | Drop the general model preference (revert to first available). | +| `/model clear hard` | Drop the **hard** tier override (revert to general default). | +| `/model help` | Print the table above. | + +The status bar and the next `/agents` run pick up the change immediately — no reload needed. + +### Pick models from VS Code Settings GUI + +1. Open Settings (`Ctrl+,`). +2. Search for `CoClaw.models`. +3. Each of the three tier settings (`light`, `medium`, `hard`) plus the top-level `CoClaw.model.family` now renders as a **dropdown** of known Copilot model families, with friendly labels like "GPT-5", "Claude Opus 4", and "Gemini 2.5 Pro". +4. Pick `(default — use top-level model)` on a tier to clear its override. + +> **Note:** the dropdown ships with a curated list of stable Copilot model family identifiers. If Copilot ever exposes a new family that isn't in the dropdown yet, use **CoClaw: Select Tier Models** from the Command Palette (or `/models` in Telegram) — both surfaces pull the **live** list from Copilot at the moment you open them, so they always reflect what's actually available to you right now. + +If you'd rather edit `settings.json` directly: + +| Setting | Example value | +|---|---| +| `CoClaw.models.light` | `gpt-4o-mini` | +| `CoClaw.models.medium` | `gpt-4o` | +| `CoClaw.models.hard` | `claude-3.5-sonnet` | + +Leave any tier empty to fall back to the general `CoClaw.model.family` selection (or the first available model). + +### How It Works End-to-End + +1. The Planner agent emits a `"difficulty"` field on each sub-task in its JSON DAG (`"light"`, `"medium"`, or `"hard"`). +2. The Orchestrator calls `ModelManager.getModelForTier(difficulty ?? "medium")` immediately before invoking each sub-task agent. +3. If no tier-specific override is configured — or the configured family isn't currently available — the call falls back to the general active model. A one-shot warning is logged to the **CoClaw** output channel whenever a configured family resolves to nothing, so silent fallback can't slip past you. +4. Dynamic tasks spawned via `CoClaw_spawn_agent` can also pass an explicit `"difficulty"`; without it they default to medium. +5. When the Coder fan-out splits a single task into parallel units, each child inherits the parent's difficulty. diff --git a/docs/telegram.md b/docs/telegram.md index cd31aaa..4d5770c 100644 --- a/docs/telegram.md +++ b/docs/telegram.md @@ -91,7 +91,7 @@ In `/open` mode the assistant adopts a configurable tone. Switch via `/settings` ## Settings Panel -Send `/settings` (or `/s`) to open an interactive panel with inline buttons. Settings are grouped into Agents, Memory, Heartbeat, Telegram, and Model. Each entry supports the right input style: +Send `/settings` (or `/s`) to open an interactive panel with inline buttons. Settings are grouped into Agents, Memory, Heartbeat, Telegram, Model, and Model Tiers. Send `/models` (or `/m`) to jump straight into the **Model Tiers** group. Each entry supports the right input style: - **Booleans** — on/off toggle - **Enums** — one button per allowed value diff --git a/package.json b/package.json index 7b674dc..a4595c4 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "CoClaw", "displayName": "CoClaw", "description": "AI coding assistant with persistent memory, powered by GitHub Copilot", - "version": "0.2.3", + "version": "0.3.0", "publisher": "gdhanush270", "license": "Apache-2.0", "icon": "icon.png", @@ -70,6 +70,10 @@ { "name": "agents", "description": "Run a multi-agent orchestration: Planner decomposes the task, Coder/Reviewer/Tester/Memory agents execute in parallel" + }, + { + "name": "model", + "description": "Show or change the active Copilot model. Try /model, /model list, /model gpt-4o, /model tier hard claude-opus-4, /model clear" } ] } @@ -260,7 +264,8 @@ "runId": {"type": "string"}, "agent": {"type": "string", "enum": ["coder", "reviewer", "tester", "memory"]}, "prompt": {"type": "string"}, - "dependsOn": {"type": "array", "items": {"type": "string"}, "description": "Optional task ids this dynamic task should wait for"} + "dependsOn": {"type": "array", "items": {"type": "string"}, "description": "Optional task ids this dynamic task should wait for"}, + "difficulty": {"type": "string", "enum": ["light", "medium", "hard"], "description": "Optional difficulty tier controlling which configured model handles this task. Defaults to 'medium'."} } } }, @@ -283,6 +288,10 @@ "command": "CoClaw.selectModel", "title": "CoClaw: Select Model" }, + { + "command": "CoClaw.selectTierModels", + "title": "CoClaw: Select Tier Models (Light / Medium / Hard)" + }, { "command": "CoClaw.showMemory", "title": "CoClaw: Browse Memory" @@ -418,8 +427,239 @@ "properties": { "CoClaw.model.family": { "type": "string", - "description": "Preferred Copilot model family (workspace override)", - "default": "" + "default": "", + "enum": [ + "", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-5", + "gpt-5-mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "claude-3.5-sonnet", + "claude-3.7-sonnet", + "claude-sonnet-4", + "claude-opus-4", + "gemini-2.0-flash", + "gemini-2.5-pro" + ], + "enumItemLabels": [ + "(auto — first available)", + "GPT-4o", + "GPT-4o mini", + "GPT-4.1", + "GPT-5", + "GPT-5 mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "Claude 3.5 Sonnet", + "Claude 3.7 Sonnet", + "Claude Sonnet 4", + "Claude Opus 4", + "Gemini 2.0 Flash", + "Gemini 2.5 Pro" + ], + "enumDescriptions": [ + "Use whichever Copilot model is currently active / first available.", + "OpenAI GPT-4o — balanced general-purpose model.", + "OpenAI GPT-4o mini — cheap and fast.", + "OpenAI GPT-4.1.", + "OpenAI GPT-5 — most capable general model.", + "OpenAI GPT-5 mini.", + "OpenAI o1 — reasoning model.", + "OpenAI o1-mini.", + "OpenAI o3 — reasoning model.", + "OpenAI o3-mini.", + "Anthropic Claude 3.5 Sonnet.", + "Anthropic Claude 3.7 Sonnet.", + "Anthropic Claude Sonnet 4.", + "Anthropic Claude Opus 4 — strongest Anthropic model.", + "Google Gemini 2.0 Flash.", + "Google Gemini 2.5 Pro." + ], + "markdownDescription": "Preferred Copilot model family (workspace override). Pick a model from the dropdown, or [pick from the live Copilot list](command:CoClaw.selectModel) (always reflects the current Copilot inventory, including families newer than this dropdown)." + }, + "CoClaw.models.light": { + "type": "string", + "default": "", + "enum": [ + "", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-5", + "gpt-5-mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "claude-3.5-sonnet", + "claude-3.7-sonnet", + "claude-sonnet-4", + "claude-opus-4", + "gemini-2.0-flash", + "gemini-2.5-pro" + ], + "enumItemLabels": [ + "(default — use top-level model)", + "GPT-4o", + "GPT-4o mini", + "GPT-4.1", + "GPT-5", + "GPT-5 mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "Claude 3.5 Sonnet", + "Claude 3.7 Sonnet", + "Claude Sonnet 4", + "Claude Opus 4", + "Gemini 2.0 Flash", + "Gemini 2.5 Pro" + ], + "enumDescriptions": [ + "No override — fall back to CoClaw.model.family.", + "OpenAI GPT-4o.", + "OpenAI GPT-4o mini — recommended for light tasks.", + "OpenAI GPT-4.1.", + "OpenAI GPT-5.", + "OpenAI GPT-5 mini.", + "OpenAI o1.", + "OpenAI o1-mini.", + "OpenAI o3.", + "OpenAI o3-mini.", + "Anthropic Claude 3.5 Sonnet.", + "Anthropic Claude 3.7 Sonnet.", + "Anthropic Claude Sonnet 4.", + "Anthropic Claude Opus 4.", + "Google Gemini 2.0 Flash.", + "Google Gemini 2.5 Pro." + ], + "markdownDescription": "Model family for **light**-difficulty tasks (typos, renames, version bumps, mechanical formatting). Pick a model from the dropdown, or [pick from the live Copilot list](command:CoClaw.selectTierModels?%5B%22light%22%5D) (always reflects the current Copilot inventory, including families newer than this dropdown)." + }, + "CoClaw.models.medium": { + "type": "string", + "default": "", + "enum": [ + "", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-5", + "gpt-5-mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "claude-3.5-sonnet", + "claude-3.7-sonnet", + "claude-sonnet-4", + "claude-opus-4", + "gemini-2.0-flash", + "gemini-2.5-pro" + ], + "enumItemLabels": [ + "(default — use top-level model)", + "GPT-4o", + "GPT-4o mini", + "GPT-4.1", + "GPT-5", + "GPT-5 mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "Claude 3.5 Sonnet", + "Claude 3.7 Sonnet", + "Claude Sonnet 4", + "Claude Opus 4", + "Gemini 2.0 Flash", + "Gemini 2.5 Pro" + ], + "enumDescriptions": [ + "No override — fall back to CoClaw.model.family.", + "OpenAI GPT-4o — recommended for medium tasks.", + "OpenAI GPT-4o mini.", + "OpenAI GPT-4.1.", + "OpenAI GPT-5.", + "OpenAI GPT-5 mini.", + "OpenAI o1.", + "OpenAI o1-mini.", + "OpenAI o3.", + "OpenAI o3-mini.", + "Anthropic Claude 3.5 Sonnet.", + "Anthropic Claude 3.7 Sonnet.", + "Anthropic Claude Sonnet 4.", + "Anthropic Claude Opus 4.", + "Google Gemini 2.0 Flash.", + "Google Gemini 2.5 Pro." + ], + "markdownDescription": "Model family for **medium**-difficulty tasks (typical implementation, review, and tests — the default tier when the planner is unsure). Pick a model from the dropdown, or [pick from the live Copilot list](command:CoClaw.selectTierModels?%5B%22medium%22%5D) (always reflects the current Copilot inventory, including families newer than this dropdown)." + }, + "CoClaw.models.hard": { + "type": "string", + "default": "", + "enum": [ + "", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-5", + "gpt-5-mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "claude-3.5-sonnet", + "claude-3.7-sonnet", + "claude-sonnet-4", + "claude-opus-4", + "gemini-2.0-flash", + "gemini-2.5-pro" + ], + "enumItemLabels": [ + "(default — use top-level model)", + "GPT-4o", + "GPT-4o mini", + "GPT-4.1", + "GPT-5", + "GPT-5 mini", + "o1", + "o1-mini", + "o3", + "o3-mini", + "Claude 3.5 Sonnet", + "Claude 3.7 Sonnet", + "Claude Sonnet 4", + "Claude Opus 4", + "Gemini 2.0 Flash", + "Gemini 2.5 Pro" + ], + "enumDescriptions": [ + "No override — fall back to CoClaw.model.family.", + "OpenAI GPT-4o.", + "OpenAI GPT-4o mini.", + "OpenAI GPT-4.1.", + "OpenAI GPT-5 — recommended for hard tasks.", + "OpenAI GPT-5 mini.", + "OpenAI o1 — reasoning model, good for hard tasks.", + "OpenAI o1-mini.", + "OpenAI o3 — reasoning model.", + "OpenAI o3-mini.", + "Anthropic Claude 3.5 Sonnet.", + "Anthropic Claude 3.7 Sonnet.", + "Anthropic Claude Sonnet 4.", + "Anthropic Claude Opus 4 — recommended for hard tasks.", + "Google Gemini 2.0 Flash.", + "Google Gemini 2.5 Pro." + ], + "markdownDescription": "Model family for **hard**-difficulty tasks (multi-file refactors, architectural design, security review, complex reasoning). Pick a model from the dropdown, or [pick from the live Copilot list](command:CoClaw.selectTierModels?%5B%22hard%22%5D) (always reflects the current Copilot inventory, including families newer than this dropdown)." }, "CoClaw.memory.maxLongTermEntries": { "type": "number", @@ -540,6 +780,22 @@ "default": false, "description": "When true, every /agents run prints each agent's full output without truncation — equivalent to passing '--full' on every prompt. Overrides 'CoClaw.agents.summaryMaxChars'. Use this if you regularly need long, untrimmed summaries; leave off (default) and pass '--full' ad-hoc when you want occasional long output." }, + "CoClaw.agents.finalReviewer": { + "type": "string", + "default": "auto", + "enum": ["auto", "always", "off"], + "enumItemLabels": [ + "auto (inject if no covering reviewer)", + "always (inject every run)", + "off (never inject)" + ], + "enumDescriptions": [ + "Inject a final reviewer task at the end of the run, but ONLY if the plan doesn't already contain a reviewer that depends on every coder + tester task.", + "Always inject a final reviewer that depends on every coder + tester task, even if the plan already contains other reviewers.", + "Never inject. The plan's reviewers (if any) are used as-is." + ], + "markdownDescription": "Controls the orchestrator's **final reviewer** — an automatically-injected `reviewer` task at the end of every `/agents` run that reads every coder/tester sibling's output and produces a single consolidated review with a verdict line (`APPROVED` / `CHANGES_REQUESTED`). The final reviewer is configured for the **hard** difficulty tier so it picks up the strongest model you have assigned to that tier." + }, "CoClaw.tools.maxPerRequest": { "type": "integer", "default": 120, diff --git a/src/agents/AgentDefinitions.ts b/src/agents/AgentDefinitions.ts index 5e68368..3e1fba0 100644 --- a/src/agents/AgentDefinitions.ts +++ b/src/agents/AgentDefinitions.ts @@ -32,6 +32,7 @@ Schema: "agent": "coder" | "reviewer" | "tester" | "memory", "prompt": "string (focused instruction for this agent)", "units": ["optional", "array", "of", "file/paths/or/feature/slices"], + "difficulty": "light" | "medium" | "hard" // optional, defaults to "medium" "dependsOn": ["task-id", ...] } ] @@ -51,6 +52,11 @@ Schema: - A typical plan looks like: 1 coder task (with units) -> 1 reviewer (depends on coder) -> 1 tester (depends on reviewer). Add a memory task at the end ONLY if the user's request implies long-term knowledge worth saving. - Use 1-4 tasks total. Be terse in prompts (1-2 sentences). - NEVER include a "planner" or "orchestrator" task. +- DIFFICULTY: tag each task with a tier so the orchestrator can route it to the right model. + * "light" — trivial, formulaic work: typo fixes, rename/version bumps, tiny doc tweaks, mechanical formatting, single-line edits. + * "medium" — typical implementation, review, or test work; the default when unsure. + * "hard" — multi-file refactoring, architectural design, security-sensitive logic, complex algorithms, deep reasoning over large context. + Be conservative: only escalate to "hard" when extra reasoning capacity would clearly help. Memory distillation is usually "light"; reviewing security-sensitive code is "hard". - Output ONLY the JSON object. No explanation. @@ -58,16 +64,22 @@ Schema: User task: "Add a rate-limited /api/comments endpoint backed by Postgres." Good plan: {"tasks":[ - {"id":"build","agent":"coder","prompt":"Add a rate-limited POST /api/comments endpoint persisted to Postgres.","units":["api-routes","data-model","auth-security","tests"],"dependsOn":[]}, - {"id":"review","agent":"reviewer","prompt":"Review the new endpoint for security, validation and error handling.","dependsOn":["build"]} + {"id":"build","agent":"coder","prompt":"Add a rate-limited POST /api/comments endpoint persisted to Postgres.","units":["api-routes","data-model","auth-security","tests"],"difficulty":"hard","dependsOn":[]}, + {"id":"review","agent":"reviewer","prompt":"Review the new endpoint for security, validation and error handling.","difficulty":"hard","dependsOn":["build"]} ]} User task: "Redesign the sign-in and sign-up pages with a modern look." Good plan: {"tasks":[ - {"id":"redesign","agent":"coder","prompt":"Redesign sign-in and sign-up pages with a modern, clean look.","units":["html-markup","css-styling","js-behavior"],"dependsOn":[]}, - {"id":"review","agent":"reviewer","prompt":"Review the redesign for accessibility, responsiveness and consistency.","dependsOn":["redesign"]}, - {"id":"test","agent":"tester","prompt":"Add/update tests for rendering and form submission.","dependsOn":["review"]} + {"id":"redesign","agent":"coder","prompt":"Redesign sign-in and sign-up pages with a modern, clean look.","units":["html-markup","css-styling","js-behavior"],"difficulty":"medium","dependsOn":[]}, + {"id":"review","agent":"reviewer","prompt":"Review the redesign for accessibility, responsiveness and consistency.","difficulty":"medium","dependsOn":["redesign"]}, + {"id":"test","agent":"tester","prompt":"Add/update tests for rendering and form submission.","difficulty":"light","dependsOn":["review"]} +]} + +User task: "Fix the typo 'recieve' -> 'receive' in README." +Good plan: +{"tasks":[ + {"id":"fix","agent":"coder","prompt":"Fix the 'recieve' -> 'receive' typo in README.","difficulty":"light","dependsOn":[]} ]} `; diff --git a/src/agents/CoderSplitter.ts b/src/agents/CoderSplitter.ts index b03c2a3..481beb7 100644 --- a/src/agents/CoderSplitter.ts +++ b/src/agents/CoderSplitter.ts @@ -93,6 +93,9 @@ export function splitCoderTask( agent: 'coder', prompt: `${task.prompt}\n\nFOCUS UNIT: ${unit}\n${concernGuidance(unit)}\nWork on this unit ONLY. Other parallel coders are handling the rest. Do NOT touch files outside your unit.`, units: [unit], + // Inherit the parent's difficulty tier so per-task model routing + // survives the auto-fanout step. + difficulty: task.difficulty, dependsOn, status: 'pending', }; diff --git a/src/agents/Orchestrator.ts b/src/agents/Orchestrator.ts index 756c837..15e4a6c 100644 --- a/src/agents/Orchestrator.ts +++ b/src/agents/Orchestrator.ts @@ -6,10 +6,121 @@ import { RunRegistry } from './RunRegistry'; import { SharedMemoryStore } from './SharedMemoryStore'; import { splitCoderTask } from './CoderSplitter'; import { AGENT_DEFINITIONS } from './AgentDefinitions'; -import { AgentRole, PlanDocument, SubTask } from './types'; +import { AgentRole, PlanDocument, SubTask, TaskDifficulty } from './types'; import { AgentSpawner } from '../tools/spawnAgentTool'; import { Logger } from '../util/Logger'; +/** Allow-list for the `difficulty` field coming back from the planner. */ +const VALID_DIFFICULTIES: ReadonlySet = new Set(['light', 'medium', 'hard']); + +function coerceDifficulty(raw: unknown): TaskDifficulty | undefined { + if (typeof raw !== 'string') { return undefined; } + const v = raw.trim().toLowerCase(); + return VALID_DIFFICULTIES.has(v as TaskDifficulty) ? (v as TaskDifficulty) : undefined; +} + +/** Possible modes for the auto-injected final reviewer. */ +export type FinalReviewerMode = 'auto' | 'always' | 'off'; + +const VALID_FINAL_REVIEWER_MODES: ReadonlySet = new Set(['auto', 'always', 'off']); + +/** Roles whose output the final reviewer needs to cover. */ +const CODE_PRODUCING_ROLES: ReadonlySet = new Set(['coder', 'tester']); + +/** + * Prompt handed to the auto-injected final reviewer task. The reviewer's + * SYSTEM prompt already locks it to read-only tools and a verdict line; + * this USER prompt scopes it explicitly to a holistic, run-wide review. + */ +const FINAL_REVIEW_PROMPT = `Final consolidated review of this /agents run. + +All coder and tester siblings above have completed (their task ids are listed in your dependsOn). Your job: + +1. Use CoClaw_shared_memory_read with the runId from your to pull EVERY coder:* and tester:* entry. Do NOT just review a single task — the value of a final reviewer is the cross-cutting view. +2. Inspect the actual files that changed using read-only file tools (read / search). Do NOT use any edit/write/delete tools — they are denied for your role. +3. Look at the run as a WHOLE: + - Consistency between sibling tasks (naming, error handling, types, contracts). + - Cross-cutting concerns: security, input validation, authentication, secret handling, logging, performance, test coverage. + - Integration issues no individual coder could see (e.g. did the API change but the caller wasn't updated?). +4. Produce a SINGLE consolidated review report with this structure: + - One short paragraph summarizing what was built. + - Issues grouped by severity: **Critical**, **Major**, **Minor**, **Nit**. Cite file paths and line ranges where possible. + - Concrete suggested fixes for each issue. +5. End with EXACTLY ONE LINE on its own: + - "APPROVED" if you'd merge as-is. + - "CHANGES_REQUESTED: " otherwise. + +Constraints: do NOT modify any code, do NOT spawn other agents (CoClaw_spawn_agent is denied for your role anyway), do NOT ask the user questions.`; + +/** + * Detect whether a task is the auto-injected final reviewer. Used only for + * cosmetic chat rendering — orchestration logic does not branch on this. + */ +export function isFinalReviewTask(task: SubTask): boolean { + if (task.agent !== 'reviewer') { return false; } + return task.id === 'final-review' || /^final-review-\d+$/.test(task.id); +} + +/** + * Decide whether to append a "final reviewer" task to `tasks` and return the + * (possibly extended) list. Pure function — exported for unit tests. + * + * A reviewer "covers" the code-producing tasks if every coder/tester id + * appears in its transitive `dependsOn` closure. If `auto` and at least one + * existing reviewer already covers them, no injection happens. `always` always + * appends. `off` never appends. If there are no code-producing tasks at all, + * we never inject (nothing to review). + */ +export function injectFinalReviewer(tasks: SubTask[], mode: FinalReviewerMode): SubTask[] { + if (mode === 'off') { return tasks; } + + const codeTasks = tasks.filter(t => CODE_PRODUCING_ROLES.has(t.agent)); + if (codeTasks.length === 0) { return tasks; } + const codeIds = new Set(codeTasks.map(t => t.id)); + + if (mode === 'auto') { + const byId = new Map(tasks.map(t => [t.id, t])); + const coveredBy = (reviewer: SubTask): boolean => { + const seen = new Set(); + const stack: string[] = [...reviewer.dependsOn]; + while (stack.length > 0) { + const id = stack.pop()!; + if (seen.has(id)) { continue; } + seen.add(id); + const dep = byId.get(id); + if (dep) { + for (const d of dep.dependsOn) { stack.push(d); } + } + } + for (const cid of codeIds) { + if (!seen.has(cid)) { return false; } + } + return true; + }; + const alreadyCovered = tasks.some(t => t.agent === 'reviewer' && coveredBy(t)); + if (alreadyCovered) { return tasks; } + } + + // Choose a unique id (`final-review`, `final-review-2`, ...). + const taken = new Set(tasks.map(t => t.id)); + let id = 'final-review'; + let n = 1; + while (taken.has(id)) { + n++; + id = `final-review-${n}`; + } + + const finalTask: SubTask = { + id, + agent: 'reviewer', + prompt: FINAL_REVIEW_PROMPT, + dependsOn: codeTasks.map(t => t.id), + difficulty: 'hard', + status: 'pending', + }; + return [...tasks, finalTask]; +} + export interface SpawnerHolder { current: AgentSpawner | undefined; } @@ -124,6 +235,9 @@ export class Orchestrator implements AgentSpawner { // --- Step A.5: Auto-fanout coder tasks --- plan.tasks = this.applyAutoFanout(plan.tasks, prompt); + // --- Step A.6: Inject final reviewer (auto / always) --- + plan.tasks = this.ensureFinalReviewer(plan.tasks); + // Cycle check if (this.hasCycle(plan.tasks)) { stream.markdown('_Plan contains cyclic dependencies; aborting._\n\n'); @@ -151,11 +265,24 @@ export class Orchestrator implements AgentSpawner { } // --- AgentSpawner --- - async spawnDynamicTask(runId: string, role: AgentRole, prompt: string, dependsOn: string[] = []): Promise<{ taskId: string; status: string; output?: string; error?: string }> { + async spawnDynamicTask( + runId: string, + role: AgentRole, + prompt: string, + dependsOn: string[] = [], + difficulty?: TaskDifficulty, + ): Promise<{ taskId: string; status: string; output?: string; error?: string }> { const run = this.registry.getRun(runId); if (!run) { throw new Error(`Run ${runId} not found`); } const taskId = `dyn-${role}-${run.tasks.length + 1}`; - const newTask: SubTask = { id: taskId, agent: role, prompt, dependsOn, status: 'pending' }; + const newTask: SubTask = { + id: taskId, + agent: role, + prompt, + dependsOn, + difficulty: difficulty ?? undefined, + status: 'pending', + }; run.tasks.push(newTask); this.registry.updateTask(runId, taskId, {}); @@ -183,7 +310,8 @@ export class Orchestrator implements AgentSpawner { if (!AGENT_DEFINITIONS[agent] || agent === 'planner' || agent === 'orchestrator') { continue; } const dependsOn = Array.isArray(raw.dependsOn) ? (raw.dependsOn as unknown[]).map(String) : []; const units = Array.isArray(raw.units) ? (raw.units as unknown[]).map(String) : undefined; - tasks.push({ id, agent, prompt, units, dependsOn, status: 'pending' }); + const difficulty = coerceDifficulty(raw.difficulty); + tasks.push({ id, agent, prompt, units, difficulty, dependsOn, status: 'pending' }); } return tasks.length > 0 ? { tasks } : undefined; } catch { @@ -223,6 +351,27 @@ export class Orchestrator implements AgentSpawner { return out; } + /** + * Inject a final reviewer task that depends on every code-producing + * sibling (coder + tester). Honors `CoClaw.agents.finalReviewer`: + * - "off" → no-op. + * - "auto" → inject only if no existing reviewer already covers + * every code-producing task (transitive deps). + * - "always" → inject unconditionally (in addition to any existing + * reviewers in the plan). + * + * Pure-ish wrapper over {@link injectFinalReviewer} so the orchestrator + * is the only place that touches `vscode.workspace.getConfiguration`. + */ + private ensureFinalReviewer(tasks: SubTask[]): SubTask[] { + const raw = vscode.workspace.getConfiguration('CoClaw.agents') + .get('finalReviewer', 'auto'); + const mode: FinalReviewerMode = VALID_FINAL_REVIEWER_MODES.has(raw as FinalReviewerMode) + ? (raw as FinalReviewerMode) + : 'auto'; + return injectFinalReviewer(tasks, mode); + } + private hasCycle(tasks: SubTask[]): boolean { const byId = new Map(tasks.map(t => [t.id, t])); const WHITE = 0, GRAY = 1, BLACK = 2; @@ -253,10 +402,12 @@ export class Orchestrator implements AgentSpawner { for (const t of tasks) { const deps = t.dependsOn.length > 0 ? ` _(depends on: ${t.dependsOn.join(', ')})_` : ''; const unit = (t.units && t.units.length > 0) ? ` — _focus: ${t.units.join(', ').substring(0, 80)}_` : ''; + const tierTag = t.difficulty ? ` _[${t.difficulty}]_` : ''; + const finalTag = isFinalReviewTask(t) ? ' **[final review]**' : ''; // Use the first non-empty line that isn't the FOCUS UNIT marker for the title const lines = t.prompt.split('\n').map(l => l.trim()).filter(l => l.length > 0 && !l.startsWith('FOCUS UNIT:')); const title = (lines[0] ?? t.prompt).substring(0, 100); - stream.markdown(`- \`${t.id}\` **${t.agent}** — ${title}${unit}${deps}\n`); + stream.markdown(`- \`${t.id}\` **${t.agent}**${tierTag}${finalTag} — ${title}${unit}${deps}\n`); } stream.markdown('\n'); } @@ -323,13 +474,25 @@ export class Orchestrator implements AgentSpawner { const launchTask = (task: SubTask): Promise => { this.registry.updateTask(runId, task.id, { status: 'running', startedAt: Date.now() }); - stream.markdown(`▶ \`${task.id}\` (${task.agent}) started\n`); + const tier: TaskDifficulty = task.difficulty ?? 'medium'; const work = (async () => { try { if (token.isCancellationRequested) { throw new Error('cancelled'); } + // Resolve the per-task model from its difficulty tier. + // Falls back to the orchestrator's default model when no + // tier override is configured (see ModelManager). + const taskModel = await this.modelManager.getModelForTier(tier); + // Surface the actual model on each task so the user can + // verify per-tier routing in real time. Also persisted to + // the CoClaw output channel at info level for audit. + stream.markdown(`▶ \`${task.id}\` (${task.agent} · ${tier} → **${taskModel.name}** \`${taskModel.family}\`) started\n`); + Logger.info( + 'orchestrator', + `task=${task.id} agent=${task.agent} tier=${tier} model=${taskModel.family} (${taskModel.name})`, + ); const result = await agentRunner.runAgent( task.agent, task.prompt, @@ -337,6 +500,8 @@ export class Orchestrator implements AgentSpawner { task.id, token, toolInvocationToken, + undefined, + taskModel, ); // Persist the agent's text output to shared memory automatically. // Same cap source as the chat renderer so raising one raises diff --git a/src/agents/SpecializedAgent.ts b/src/agents/SpecializedAgent.ts index e6f894a..4641828 100644 --- a/src/agents/SpecializedAgent.ts +++ b/src/agents/SpecializedAgent.ts @@ -15,10 +15,19 @@ export interface AgentRunResult { * Runs a single specialized agent (one role) as a self-contained * model.sendRequest tool-loop. Uses all registered vscode.lm.tools that * the role's allowsTool() permits, plus the shared-memory tools. + * + * The model is supplied per `runAgent` call so the orchestrator can route + * different sub-tasks to different model tiers (light / medium / hard) + * without rebuilding the agent runner. */ export class SpecializedAgent { + /** + * @param defaultModel Fallback model used when a caller does not pass one + * to `runAgent`. Typically the result of + * `ModelManager.getActiveModel()`. + */ constructor( - private readonly model: vscode.LanguageModelChat, + private readonly defaultModel: vscode.LanguageModelChat, ) {} async runAgent( @@ -29,9 +38,11 @@ export class SpecializedAgent { token: vscode.CancellationToken, toolInvocationToken?: vscode.ChatParticipantToolToken, onText?: (chunk: string) => void, + model?: vscode.LanguageModelChat, ): Promise { const def = AGENT_DEFINITIONS[role]; if (!def) { throw new Error(`Unknown agent role: ${role}`); } + const activeModel = model ?? this.defaultModel; // Two-stage selection: // 1. Drop everything the role's own allow-list rejects. @@ -57,7 +68,7 @@ You can publish results by calling CoClaw_shared_memory_write with runId="${runI let fullText = ''; let toolCallsMade = 0; - let response = await this.model.sendRequest(messages, { tools }, token); + let response = await activeModel.sendRequest(messages, { tools }, token); for (let round = 0; round < MAX_TOOL_ROUNDS; round++) { if (token.isCancellationRequested) { break; } @@ -96,7 +107,7 @@ You can publish results by calling CoClaw_shared_memory_write with runId="${runI messages.push(vscode.LanguageModelChatMessage.User(resultParts)); try { - response = await this.model.sendRequest(messages, { tools }, token); + response = await activeModel.sendRequest(messages, { tools }, token); } catch (e) { fullText += `\n[agent error: ${e instanceof Error ? e.message : String(e)}]`; break; diff --git a/src/agents/types.ts b/src/agents/types.ts index d3d00ad..cf0c8c6 100644 --- a/src/agents/types.ts +++ b/src/agents/types.ts @@ -2,12 +2,17 @@ export type AgentRole = 'planner' | 'coder' | 'reviewer' | 'tester' | 'memory' | export type AgentStatus = 'pending' | 'running' | 'done' | 'failed'; +/** Difficulty tier that determines which model is used for a task. */ +export type TaskDifficulty = 'light' | 'medium' | 'hard'; + export interface SubTask { id: string; agent: AgentRole; prompt: string; /** Optional list of file paths or feature slices the planner suggests. */ units?: string[]; + /** Difficulty tier controlling which model to use. Defaults to 'medium'. */ + difficulty?: TaskDifficulty; dependsOn: string[]; status: AgentStatus; output?: string; diff --git a/src/commands/selectTierModels.ts b/src/commands/selectTierModels.ts new file mode 100644 index 0000000..e0b1b7f --- /dev/null +++ b/src/commands/selectTierModels.ts @@ -0,0 +1,82 @@ +import * as vscode from 'vscode'; +import { ModelManager, TIERS } from '../lm/ModelManager'; +import type { TaskDifficulty } from '../agents/types'; + +const VALID_TIERS: ReadonlySet = new Set(TIERS); + +function isValidTier(v: unknown): v is TaskDifficulty { + return typeof v === 'string' && VALID_TIERS.has(v as TaskDifficulty); +} + +/** + * Register the `CoClaw: Select Tier Models` command. + * + * Without arguments the command asks which tier to configure (light/medium/ + * hard) and then shows a model QuickPick for that tier. When invoked with + * a tier argument — e.g. from a settings.json command link like + * `command:CoClaw.selectTierModels?%5B%22hard%22%5D` — the tier picker is + * skipped and we jump straight to the model picker for that tier. + * + * Persistence flows through {@link ModelManager.setTierFamily} which writes + * VS Code workspace settings — the same store used by the Telegram + * /settings → Model Tiers UI, so both surfaces stay in sync. + */ +export function registerSelectTierModelsCommand( + modelManager: ModelManager, +): vscode.Disposable { + return vscode.commands.registerCommand('CoClaw.selectTierModels', async (tierArg?: unknown) => { + let tier: TaskDifficulty | undefined = isValidTier(tierArg) ? tierArg : undefined; + + if (!tier) { + type TierItem = vscode.QuickPickItem & { tier: TaskDifficulty }; + const current = modelManager.getAllTierFamilies(); + const tierItems: TierItem[] = TIERS.map(t => ({ + tier: t, + label: `${tierIcon(t)} ${capitalize(t)}`, + description: current[t] ? `current: ${current[t]}` : 'current: (default)', + detail: tierBlurb(t), + })); + + const picked = await vscode.window.showQuickPick(tierItems, { + placeHolder: 'Which difficulty tier do you want to configure?', + title: 'CoClaw: Select Tier Models', + }); + if (!picked) { return; } + tier = picked.tier; + } + + const before = modelManager.getTierFamily(tier); + const model = await modelManager.selectTierModelViaQuickPick(tier); + if (model) { + vscode.window.showInformationMessage( + `CoClaw: ${capitalize(tier)} tier set to ${model.name} (${model.family}).`, + ); + } else if (before !== undefined && modelManager.getTierFamily(tier) === undefined) { + // Re-read so we can tell whether the user actively cleared the + // override vs. just dismissed the QuickPick (Escape). + vscode.window.showInformationMessage( + `CoClaw: ${capitalize(tier)} tier override cleared.`, + ); + } + }); +} + +function tierIcon(t: TaskDifficulty): string { + switch (t) { + case 'light': return '$(zap)'; + case 'medium': return '$(symbol-method)'; + case 'hard': return '$(rocket)'; + } +} + +function tierBlurb(t: TaskDifficulty): string { + switch (t) { + case 'light': return 'Trivial / formulaic work: typos, rename, version bumps, mechanical formatting.'; + case 'medium': return 'Typical implementation, review, and test work. The default.'; + case 'hard': return 'Multi-file refactors, architectural design, security review, complex reasoning.'; + } +} + +function capitalize(s: string): string { + return s.length === 0 ? s : s[0].toUpperCase() + s.slice(1); +} diff --git a/src/commands/setModelCommands.ts b/src/commands/setModelCommands.ts new file mode 100644 index 0000000..d7982e3 --- /dev/null +++ b/src/commands/setModelCommands.ts @@ -0,0 +1,81 @@ +import * as vscode from 'vscode'; +import { ModelManager, TIERS } from '../lm/ModelManager'; +import type { TaskDifficulty } from '../agents/types'; +import { StatusBar } from '../ui/statusBar'; + +const VALID_TIERS: ReadonlySet = new Set(TIERS); + +/** + * Internal command invoked by chat buttons from `/model`. Persists the + * general model preference, refreshes the status bar, and surfaces a + * confirmation toast. Not contributed in `package.json#contributes.commands` + * on purpose — it's not meant for the Command Palette. + */ +export function registerSetModelFamilyCommand( + modelManager: ModelManager, + statusBar: StatusBar, +): vscode.Disposable { + return vscode.commands.registerCommand('CoClaw.setModelFamily', async (familyArg: unknown) => { + const family = typeof familyArg === 'string' ? familyArg.trim() : ''; + if (!family) { + vscode.window.showWarningMessage('CoClaw: missing model family.'); + return; + } + // Verify the family is actually available before we persist it so a + // bogus button payload doesn't silently mis-route every subsequent + // request to the default model via getModelForTier's fallback. + const models = await modelManager.getAvailableModels(true); + const match = models.find(m => m.family === family); + if (!match) { + vscode.window.showWarningMessage( + `CoClaw: model family "${family}" is not available from Copilot right now.`, + ); + return; + } + await modelManager.setPreferredFamily(family); + await statusBar.update(); + vscode.window.showInformationMessage( + `CoClaw: Switched to ${match.name} (${match.family}).`, + ); + }); +} + +/** + * Internal command invoked by chat buttons from `/model tier ...`. Persists + * the tier model and surfaces a confirmation toast. + */ +export function registerSetTierModelCommand( + modelManager: ModelManager, +): vscode.Disposable { + return vscode.commands.registerCommand( + 'CoClaw.setTierModel', + async (tierArg: unknown, familyArg: unknown) => { + const tier = typeof tierArg === 'string' ? tierArg.toLowerCase() : ''; + const family = typeof familyArg === 'string' ? familyArg.trim() : ''; + if (!VALID_TIERS.has(tier as TaskDifficulty)) { + vscode.window.showWarningMessage(`CoClaw: unknown tier "${tierArg}".`); + return; + } + if (!family) { + vscode.window.showWarningMessage('CoClaw: missing model family.'); + return; + } + const models = await modelManager.getAvailableModels(true); + const match = models.find(m => m.family === family); + if (!match) { + vscode.window.showWarningMessage( + `CoClaw: model family "${family}" is not available from Copilot right now.`, + ); + return; + } + await modelManager.setTierFamily(tier as TaskDifficulty, family); + vscode.window.showInformationMessage( + `CoClaw: ${capitalize(tier)} tier set to ${match.name} (${match.family}).`, + ); + }, + ); +} + +function capitalize(s: string): string { + return s.length === 0 ? s : s[0].toUpperCase() + s.slice(1); +} diff --git a/src/extension.ts b/src/extension.ts index c5c0f3b..87028ed 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -23,6 +23,8 @@ import { SharedMemoryStore } from './agents/SharedMemoryStore'; import { Orchestrator, SpawnerHolder } from './agents/Orchestrator'; import { AgentTreeProvider } from './ui/agentTreeProvider'; import { registerSelectModelCommand } from './commands/selectModel'; +import { registerSelectTierModelsCommand } from './commands/selectTierModels'; +import { registerSetModelFamilyCommand, registerSetTierModelCommand } from './commands/setModelCommands'; import { registerShowMemoryCommand } from './commands/showMemory'; import { registerClearMemoryCommand } from './commands/clearMemory'; import { registerEditSoulCommand, registerEditProfileCommand } from './commands/editSoul'; @@ -118,6 +120,9 @@ export function activate(context: vscode.ExtensionContext) { // Commands context.subscriptions.push( registerSelectModelCommand(modelManager, statusBar), + registerSelectTierModelsCommand(modelManager), + registerSetModelFamilyCommand(modelManager, statusBar), + registerSetTierModelCommand(modelManager), registerShowMemoryCommand(memoryPanel), registerClearMemoryCommand(memoryEngine), registerEditSoulCommand(soulConfig), diff --git a/src/lm/ModelManager.ts b/src/lm/ModelManager.ts index 65c9834..fb74b4f 100644 --- a/src/lm/ModelManager.ts +++ b/src/lm/ModelManager.ts @@ -1,12 +1,33 @@ import * as vscode from 'vscode'; +import type { TaskDifficulty } from '../agents/types'; +import { Logger } from '../util/Logger'; + +/** All known difficulty tiers. Single source of truth for UI iteration. */ +export const TIERS: readonly TaskDifficulty[] = ['light', 'medium', 'hard'] as const; + +/** + * Short cache for `vscode.lm.selectChatModels`. The Copilot model list rarely + * changes within a session and Telegram's settings UI can ask for it 4× when + * the user is flipping between tier panels. A few seconds is enough. + */ +const MODELS_CACHE_TTL_MS = 5_000; export class ModelManager { private static readonly SELECTED_FAMILY_KEY = 'selectedFamily'; + private modelsCache: { fetchedAt: number; models: vscode.LanguageModelChat[] } | undefined; + /** Track tier families we have already warned about being unavailable. */ + private readonly warnedMissingFamilies = new Set(); + constructor(private readonly globalState: vscode.Memento) {} - async getAvailableModels(): Promise { - return vscode.lm.selectChatModels({ vendor: 'copilot' }); + async getAvailableModels(forceRefresh = false): Promise { + if (!forceRefresh && this.modelsCache && Date.now() - this.modelsCache.fetchedAt < MODELS_CACHE_TTL_MS) { + return this.modelsCache.models; + } + const models = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + this.modelsCache = { fetchedAt: Date.now(), models }; + return models; } async getActiveModel(): Promise { @@ -23,12 +44,11 @@ export class ModelManager { } } - // Fallback to first available return models[0]; } async selectModelViaQuickPick(): Promise { - const models = await this.getAvailableModels(); + const models = await this.getAvailableModels(true); if (models.length === 0) { vscode.window.showWarningMessage('No Copilot models available.'); return undefined; @@ -60,7 +80,6 @@ export class ModelManager { } getPreferredFamily(): string | undefined { - // Workspace setting takes priority const workspaceSetting = vscode.workspace.getConfiguration('CoClaw.model').get('family'); if (workspaceSetting) { return workspaceSetting; @@ -71,4 +90,122 @@ export class ModelManager { async setPreferredFamily(family: string): Promise { await this.globalState.update(ModelManager.SELECTED_FAMILY_KEY, family); } + + // ── Tier-based model selection ────────────────────────────────────── + // + // Tier overrides are persisted exclusively through VS Code settings + // (`CoClaw.models.`). Both the Telegram /settings UI and the VS Code + // QuickPick write through this same path, so there is exactly one source + // of truth and no risk of two stores drifting apart. + + /** Return the model family configured for a given difficulty tier, if any. */ + getTierFamily(tier: TaskDifficulty): string | undefined { + const setting = vscode.workspace.getConfiguration('CoClaw.models').get(tier); + return setting && setting.trim().length > 0 ? setting : undefined; + } + + /** Persist the model family for a difficulty tier. */ + async setTierFamily(tier: TaskDifficulty, family: string | undefined): Promise { + await vscode.workspace.getConfiguration('CoClaw.models') + .update(tier, family && family.length > 0 ? family : undefined, vscode.ConfigurationTarget.Global); + } + + /** Get all tier assignments in a single pass (one config read per tier). */ + getAllTierFamilies(): Record { + const cfg = vscode.workspace.getConfiguration('CoClaw.models'); + const read = (t: TaskDifficulty): string | undefined => { + const v = cfg.get(t); + return v && v.trim().length > 0 ? v : undefined; + }; + return { light: read('light'), medium: read('medium'), hard: read('hard') }; + } + + /** + * Resolve the active model for a specific task difficulty tier. + * Falls back to `getActiveModel()` when no tier-specific override is set, + * or when the configured family isn't currently available (with a + * one-shot warning so silent fallback doesn't go unnoticed). + */ + async getModelForTier(tier: TaskDifficulty): Promise { + const tierFamily = this.getTierFamily(tier); + if (tierFamily) { + const models = await this.getAvailableModels(); + const match = models.find(m => m.family === tierFamily); + if (match) { + return match; + } + const warnKey = `${tier}:${tierFamily}`; + if (!this.warnedMissingFamilies.has(warnKey)) { + this.warnedMissingFamilies.add(warnKey); + Logger.warn( + 'ModelManager', + `Configured model family for tier '${tier}' is '${tierFamily}' but no such Copilot model is available; ` + + `falling back to the default model. Update 'CoClaw.models.${tier}' or run 'CoClaw: Select Tier Models'.`, + ); + } + } + return this.getActiveModel(); + } + + /** + * Show a QuickPick to assign a Copilot model to a single tier. + * Returns the picked model (or undefined if the user cancelled). + */ + async selectTierModelViaQuickPick(tier: TaskDifficulty): Promise { + const models = await this.getAvailableModels(true); + if (models.length === 0) { + vscode.window.showWarningMessage('No Copilot models available.'); + return undefined; + } + + const currentFamily = this.getTierFamily(tier); + // Dedupe by family so the picker doesn't show the same family twice. + const seen = new Set(); + const uniqueModels = models.filter(m => (seen.has(m.family) ? false : (seen.add(m.family), true))); + + type Item = vscode.QuickPickItem & { family?: string; clear?: boolean }; + const items: Item[] = [ + ...uniqueModels.map(m => ({ + label: m.name, + description: m.family, + detail: `Max tokens: ${m.maxInputTokens}${m.family === currentFamily ? ' • current' : ''}`, + family: m.family, + picked: m.family === currentFamily, + })), + { label: '$(close) Clear override', description: 'Use the default model for this tier', clear: true }, + ]; + + const selected = await vscode.window.showQuickPick(items, { + placeHolder: `Select model for ${tier}-difficulty tasks`, + title: `CoClaw: ${tier[0].toUpperCase() + tier.slice(1)} Tier Model`, + }); + if (!selected) { return undefined; } + + if (selected.clear) { + await this.setTierFamily(tier, undefined); + return undefined; + } + if (selected.family) { + await this.setTierFamily(tier, selected.family); + return models.find(m => m.family === selected.family); + } + return undefined; + } + + /** + * Walk all three tiers through QuickPick, in order. Useful for a single + * "configure tier models" command. + */ + async selectAllTierModelsViaQuickPick(): Promise { + for (const tier of TIERS) { + const picked = await this.selectTierModelViaQuickPick(tier); + // If the user dismissed (Escape) on any step, stop the wizard + // rather than silently advancing — matches VS Code conventions. + if (picked === undefined && this.getTierFamily(tier) === undefined) { + // user cancelled OR cleared — either way they made a choice; + // continue to next tier instead of aborting. + continue; + } + } + } } diff --git a/src/lm/modelLookup.ts b/src/lm/modelLookup.ts new file mode 100644 index 0000000..5d86f21 --- /dev/null +++ b/src/lm/modelLookup.ts @@ -0,0 +1,70 @@ +import type * as vscode from 'vscode'; + +/** + * Slim view of a Copilot model surface used by the chat /model command's + * name-matching helper. Mirrors the fields we actually read so unit tests + * can pass plain objects instead of mocking the full `LanguageModelChat`. + */ +export interface ModelLike { + name: string; + family: string; +} + +/** + * Resolve a user-typed model token to one (or more) Copilot models. + * + * Matching is intentionally forgiving: exact identifiers win, then case- + * insensitive equality, then `startsWith`, then substring. We bias toward + * `family` matches over `name` matches because the family id is the stable + * identifier we persist to settings. + * + * Returns `[]` when nothing plausibly matches. When multiple results tie at + * the same precedence level (e.g. "claude" matches multiple Claude models) + * the caller is expected to disambiguate via UI (buttons / picker), so the + * full set is returned rather than silently picking the first hit. + */ +export function matchModels(models: readonly T[], query: string): T[] { + const q = query.trim().toLowerCase(); + if (!q) { return []; } + + // 1. Exact family / name (case-insensitive). + const exactFamily = models.filter(m => m.family.toLowerCase() === q); + if (exactFamily.length > 0) { return uniqueByFamily(exactFamily); } + const exactName = models.filter(m => m.name.toLowerCase() === q); + if (exactName.length > 0) { return uniqueByFamily(exactName); } + + // 2. startsWith on family, then on name. + const startsFamily = models.filter(m => m.family.toLowerCase().startsWith(q)); + if (startsFamily.length > 0) { return uniqueByFamily(startsFamily); } + const startsName = models.filter(m => m.name.toLowerCase().startsWith(q)); + if (startsName.length > 0) { return uniqueByFamily(startsName); } + + // 3. substring on family or name. + const sub = models.filter(m => + m.family.toLowerCase().includes(q) || m.name.toLowerCase().includes(q), + ); + return uniqueByFamily(sub); +} + +/** + * Many Copilot families expose multiple `LanguageModelChat` instances with + * the same `family` (different `version`/`name` combos). For settings + * purposes we only care about distinct families. + */ +function uniqueByFamily(models: readonly T[]): T[] { + const seen = new Set(); + const out: T[] = []; + for (const m of models) { + if (!seen.has(m.family)) { + seen.add(m.family); + out.push(m); + } + } + return out; +} + +/** + * Re-export the LanguageModelChat type for callers that want a thin alias. + * Kept here so consumers don't need to import `vscode` just for the type. + */ +export type LanguageModelChat = vscode.LanguageModelChat; diff --git a/src/participant/handler.ts b/src/participant/handler.ts index 5787252..5e9092f 100644 --- a/src/participant/handler.ts +++ b/src/participant/handler.ts @@ -1,13 +1,20 @@ import * as vscode from 'vscode'; -import { ModelManager } from '../lm/ModelManager'; +import { ModelManager, TIERS } from '../lm/ModelManager'; import { PromptBuilder } from '../lm/PromptBuilder'; import { ToolRunner } from '../lm/ToolRunner'; import { ToolResultCache } from '../lm/ToolResultCache'; import { getAutonomousTools } from '../lm/toolFilter'; +import { matchModels } from '../lm/modelLookup'; import { MemoryEngine } from '../memory/MemoryEngine'; import { StatusBar } from '../ui/statusBar'; import { TelegramBot } from '../telegram/TelegramBot'; import { Orchestrator } from '../agents/Orchestrator'; +import type { TaskDifficulty } from '../agents/types'; + +const VALID_TIERS: ReadonlySet = new Set(TIERS); +function isValidTier(v: string): v is TaskDifficulty { + return VALID_TIERS.has(v as TaskDifficulty); +} export class ParticipantHandler { private readonly toolRunner = new ToolRunner(); @@ -190,12 +197,229 @@ export class ParticipantHandler { return this.handleOpenCommand(request, stream, token); case 'agents': return this.runOrchestrator(request, stream, token); + case 'model': + return this.handleModelCommand(request, stream); default: stream.markdown(`Unknown command: /${request.command}`); return {}; } } + /** + * `/model` chat command. Sub-syntax: + * + * /model → show status (general + per-tier) with one-click buttons + * /model list → list available Copilot model families + * /model help → usage reminder + * /model → switch the GENERAL model (e.g. "/model gpt-4o-mini") + * /model tier → open the QuickPick for that tier (light/medium/hard) + * /model tier → set the tier model directly (e.g. "/model tier hard claude-opus-4") + * /model clear → drop the general model override (revert to first available) + * /model clear → drop the tier override (revert to general default) + * + * All persistence flows through ModelManager (same path used by the + * settings GUI, the QuickPick, and Telegram /models) so every surface + * sees the change immediately. + */ + private async handleModelCommand( + request: vscode.ChatRequest, + stream: vscode.ChatResponseStream, + ): Promise { + const args = request.prompt.trim().split(/\s+/).filter(s => s.length > 0); + const sub = (args[0] ?? '').toLowerCase(); + + try { + if (args.length === 0) { + await this.renderModelStatus(stream); + return {}; + } + if (sub === 'help' || sub === '?') { + this.renderModelHelp(stream); + return {}; + } + if (sub === 'list' || sub === 'ls') { + await this.renderModelList(stream); + return {}; + } + if (sub === 'clear') { + await this.handleModelClear(stream, args[1]); + return {}; + } + if (sub === 'tier') { + await this.handleModelTier(stream, args[1], args.slice(2).join(' ')); + return {}; + } + // Default: treat the whole prompt as a model identifier. + await this.handleModelSwitch(stream, request.prompt.trim()); + return {}; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + stream.markdown(`❌ ${msg}\n`); + return {}; + } + } + + private async renderModelStatus(stream: vscode.ChatResponseStream): Promise { + const active = await this.modelManager.getActiveModel().catch(() => undefined); + const generalPref = this.modelManager.getPreferredFamily(); + const tiers = this.modelManager.getAllTierFamilies(); + + stream.markdown(`## 🤖 CoClaw model\n\n`); + if (active) { + stream.markdown(`**Active general model:** \`${active.name}\` (family \`${active.family}\`, ${active.maxInputTokens.toLocaleString()} tokens)\n`); + } else { + stream.markdown(`**Active general model:** _(no Copilot model available — install/sign in to Copilot)_\n`); + } + if (generalPref) { + stream.markdown(`Preferred family: \`${generalPref}\`\n`); + } else { + stream.markdown(`Preferred family: _(none — auto-pick first available)_\n`); + } + stream.markdown(`\n**Per-tier overrides (used by \`/agents\`):**\n`); + for (const t of TIERS) { + const v = tiers[t]; + stream.markdown(`- \`${t}\` → ${v ? `\`${v}\`` : '_(default — uses general model)_'}\n`); + } + stream.markdown(`\n`); + stream.button({ title: '🔧 Pick general model', command: 'CoClaw.selectModel' }); + for (const t of TIERS) { + stream.button({ + title: `🎯 Pick ${t} tier`, + command: 'CoClaw.selectTierModels', + arguments: [t], + }); + } + stream.markdown( + `\n_Quick commands:_ \`/model list\`, \`/model gpt-4o-mini\`, \`/model tier hard claude-opus-4\`, \`/model clear\`, \`/model clear hard\`\n`, + ); + } + + private renderModelHelp(stream: vscode.ChatResponseStream): void { + stream.markdown( + `### \`/model\` usage\n\n` + + `| Command | Effect |\n` + + `|---|---|\n` + + `| \`/model\` | Show current model and per-tier overrides |\n` + + `| \`/model list\` | List available Copilot model families |\n` + + `| \`/model \` | Switch the general model (e.g. \`/model gpt-4o-mini\`) |\n` + + `| \`/model tier \` | Open a picker for that tier (\`light\`/\`medium\`/\`hard\`) |\n` + + `| \`/model tier \` | Set tier model directly (e.g. \`/model tier hard claude-opus-4\`) |\n` + + `| \`/model clear\` | Drop the general override |\n` + + `| \`/model clear \` | Drop the tier override |\n`, + ); + } + + private async renderModelList(stream: vscode.ChatResponseStream): Promise { + const models = await this.modelManager.getAvailableModels(true); + if (models.length === 0) { + stream.markdown(`_No Copilot models available. Install / sign in to GitHub Copilot first._\n`); + return; + } + const seen = new Map(); + for (const m of models) { + if (!seen.has(m.family)) { seen.set(m.family, m); } + } + stream.markdown(`### Available Copilot model families (${seen.size})\n\n`); + for (const m of seen.values()) { + stream.markdown(`- \`${m.family}\` — **${m.name}** (${m.maxInputTokens.toLocaleString()} tokens)\n`); + } + stream.markdown(`\n_Switch with \`/model \` — e.g. \`/model ${[...seen.keys()][0]}\`._\n`); + } + + private async handleModelSwitch(stream: vscode.ChatResponseStream, query: string): Promise { + const models = await this.modelManager.getAvailableModels(true); + if (models.length === 0) { + stream.markdown(`_No Copilot models available. Install / sign in to GitHub Copilot first._\n`); + return; + } + const matches = matchModels(models, query); + if (matches.length === 0) { + stream.markdown(`No Copilot model matched \`${escapeInline(query)}\`.\n\n`); + await this.renderModelList(stream); + return; + } + if (matches.length === 1) { + const m = matches[0]; + await this.modelManager.setPreferredFamily(m.family); + await this.statusBar?.update(); + stream.markdown(`✅ Switched general model to **${m.name}** (\`${m.family}\`).\n`); + return; + } + // Ambiguous — render disambiguation buttons. + stream.markdown(`\`${escapeInline(query)}\` matched ${matches.length} models. Pick one:\n\n`); + for (const m of matches) { + stream.button({ + title: `${m.name} (${m.family})`, + command: 'CoClaw.setModelFamily', + arguments: [m.family], + }); + } + } + + private async handleModelTier( + stream: vscode.ChatResponseStream, + rawTier: string | undefined, + familyQuery: string, + ): Promise { + if (!rawTier) { + stream.markdown( + `Usage: \`/model tier [family]\`. Without a family, a picker opens.\n`, + ); + return; + } + const tier = rawTier.toLowerCase(); + if (!isValidTier(tier)) { + stream.markdown(`Unknown tier \`${escapeInline(rawTier)}\`. Use one of: \`light\`, \`medium\`, \`hard\`.\n`); + return; + } + if (!familyQuery) { + // Defer to the QuickPick — it owns the live model list. + await vscode.commands.executeCommand('CoClaw.selectTierModels', tier); + return; + } + const models = await this.modelManager.getAvailableModels(true); + if (models.length === 0) { + stream.markdown(`_No Copilot models available. Install / sign in to GitHub Copilot first._\n`); + return; + } + const matches = matchModels(models, familyQuery); + if (matches.length === 0) { + stream.markdown(`No Copilot model matched \`${escapeInline(familyQuery)}\`.\n\n`); + await this.renderModelList(stream); + return; + } + if (matches.length === 1) { + const m = matches[0]; + await this.modelManager.setTierFamily(tier, m.family); + stream.markdown(`✅ \`${tier}\` tier set to **${m.name}** (\`${m.family}\`).\n`); + return; + } + stream.markdown(`\`${escapeInline(familyQuery)}\` matched ${matches.length} models for the **${tier}** tier. Pick one:\n\n`); + for (const m of matches) { + stream.button({ + title: `${m.name} (${m.family})`, + command: 'CoClaw.setTierModel', + arguments: [tier, m.family], + }); + } + } + + private async handleModelClear(stream: vscode.ChatResponseStream, target: string | undefined): Promise { + if (!target || target.toLowerCase() === 'general' || target.toLowerCase() === 'family') { + await this.modelManager.setPreferredFamily(''); + await this.statusBar?.update(); + stream.markdown(`🗑 Cleared the general model preference. CoClaw will pick the first available Copilot model.\n`); + return; + } + const tier = target.toLowerCase(); + if (!isValidTier(tier)) { + stream.markdown(`Unknown clear target \`${escapeInline(target)}\`. Use \`/model clear\` (general) or \`/model clear \`.\n`); + return; + } + await this.modelManager.setTierFamily(tier, undefined); + stream.markdown(`🗑 Cleared the \`${tier}\` tier override. That tier will now use the general model.\n`); + } + private async handleMemoryCommand(stream: vscode.ChatResponseStream): Promise { const { daily, longterm } = await this.memoryEngine.getAllMemories(); @@ -384,3 +608,12 @@ export class ParticipantHandler { return Math.ceil(text.length / 4); } } + +/** + * Strip / escape characters that would close an inline-code span when we + * echo user-supplied tokens back inside backticks. Defense-in-depth: keeps + * a user typing `` ` `` or `` from poisoning the chat stream. + */ +function escapeInline(s: string): string { + return s.replace(/`/g, '').replace(/[\r\n]+/g, ' ').slice(0, 120); +} diff --git a/src/telegram/TelegramBot.ts b/src/telegram/TelegramBot.ts index 0f103c4..c3a7d43 100644 --- a/src/telegram/TelegramBot.ts +++ b/src/telegram/TelegramBot.ts @@ -472,6 +472,7 @@ export class TelegramBot { '`/stop` — Stop the Telegram bridge', '`/memory` — Show memory summary', '`/settings` — Open interactive settings UI (alias: `/s`)', + '`/models` — Pick the model for each difficulty tier (alias: `/m`)', ]; if (this._openMode) { helpLines.push( @@ -500,6 +501,10 @@ export class TelegramBot { await this.openSettingsPanel(chatId); return; } + if (text === '/models' || text === '/m') { + await this.openModelTiersPanel(chatId); + return; + } // Heartbeat commands (/open mode only) if (text.startsWith('/heartbeat') && this._openMode) { await this.handleHeartbeatCommand(chatId, text); @@ -1127,6 +1132,17 @@ export class TelegramBot { this.vscodeStream?.markdown(`> **Telegram:** /settings opened\n\n`); } + /** + * Shortcut that jumps straight into the "Model Tiers" group so the user + * can pick the model for each difficulty tier (light/medium/hard) without + * navigating /settings → group selection first. + */ + private async openModelTiersPanel(chatId: number): Promise { + const { text, buttons } = buildSettingsGroupPanel('Model Tiers'); + await this.api!.sendMessageWithButtons(chatId, text, buttons); + this.vscodeStream?.markdown(`> **Telegram:** /models opened\n\n`); + } + private async handleSettingsUiCallback( callbackQueryId: string, data: string, diff --git a/src/telegram/TelegramSettingsUi.ts b/src/telegram/TelegramSettingsUi.ts index 4b7e605..521a7a5 100644 --- a/src/telegram/TelegramSettingsUi.ts +++ b/src/telegram/TelegramSettingsUi.ts @@ -28,6 +28,28 @@ export interface SettingDefinition { dynamicOptions?: () => Promise; } +/** + * Discover the currently-available Copilot model families and surface them + * as dropdown buttons. Extracted once so the four "pick a model" settings + * (general model + 3 tier overrides) share a single implementation instead + * of duplicating the same try/catch + dedupe block. + * + * Failures (e.g. Copilot not installed) intentionally degrade to an empty + * list so the settings panel still renders without throwing. + */ +const copilotFamilyOptions = async (): Promise => { + try { + const models = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + const seen = new Map(); + for (const m of models) { + if (!seen.has(m.family)) { seen.set(m.family, m.name); } + } + return Array.from(seen.entries()).map(([family, name]) => ({ value: family, label: name })); + } catch { + return []; + } +}; + /** * Curated list of CoClaw settings exposed via the Telegram /settings UI. * Mirrors the contributions in package.json. @@ -65,19 +87,33 @@ export const SETTINGS: SettingDefinition[] = [ description: 'Preferred Copilot model family (blank = default)', type: 'string', group: 'Model', - dynamicOptions: async () => { - try { - const models = await vscode.lm.selectChatModels({ vendor: 'copilot' }); - // Deduplicate by family while keeping the first model's name as label. - const seen = new Map(); - for (const m of models) { - if (!seen.has(m.family)) { seen.set(m.family, m.name); } - } - return Array.from(seen.entries()).map(([family, name]) => ({ value: family, label: name })); - } catch { - return []; - } - }, + dynamicOptions: copilotFamilyOptions, + }, + + // Model Tiers — per-difficulty model overrides used by /agents orchestration. + { + key: 'CoClaw.models.light', + label: 'Light model', + description: 'Model for light/simple tasks (fast, cheap)', + type: 'string', + group: 'Model Tiers', + dynamicOptions: copilotFamilyOptions, + }, + { + key: 'CoClaw.models.medium', + label: 'Medium model', + description: 'Model for medium-complexity tasks (balanced)', + type: 'string', + group: 'Model Tiers', + dynamicOptions: copilotFamilyOptions, + }, + { + key: 'CoClaw.models.hard', + label: 'Hard model', + description: 'Model for hard/complex tasks (most capable)', + type: 'string', + group: 'Model Tiers', + dynamicOptions: copilotFamilyOptions, }, ]; diff --git a/src/test/coderSplitter.test.ts b/src/test/coderSplitter.test.ts index 6db8652..8eb4a67 100644 --- a/src/test/coderSplitter.test.ts +++ b/src/test/coderSplitter.test.ts @@ -88,4 +88,27 @@ describe('CoderSplitter splitCoderTask', () => { assert.strictEqual(r.replacements.length, 1); assert.strictEqual(r.replacements[0], t, 'non-coder tasks should be returned unchanged'); }); + + it('propagates parent difficulty into split child tasks', () => { + // Per-tier model routing relies on every fanned-out coder inheriting + // the parent's difficulty; if this regresses, all child coders silently + // run on the default tier instead of the requested one. + const t: SubTask = { ...task('build feature', ['ui.tsx', 'api.ts']), difficulty: 'hard' }; + const r = splitCoderTask(t, '', 4, 1); + assert.strictEqual(r.didSplit, true); + for (const child of r.replacements) { + assert.strictEqual(child.difficulty, 'hard', + `child ${child.id} should inherit parent's 'hard' difficulty`); + } + }); + + it('leaves child difficulty undefined when parent has none', () => { + const t = task('build feature', ['ui.tsx', 'api.ts']); + const r = splitCoderTask(t, '', 4, 1); + assert.strictEqual(r.didSplit, true); + for (const child of r.replacements) { + assert.strictEqual(child.difficulty, undefined, + 'absent parent difficulty must not be invented by the splitter'); + } + }); }); diff --git a/src/test/finalReviewer.test.ts b/src/test/finalReviewer.test.ts new file mode 100644 index 0000000..72b113f --- /dev/null +++ b/src/test/finalReviewer.test.ts @@ -0,0 +1,149 @@ +import * as assert from 'assert'; +import { injectFinalReviewer, isFinalReviewTask } from '../agents/Orchestrator'; +import { SubTask } from '../agents/types'; + +/** + * Unit tests for the pure, configurable injection of an automatic final + * reviewer at the end of an /agents plan. The orchestrator wires this up + * via `ensureFinalReviewer()`; here we exercise the pure function directly + * to keep tests free of vscode workspace configuration mocking. + */ +describe('injectFinalReviewer', () => { + function task(over: Partial & Pick): SubTask { + return { + prompt: 'do work', + dependsOn: [], + status: 'pending', + ...over, + }; + } + + it('is a no-op when mode is "off"', () => { + const tasks = [task({ id: 'c1', agent: 'coder' })]; + const out = injectFinalReviewer(tasks, 'off'); + assert.strictEqual(out.length, 1); + assert.strictEqual(out[0].id, 'c1'); + }); + + it('does nothing if there are no code-producing tasks (auto)', () => { + const tasks = [ + task({ id: 'm1', agent: 'memory' }), + task({ id: 'r1', agent: 'reviewer' }), + ]; + const out = injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(out.length, tasks.length); + assert.ok(!out.some(t => isFinalReviewTask(t)), 'no final reviewer should be added'); + }); + + it('does nothing if there are no code-producing tasks (always)', () => { + const tasks = [task({ id: 'm1', agent: 'memory' })]; + const out = injectFinalReviewer(tasks, 'always'); + assert.strictEqual(out.length, tasks.length); + assert.ok(!out.some(t => isFinalReviewTask(t))); + }); + + it('injects a final reviewer when mode=auto and no reviewer covers all coders', () => { + const tasks = [ + task({ id: 'c1', agent: 'coder' }), + task({ id: 'c2', agent: 'coder' }), + task({ id: 't1', agent: 'tester', dependsOn: ['c1', 'c2'] }), + ]; + const out = injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(out.length, 4); + const fr = out[out.length - 1]; + assert.ok(isFinalReviewTask(fr)); + assert.strictEqual(fr.agent, 'reviewer'); + assert.strictEqual(fr.difficulty, 'hard'); + // Depends on every code-producing task (coders + testers). + assert.deepStrictEqual([...fr.dependsOn].sort(), ['c1', 'c2', 't1']); + }); + + it('skips injection in auto mode when an existing reviewer transitively covers everything', () => { + const tasks = [ + task({ id: 'c1', agent: 'coder' }), + task({ id: 'c2', agent: 'coder' }), + task({ id: 't1', agent: 'tester', dependsOn: ['c1', 'c2'] }), + // The existing reviewer depends directly on t1 which transitively + // covers c1 and c2 — so a final reviewer would be redundant. + task({ id: 'r1', agent: 'reviewer', dependsOn: ['t1'] }), + ]; + const out = injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(out.length, tasks.length, 'should not append a final reviewer'); + assert.ok(!out.some(t => isFinalReviewTask(t))); + }); + + it('still injects in auto mode when the existing reviewer covers only some coders', () => { + const tasks = [ + task({ id: 'c1', agent: 'coder' }), + task({ id: 'c2', agent: 'coder' }), // <- not covered by r1 + task({ id: 'r1', agent: 'reviewer', dependsOn: ['c1'] }), + ]; + const out = injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(out.length, 4); + const fr = out[out.length - 1]; + assert.ok(isFinalReviewTask(fr)); + assert.deepStrictEqual([...fr.dependsOn].sort(), ['c1', 'c2']); + }); + + it('always-mode injects a final reviewer even when a covering reviewer already exists', () => { + const tasks = [ + task({ id: 'c1', agent: 'coder' }), + task({ id: 'r1', agent: 'reviewer', dependsOn: ['c1'] }), + ]; + const out = injectFinalReviewer(tasks, 'always'); + assert.strictEqual(out.length, 3); + const fr = out[out.length - 1]; + assert.ok(isFinalReviewTask(fr)); + assert.deepStrictEqual(fr.dependsOn, ['c1']); + }); + + it('chooses a unique id when "final-review" collides', () => { + const tasks = [ + task({ id: 'final-review', agent: 'coder' }), // intentionally collides + task({ id: 'final-review-2', agent: 'tester' }), // and so does this one + ]; + const out = injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(out.length, 3); + const fr = out[out.length - 1]; + assert.strictEqual(fr.id, 'final-review-3'); + assert.ok(isFinalReviewTask(fr)); + }); + + it('uses the same prompt body every time so model caches can reuse it', () => { + const a = injectFinalReviewer([task({ id: 'c1', agent: 'coder' })], 'auto'); + const b = injectFinalReviewer([task({ id: 'c1', agent: 'coder' })], 'always'); + const aFinal = a[a.length - 1]; + const bFinal = b[b.length - 1]; + assert.strictEqual(aFinal.prompt, bFinal.prompt); + assert.match(aFinal.prompt, /Final consolidated review/); + assert.match(aFinal.prompt, /APPROVED/); + assert.match(aFinal.prompt, /CHANGES_REQUESTED/); + }); + + it('does not mutate the input task list', () => { + const tasks = [task({ id: 'c1', agent: 'coder' })]; + const before = tasks.length; + injectFinalReviewer(tasks, 'auto'); + assert.strictEqual(tasks.length, before, 'input should be left untouched'); + }); +}); + +describe('isFinalReviewTask', () => { + it('matches the canonical final-review id', () => { + assert.ok(isFinalReviewTask({ id: 'final-review', agent: 'reviewer', prompt: '', dependsOn: [], status: 'pending' })); + }); + + it('matches numbered fallback ids like final-review-2', () => { + assert.ok(isFinalReviewTask({ id: 'final-review-2', agent: 'reviewer', prompt: '', dependsOn: [], status: 'pending' })); + assert.ok(isFinalReviewTask({ id: 'final-review-37', agent: 'reviewer', prompt: '', dependsOn: [], status: 'pending' })); + }); + + it('does not match other reviewer ids', () => { + assert.ok(!isFinalReviewTask({ id: 'reviewer-1', agent: 'reviewer', prompt: '', dependsOn: [], status: 'pending' })); + assert.ok(!isFinalReviewTask({ id: 'final-review-suffix', agent: 'reviewer', prompt: '', dependsOn: [], status: 'pending' })); + }); + + it('does not match non-reviewer agents even with a matching id', () => { + assert.ok(!isFinalReviewTask({ id: 'final-review', agent: 'coder', prompt: '', dependsOn: [], status: 'pending' })); + }); +}); diff --git a/src/test/modelLookup.test.ts b/src/test/modelLookup.test.ts new file mode 100644 index 0000000..ccccfd5 --- /dev/null +++ b/src/test/modelLookup.test.ts @@ -0,0 +1,73 @@ +import * as assert from 'assert'; +import { describe, it } from 'mocha'; +import { matchModels, ModelLike } from '../lm/modelLookup'; + +const MODELS: ModelLike[] = [ + { family: 'gpt-4o', name: 'GPT-4o' }, + { family: 'gpt-4o-mini', name: 'GPT-4o mini' }, + { family: 'gpt-5', name: 'GPT-5' }, + { family: 'gpt-5-mini', name: 'GPT-5 mini' }, + { family: 'claude-3.5-sonnet', name: 'Claude 3.5 Sonnet' }, + { family: 'claude-opus-4', name: 'Claude Opus 4' }, + { family: 'gemini-2.5-pro', name: 'Gemini 2.5 Pro' }, +]; + +describe('matchModels', () => { + it('returns [] for an empty query', () => { + assert.deepStrictEqual(matchModels(MODELS, ''), []); + assert.deepStrictEqual(matchModels(MODELS, ' '), []); + }); + + it('prefers exact family match (case-insensitive)', () => { + const r = matchModels(MODELS, 'GPT-4o'); + assert.strictEqual(r.length, 1); + assert.strictEqual(r[0].family, 'gpt-4o', 'exact family must win over startsWith/substring'); + }); + + it('prefers exact name match when family does not match exactly', () => { + const r = matchModels(MODELS, 'Claude Opus 4'); + assert.strictEqual(r.length, 1); + assert.strictEqual(r[0].family, 'claude-opus-4'); + }); + + it('falls back to startsWith on family', () => { + const r = matchModels(MODELS, 'gpt-5'); + // gpt-5 is an exact family match → exactly one result + assert.strictEqual(r.length, 1); + assert.strictEqual(r[0].family, 'gpt-5'); + }); + + it('returns multiple startsWith matches when a prefix is ambiguous', () => { + // "claude" doesn't exact-match anything, but startsWith on family + // matches both claude-3.5-sonnet and claude-opus-4. + const r = matchModels(MODELS, 'claude'); + assert.strictEqual(r.length, 2); + const families = r.map(m => m.family).sort(); + assert.deepStrictEqual(families, ['claude-3.5-sonnet', 'claude-opus-4']); + }); + + it('falls back to substring matching as the last resort', () => { + // Substring tier deliberately scans both family AND name, so a query + // like "sonnet" hits the family containing "sonnet" anywhere — not + // just the prefix. (We use "sonnet" instead of "mini" because + // "Gemini" also contains "mini" in its name, which would correctly + // but unhelpfully widen the test.) + const r = matchModels(MODELS, 'sonnet'); + const families = r.map(m => m.family); + assert.deepStrictEqual(families, ['claude-3.5-sonnet']); + }); + + it('returns [] when nothing plausibly matches', () => { + assert.deepStrictEqual(matchModels(MODELS, 'phantom-model-xyz'), []); + }); + + it('deduplicates by family when the input lists the same family twice', () => { + const dup: ModelLike[] = [ + { family: 'gpt-4o', name: 'GPT-4o' }, + { family: 'gpt-4o', name: 'GPT-4o (preview)' }, + ]; + const r = matchModels(dup, 'gpt-4o'); + assert.strictEqual(r.length, 1, 'duplicate families must collapse'); + assert.strictEqual(r[0].family, 'gpt-4o'); + }); +}); diff --git a/src/test/modelManagerTiers.test.ts b/src/test/modelManagerTiers.test.ts new file mode 100644 index 0000000..cf53585 --- /dev/null +++ b/src/test/modelManagerTiers.test.ts @@ -0,0 +1,113 @@ +import * as assert from 'assert'; +import { describe, it, afterEach } from 'mocha'; +import * as vscode from 'vscode'; +import { ModelManager } from '../lm/ModelManager'; + +/** + * Tier-based model resolution is the critical contract of ModelManager: when + * a tier override is set we must hand back that model; when it isn't, or when + * the configured family isn't available, we must fall back to the default + * model. These tests stub `vscode.workspace.getConfiguration` and + * `vscode.lm.selectChatModels` directly because the production code reads + * from both stores on every resolution. + */ +describe('ModelManager tier resolution', () => { + const originalGetConfig = vscode.workspace.getConfiguration; + const originalSelectModels = vscode.lm.selectChatModels; + + function stubConfig(values: Partial> & { family?: string } = {}): void { + (vscode.workspace as any).getConfiguration = (section?: string) => ({ + get: (key: string, defaultValue?: T): T => { + if (section === 'CoClaw.models' && (key === 'light' || key === 'medium' || key === 'hard')) { + return (values[key] as T) ?? (defaultValue as T); + } + if (section === 'CoClaw.model' && key === 'family') { + return (values.family as T) ?? (defaultValue as T); + } + return defaultValue as T; + }, + update: async () => {}, + }); + } + + function stubModels(families: string[]): void { + (vscode.lm as any).selectChatModels = async () => families.map(family => ({ + name: family.toUpperCase(), + family, + maxInputTokens: 128_000, + sendRequest: async () => ({ stream: (async function* () {})() }), + })); + } + + function makeMemento(): vscode.Memento { + const store = new Map(); + const memento: any = { + get: (key: string, def?: unknown) => store.has(key) ? store.get(key) : def, + update: async (key: string, value: unknown) => { store.set(key, value); }, + keys: () => Array.from(store.keys()), + }; + return memento as vscode.Memento; + } + + afterEach(() => { + (vscode.workspace as any).getConfiguration = originalGetConfig; + (vscode.lm as any).selectChatModels = originalSelectModels; + }); + + it('getTierFamily returns the configured family for a tier', () => { + stubConfig({ light: 'gpt-4o-mini', hard: 'claude-3.5-sonnet' }); + const mm = new ModelManager(makeMemento()); + assert.strictEqual(mm.getTierFamily('light'), 'gpt-4o-mini'); + assert.strictEqual(mm.getTierFamily('hard'), 'claude-3.5-sonnet'); + assert.strictEqual(mm.getTierFamily('medium'), undefined); + }); + + it('getTierFamily ignores blank / whitespace-only settings', () => { + // Important: VS Code returns "" for an unset string contribution, so + // we must NOT treat it as a configured family or every tier would + // claim to be overridden. + stubConfig({ light: '', medium: ' ' as any }); + const mm = new ModelManager(makeMemento()); + assert.strictEqual(mm.getTierFamily('light'), undefined); + assert.strictEqual(mm.getTierFamily('medium'), undefined); + }); + + it('getModelForTier returns the configured tier model when available', async () => { + stubConfig({ hard: 'claude-3.5-sonnet' }); + stubModels(['gpt-4o-mini', 'gpt-4o', 'claude-3.5-sonnet']); + const mm = new ModelManager(makeMemento()); + const m = await mm.getModelForTier('hard'); + assert.strictEqual((m as any).family, 'claude-3.5-sonnet'); + }); + + it('getModelForTier falls back to the default model when configured family is unavailable', async () => { + // User configured a family that Copilot no longer exposes; we must + // not throw, but we must NOT silently keep picking the wrong model + // either — the fallback path uses getActiveModel which honors the + // preferred family / first available model. + stubConfig({ hard: 'phantom-model' }); + stubModels(['gpt-4o', 'gpt-4o-mini']); + const mm = new ModelManager(makeMemento()); + const m = await mm.getModelForTier('hard'); + assert.strictEqual((m as any).family, 'gpt-4o', 'should fall back to first available model'); + }); + + it('getModelForTier falls back when no override is configured', async () => { + stubConfig({}); + stubModels(['gpt-4o', 'gpt-4o-mini']); + const mm = new ModelManager(makeMemento()); + const m = await mm.getModelForTier('medium'); + assert.strictEqual((m as any).family, 'gpt-4o'); + }); + + it('getAllTierFamilies returns the current snapshot of all three tiers', () => { + stubConfig({ light: 'gpt-4o-mini', medium: 'gpt-4o', hard: 'claude-3.5-sonnet' }); + const mm = new ModelManager(makeMemento()); + const all = mm.getAllTierFamilies(); + assert.deepStrictEqual(all, { + light: 'gpt-4o-mini', + medium: 'gpt-4o', + hard: 'claude-3.5-sonnet', + }); + }); +}); diff --git a/src/test/orchestratorPlan.test.ts b/src/test/orchestratorPlan.test.ts index 4403678..9d6c984 100644 --- a/src/test/orchestratorPlan.test.ts +++ b/src/test/orchestratorPlan.test.ts @@ -83,6 +83,56 @@ describe('Orchestrator.parsePlan', () => { const plan = orch.parsePlan(text); assert.strictEqual(plan, undefined); }); + + it('parses a valid `difficulty` field on each task', () => { + const orch = makeOrchestrator(); + const text = JSON.stringify({ + tasks: [ + { id: 't1', agent: 'coder', prompt: 'fix typo', difficulty: 'light', dependsOn: [] }, + { id: 't2', agent: 'reviewer', prompt: 'audit', difficulty: 'hard', dependsOn: ['t1'] }, + { id: 't3', agent: 'tester', prompt: 'add test', difficulty: 'medium', dependsOn: ['t2'] }, + ], + }); + const plan = orch.parsePlan(text); + assert.ok(plan); + assert.strictEqual(plan!.tasks[0].difficulty, 'light'); + assert.strictEqual(plan!.tasks[1].difficulty, 'hard'); + assert.strictEqual(plan!.tasks[2].difficulty, 'medium'); + }); + + it('drops invalid `difficulty` values instead of poisoning the task', () => { + // Critical: an unknown difficulty string MUST NOT survive into the + // SubTask, otherwise getModelForTier() would either explode or fall + // through silently to the default tier without warning. + const orch = makeOrchestrator(); + const text = JSON.stringify({ + tasks: [ + { id: 't1', agent: 'coder', prompt: 'build', difficulty: 'hardest', dependsOn: [] }, + { id: 't2', agent: 'coder', prompt: 'build 2', difficulty: 42, dependsOn: [] }, + { id: 't3', agent: 'coder', prompt: 'build 3', dependsOn: [] }, + ], + }); + const plan = orch.parsePlan(text); + assert.ok(plan); + for (const t of plan!.tasks) { + assert.strictEqual(t.difficulty, undefined, + `task ${t.id} should have undefined difficulty for invalid/missing values`); + } + }); + + it('normalizes `difficulty` casing so "HARD" still routes to the hard tier', () => { + const orch = makeOrchestrator(); + const text = JSON.stringify({ + tasks: [ + { id: 't1', agent: 'coder', prompt: 'build', difficulty: 'HARD', dependsOn: [] }, + { id: 't2', agent: 'coder', prompt: 'fix', difficulty: ' Light ', dependsOn: [] }, + ], + }); + const plan = orch.parsePlan(text); + assert.ok(plan); + assert.strictEqual(plan!.tasks[0].difficulty, 'hard'); + assert.strictEqual(plan!.tasks[1].difficulty, 'light'); + }); }); describe('Orchestrator.hasCycle', () => { diff --git a/src/tools/spawnAgentTool.ts b/src/tools/spawnAgentTool.ts index adfa87d..2297239 100644 --- a/src/tools/spawnAgentTool.ts +++ b/src/tools/spawnAgentTool.ts @@ -1,12 +1,20 @@ import * as vscode from 'vscode'; -import { AgentRole } from '../agents/types'; +import { AgentRole, TaskDifficulty } from '../agents/types'; + +const VALID_DIFFICULTIES: ReadonlySet = new Set(['light', 'medium', 'hard']); export interface AgentSpawner { /** * Spawn a new agent task within the active run. Returns the new task id. * Resolves once the dynamic task has finished (done or failed). */ - spawnDynamicTask(runId: string, role: AgentRole, prompt: string, dependsOn?: string[]): Promise<{ taskId: string; status: string; output?: string; error?: string }>; + spawnDynamicTask( + runId: string, + role: AgentRole, + prompt: string, + dependsOn?: string[], + difficulty?: TaskDifficulty, + ): Promise<{ taskId: string; status: string; output?: string; error?: string }>; } interface Input { @@ -14,6 +22,7 @@ interface Input { agent: AgentRole; prompt: string; dependsOn?: string[]; + difficulty?: string; } export class SpawnAgentTool implements vscode.LanguageModelTool { @@ -23,7 +32,7 @@ export class SpawnAgentTool implements vscode.LanguageModelTool { options: vscode.LanguageModelToolInvocationOptions, _token: vscode.CancellationToken, ): Promise { - const { runId, agent, prompt, dependsOn } = options.input; + const { runId, agent, prompt, dependsOn, difficulty } = options.input; const sp = this.spawner.current; if (!sp) { return new vscode.LanguageModelToolResult([ @@ -40,8 +49,23 @@ export class SpawnAgentTool implements vscode.LanguageModelTool { new vscode.LanguageModelTextPart(`Error: cannot spawn ${agent} as a dynamic task.`), ]); } + // Validate the optional `difficulty` field against the allow-list so + // unknown values fall through to the default tier instead of + // poisoning per-task model routing downstream. + let coercedDifficulty: TaskDifficulty | undefined; + if (difficulty !== undefined) { + const v = String(difficulty).trim().toLowerCase(); + if (!VALID_DIFFICULTIES.has(v as TaskDifficulty)) { + return new vscode.LanguageModelToolResult([ + new vscode.LanguageModelTextPart( + `Error: invalid "difficulty" value "${difficulty}". Expected one of: light, medium, hard.`, + ), + ]); + } + coercedDifficulty = v as TaskDifficulty; + } try { - const result = await sp.spawnDynamicTask(runId, agent, prompt, dependsOn ?? []); + const result = await sp.spawnDynamicTask(runId, agent, prompt, dependsOn ?? [], coercedDifficulty); return new vscode.LanguageModelToolResult([ new vscode.LanguageModelTextPart( `Spawned ${agent} task ${result.taskId} -> ${result.status}\n${result.output ?? result.error ?? ''}`