diff --git a/docs/gentle-agents-activity.md b/docs/gentle-agents-activity.md index ea4bca402..6f8bd0b03 100644 --- a/docs/gentle-agents-activity.md +++ b/docs/gentle-agents-activity.md @@ -578,6 +578,12 @@ Pi's `setWidget` is the only fire-and-forget RPC push structured enough to carry Tasks are ordered `running`, `waiting`, `queued`, then finished tasks by `endedAt` descending (most recently finished first). +### Terminal error hints + +A terminal assistant failure keeps the `assistant reported an error` prefix. When a string `errorMessage` has a recognized marker in its first 2048 UTF-16 code units, the diagnostic appends one fixed hint: `[hint: quota]`, `[hint: rate_limit]`, `[hint: authentication]`, `[hint: model_unavailable]`, `[hint: timeout]`, or `[hint: transport]`, in that precedence. No provider text, credentials, URLs or request IDs are copied into the diagnostic. Unknown, missing or non-string details keep the original generic message. + +Hints are best-effort classifications, not verified provider identity or root-cause evidence. Failed status and settlement behavior are unchanged; this does not change streaming-error handling, retry a task, or reconfigure its model. + ## Bounds Every bound below fails closed: a value that cannot fit is truncated or dropped, and `lib/agents-rpc-publisher.ts`'s `encodeActivityLines` never throws. diff --git a/lib/agents-protocol.ts b/lib/agents-protocol.ts index 82935e33c..50b6c5385 100644 --- a/lib/agents-protocol.ts +++ b/lib/agents-protocol.ts @@ -185,15 +185,34 @@ function keepTail(text: string, max: number): string { return text.length <= max ? text : `${ELLIPSIS}${text.slice(text.length - max + 1)}`; } +const TERMINAL_ERROR_SCAN_MAX = 2_048; +const TERMINAL_ERROR_HINTS = [ + ["quota", /insufficient_quota|quota_exceeded|usage_limit_reached|usage limit|quota exceeded/], + ["rate_limit", /rate[_ -]?limit|too many requests/], + ["authentication", /invalid_api_key|authentication failed|authentication error|unauthorized/], + ["model_unavailable", /model_not_found|unknown model|unsupported model|model (?:not found|unavailable)/], + ["timeout", /timeout|timed out/], + ["transport", /websocket|econnreset|econnrefused|enotfound|fetch failed|connection (?:closed|reset|refused)/], +] as const; + +function terminalErrorDiagnostic(errorMessage: unknown): string { + const generic = "assistant reported an error"; + if (typeof errorMessage !== "string") return generic; + // Inspect a bounded prefix, but retain only a fixed label: no provider text, + // URLs, credentials or request identifiers may enter task state or output. + const excerpt = errorMessage.slice(0, TERMINAL_ERROR_SCAN_MAX).toLowerCase(); + const hint = TERMINAL_ERROR_HINTS.find(([, pattern]) => pattern.test(excerpt))?.[0]; + return hint ? `${generic} [hint: ${hint}]` : generic; +} + function terminalAssistant(messages: unknown): Omit { if (!Array.isArray(messages)) return { text: "", outcome: "empty", diagnostic: "assistant returned no final report" }; for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index] as { role?: string; content?: unknown; stopReason?: unknown }; + const message = messages[index] as { role?: string; content?: unknown; stopReason?: unknown; errorMessage?: unknown }; if (message?.role !== "assistant") continue; const stopReason = clean(message.stopReason).toLowerCase(); - // Do not preserve unbounded provider error payloads. The terminal reason is - // enough for an operator to distinguish failure from an empty report. - if (stopReason === "error") return { text: "", outcome: "error", diagnostic: "assistant reported an error" }; + // Preserve the generic prefix and only a bounded, allowlisted hint. + if (stopReason === "error") return { text: "", outcome: "error", diagnostic: terminalErrorDiagnostic(message.errorMessage) }; if (stopReason === "aborted") return { text: "", outcome: "aborted", diagnostic: "assistant aborted" }; const text = contentText(message.content); return text.length > 0 diff --git a/tests/agents-protocol.test.ts b/tests/agents-protocol.test.ts index 34c28512f..f38682d83 100644 --- a/tests/agents-protocol.test.ts +++ b/tests/agents-protocol.test.ts @@ -99,7 +99,7 @@ test("normalizeRpcEvent maps pi RPC events to task deltas and ignores the rest", assert.deepEqual(normalizeRpcEvent({ type: "tool_execution_end", toolCallId: "c1", toolName: "bash", isError: true, result: { content: [{ type: "text", text: "boom" }] } }), [{ type: TASK_EVENT.TOOL_END, callId: "c1", output: "boom", isError: true }]); assert.deepEqual(normalizeRpcEvent({ type: "turn_end" }), [{ type: TASK_EVENT.TURN_END }]); assert.deepEqual(normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text: "done." }], stopReason: "stop" }] }), [{ type: TASK_EVENT.AGENT_END, text: "done.", outcome: "success" }]); - assert.deepEqual(normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [], stopReason: "error", errorMessage: "WebSocket error with untrusted provider payload" }] }), [{ type: TASK_EVENT.AGENT_END, text: "", outcome: "error", diagnostic: "assistant reported an error" }]); + assert.deepEqual(normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [], stopReason: "error", errorMessage: "WebSocket error with untrusted provider payload" }] }), [{ type: TASK_EVENT.AGENT_END, text: "", outcome: "error", diagnostic: "assistant reported an error [hint: transport]" }]); assert.deepEqual(normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [], stopReason: "aborted" }] }), [{ type: TASK_EVENT.AGENT_END, text: "", outcome: "aborted", diagnostic: "assistant aborted" }]); assert.deepEqual(normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [], stopReason: "stop" }] }), [{ type: TASK_EVENT.AGENT_END, text: "", outcome: "empty", diagnostic: "assistant returned no final report" }]); assert.deepEqual(normalizeRpcEvent({ type: "agent_settled" }), [{ type: TASK_EVENT.AGENT_SETTLED }]); @@ -114,6 +114,33 @@ test("normalizeRpcEvent maps pi RPC events to task deltas and ignores the rest", assert.deepEqual(normalizeRpcEvent("garbage"), []); }); +test("terminal provider failures expose only bounded allowlisted hints", () => { + const generic = "assistant reported an error"; + const cases: Array<[unknown, string | undefined]> = [ + ["insufficient_quota", "quota"], + ["RATE LIMIT exceeded", "rate_limit"], + ["invalid_api_key", "authentication"], + ["model_not_found", "model_unavailable"], + ["WebSocket timed out", "timeout"], + ["WebSocket connection failed", "transport"], + ["opaque provider failure", undefined], + [undefined, undefined], + [{ message: "invalid_api_key", secret: "private-marker" }, undefined], + ["x".repeat(2048 - "websocket".length) + "websocket", "transport"], + ["x".repeat(2048) + "websocket", undefined], + ]; + for (const [value, hint] of cases) { + const errorMessage = typeof value === "string" + ? `${value} https://example.invalid/?secret=private-marker Authorization: Bearer sk-test-private-marker` + : value; + const diagnostic = hint ? `${generic} [hint: ${hint}]` : generic; + const events = normalizeRpcEvent({ type: "agent_end", messages: [{ role: "assistant", content: [], stopReason: "error", errorMessage }] }); + assert.deepEqual(events, [{ type: TASK_EVENT.AGENT_END, text: "", outcome: "error", diagnostic }]); + assert.ok(diagnostic.length <= 80, "only a bounded fixed-label diagnostic is emitted"); + assert.doesNotMatch(JSON.stringify(events), /private-marker|sk-test|example\.invalid|secret=/); + } +}); + // #1690: Pi drops unknown --tools names silently, so the child reports them // with one marked notify; only that notify reaches the task thread. test("normalizeRpcEvent turns the marked missing-tools notify into a note and drops every other notify", () => { diff --git a/tests/agents-runner.test.ts b/tests/agents-runner.test.ts index e402e2f00..2b7ca8992 100644 --- a/tests/agents-runner.test.ts +++ b/tests/agents-runner.test.ts @@ -973,6 +973,11 @@ test("AgentRunner classifies terminal assistant outcomes only after settlement", assert.equal(finished.status, scenario.status, scenario.name); if (scenario.error) assert.match(finished.error ?? "", scenario.error); else assert.equal(finished.result, "final report"); + if (scenario.name === "error") { + assert.equal(finished.error, "assistant reported an error [hint: transport]"); + assert.equal(finished.result, null); + assert.doesNotMatch(JSON.stringify(finished), /secret=|never-copy|WebSocket/); + } } });