From 38385b8261725f5eca7865294c1c1905428e26f7 Mon Sep 17 00:00:00 2001 From: webkubor Date: Thu, 5 Nov 2026 18:27:58 +0800 Subject: [PATCH] fovea 0.2.5: Gemini support, harness blocks are not the user's words, legacy function role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects, each with a regression test: 1. Gemini requests (contents) were never handled at all — the body passed through untouched, so installing the plugin saved nothing for Gemini clients. Now thoughts and thoughtSignatures are left alone, functionCall is left alone, a functionResponse keeps its id, name and keys while its long strings are cut, images become a note, and documents and video payloads are left alone. 2. saccade "auto" read Claude Code's (token budget, tool guidance) as the user's new question. It shares no words with the task, so the overlap test said "new topic" and the whole history collapsed. Harness-injected blocks are now stripped before the topic is judged, and a user message that is nothing but a harness block does not count as the user having said anything. 3. leadingUserEnd did not know the Responses answer items (custom_tool_call, computer_call, local_shell_call), so on a body whose first answer was a custom_tool_call the preamble exemption swallowed the history and nothing was cut. Also: the legacy functions API's role "function" message was never handled either — that is the oldest clients' tool output, the biggest cost. fovea.protocols.test.mjs adds 12 cases covering the four protocols' shapes and edges (Anthropic Messages, OpenAI Responses, Chat Completions, Gemini). bun test packages/fovea: 57 tests, 0 fail. --- packages/fovea/CHANGELOG.md | 16 ++ packages/fovea/README.md | 4 +- packages/fovea/fovea.middleware.js | 160 ++++++++++- packages/fovea/fovea.protocols.test.mjs | 359 ++++++++++++++++++++++++ packages/fovea/package.json | 3 +- packages/fovea/page/measure.js | 160 ++++++++++- 6 files changed, 683 insertions(+), 19 deletions(-) create mode 100644 packages/fovea/fovea.protocols.test.mjs diff --git a/packages/fovea/CHANGELOG.md b/packages/fovea/CHANGELOG.md index 6e4aaad5c..4b6080598 100644 --- a/packages/fovea/CHANGELOG.md +++ b/packages/fovea/CHANGELOG.md @@ -2,6 +2,22 @@ 本文件记录 fovea 的对外变化。格式遵循 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/)，版本遵循语义化版本。 +## [0.2.5] — 2026-11-05 + +### Added + +- 支持 Gemini 请求（`contents` 字段）：thought 与 thoughtSignature、functionCall 不动；functionResponse 保留 id / name / 键，其中的长字符串被裁；图片换成注记；文档与视频载荷不动。此前 Gemini 请求完全原样放行，装了也不省。 + +### Fixed + +- `saccade: "auto"` 不再把 Claude Code 的 ``（token 预算、工具指引）当成用户的新问题。它与任务没有共同词，旧规则判成换话题、整段历史折叠。 +- Responses 请求的首个答案是 `custom_tool_call` / `computer_call` / `local_shell_call` 时，任务豁免范围算错，整条历史一条都不裁。 +- Chat 旧版 functions API 的 `role: "function"` 消息此前完全没被处理，那正是最占钱的工具输出。 + +### Changed + +- 新增 `fovea.protocols.test.mjs`：四种协议各自的形状与边界（Anthropic Messages、OpenAI Responses、Chat Completions、Gemini），12 条用例。 + ## [0.2.4] — 2026-11-05 ### Fixed diff --git a/packages/fovea/README.md b/packages/fovea/README.md index 9e1f9461d..e7fce3825 100644 --- a/packages/fovea/README.md +++ b/packages/fovea/README.md @@ -7,7 +7,7 @@ **magpie 网关中间件：最近窗口看得清，之前的旧历史只带轮廓。** [![License: MIT](https://img.shields.io/badge/License-MIT-2f63a2.svg?style=for-the-badge)](LICENSE) -[![Tests](https://img.shields.io/badge/tests-45%2F45-brightgreen?style=for-the-badge)](#证据三层各管什么) +[![Tests](https://img.shields.io/badge/tests-57%2F57-brightgreen?style=for-the-badge)](#证据三层各管什么) [![Platform](https://img.shields.io/badge/magpie-middleware-8a6bbf?style=for-the-badge)](https://usemagpie.ai) [![Runtime](https://img.shields.io/badge/runtime-moejs%20%C2%B7%20zero%20deps-4a5568?style=for-the-badge)](fovea.middleware.js) @@ -78,7 +78,7 @@ git clone https://github.com/magpie-community/plugins && bun test packages/fovea ## 证据三层，各管什么 -1. **规则正确性** —— `bun test`：45 个确定性用例（离线、零依赖）。覆盖三种形状的工具配对、protect 在两种模式下生效、代理对不劈开、document 块不动、标记幂等、连续两轮已裁前缀逐字节相同。 +1. **规则正确性** —— `bun test`：57 个确定性用例（离线、零依赖）。覆盖三种形状的工具配对、protect 在两种模式下生效、代理对不劈开、document 块不动、标记幂等、连续两轮已裁前缀逐字节相同。 2. **同一请求确实变小** —— `bun benchmark.mjs [request.json] [options]`，或直接在证据页粘贴你的请求。合成样例：47,777 → 14,148 字符，已裁的 27 条消息逐字稳定。 3. **真实 token / 缓存 / 费用** —— `FOVEA_GATEWAY_KEY=<临时key> bun live-benchmark.mjs`：同模型、同请求、ABBA 四连发，记录供应商返回的 input / cache_read / output token 与耗时，只看暖缓存的第二次。0.1.0 时代的一次实测：163,143 → 42,358 prompt tokens，中位延迟 7.1s → 2.5s（因历史形状与供应商缓存而异）。**账单以 `magpie usage --csv` 的 `cost_usd` 为准；字符百分比不是美元。** diff --git a/packages/fovea/fovea.middleware.js b/packages/fovea/fovea.middleware.js index 72a4c557f..2b0c533d6 100644 --- a/packages/fovea/fovea.middleware.js +++ b/packages/fovea/fovea.middleware.js @@ -86,7 +86,7 @@ export function onRequest(body, ctx) { // carries body.input. Size is measured by walking, not by // stringifying: a 5MB body would otherwise allocate 5MB of string on // every request, on the request path. - const list = Array.isArray(body.messages) ? body.messages : Array.isArray(body.input) ? body.input : null + const list = Array.isArray(body.messages) ? body.messages : Array.isArray(body.input) ? body.input : Array.isArray(body.contents) ? body.contents : null if (!list) return if (cfg.threshold > 0 && sizeOf(body) < cfg.threshold) return @@ -115,6 +115,7 @@ export function onRequest(body, ctx) { const item = list[i] if (cfg.stateMarker && carriesState(item, cfg.stateMarker)) continue if (list === body.input) changed = (saccade ? collapseInput : slimInput)(item, cfg) || changed + else if (list === body.contents) changed = (saccade ? collapseGemini : slimGemini)(item, cfg) || changed else changed = (saccade ? collapseMessage : slimMessage)(item, cfg) || changed } return changed ? body : undefined @@ -147,12 +148,28 @@ function windowStart(list, keepCount, keepTokens) { // assistant reply, a function_call or a reasoning item, whichever comes // first — is that preamble and stays whole; the history proper begins at // the answer. +// The Responses API marks an answer with the item's type, not the role: +// a function_call, a custom_tool_call, a computer_call, a local_shell_call +// and their outputs are all "the model spoke". Leaving them out made the +// preamble exemption swallow the whole history on a body whose first +// answer was a custom_tool_call (2026-11-05). +const ANSWER_ITEMS = new Set([ + "function_call", "function_call_output", + "custom_tool_call", "custom_tool_call_output", + "computer_call", "computer_call_output", + "local_shell_call", "local_shell_call_output", + "web_search_call", "reasoning", +]) + function leadingUserEnd(list, start) { for (let i = 0; i < start; i++) { const it = list[i] if (!it || typeof it !== "object") continue - if (it.role === "assistant" || it.role === "tool" || it.type === "function_call" || - it.type === "reasoning" || it.type === "function_call_output") return i + // Gemini's model turns carry role "model"; its function responses ride + // in a user turn's parts, so the role alone cannot mark the answer. + if (it.role === "assistant" || it.role === "model" || it.role === "tool") return i + if (typeof it.type === "string" && ANSWER_ITEMS.has(it.type)) return i + if (Array.isArray(it.parts) && it.parts.some((p) => p && p.functionCall)) return i } return start } @@ -175,6 +192,13 @@ function hasStateBlock(list, marker) { function carriesState(item, marker) { if (!item || typeof item !== "object" || !marker) return false + // Gemini's model turn is the agent's own message, like an assistant one. + if (item.role === "model" && Array.isArray(item.parts)) { + for (const p of item.parts) { + if (p && typeof p === "object" && typeof p.text === "string" && startsWithMarker(p.text, marker)) return true + } + return false + } if (item.role !== "assistant") return false if (item.type !== undefined && item.type !== "message") return false if (typeof item.content === "string") return startsWithMarker(item.content, marker) @@ -195,7 +219,11 @@ function startsWithMarker(text, marker) { function slimMessage(m, cfg) { if (!m || typeof m !== "object") return false if (m.role === "system" || m.role === "developer") return false - if (m.role === "tool") return slimContent(m, "content", cfg.toolHead, cfg.toolTail, cfg) + // role "function" is the legacy functions API's answer half (older + // clients). Its content is the tool output and just as big as a + // role "tool" message — leaving it whole meant the oldest clients got + // none of the saving. + if (m.role === "tool" || m.role === "function") return slimContent(m, "content", cfg.toolHead, cfg.toolTail, cfg) if (m.role === "assistant") { // tool_calls are the pairing half of the tool results below; thinking // blocks carry signatures their next turn checks. Neither is ours. @@ -303,7 +331,8 @@ function slimContent(holder, key, head, tail, cfg) { function collapseMessage(m, cfg) { if (!m || typeof m !== "object") return false if (m.role === "system" || m.role === "developer") return false - if (m.role === "tool") return collapseContent(m, "content", cfg) + // role "function": the legacy functions API's answer half — see slimMessage. + if (m.role === "tool" || m.role === "function") return collapseContent(m, "content", cfg) if (m.role === "assistant") { let changed = false if (typeof m.content === "string") changed = collapseText(m, "content", cfg) @@ -400,24 +429,39 @@ function firstTask(list) { const it = list[i] if (it && it.role === "user") { const t = textOf(it) - if (t) return t + if (t) return withoutHarness(t) } } for (const it of list) { if (it && it.role === "user") { const t = textOf(it) - if (t) return t + if (t) return withoutHarness(t) } } return null } +// Harness-injected blocks, not the user's words. Claude Code appends +// as its own user message — token budgets, tool guidance, +// file-change notices. Reading one as "the newest question" made every +// follow-up look like a new topic: the reminder shares no words with the +// task, so the overlap test said "shifted" and the history collapsed. +const HARNESS_BLOCKS = /<(system-reminder|system-reminder-sudo)>[\s\S]*?<\/\1>/gi + +function withoutHarness(text) { + return String(text).replace(HARNESS_BLOCKS, " ").replace(/\s+/g, " ").trim() +} + function lastQuestion(list) { for (let i = list.length - 1; i >= 0; i--) { const it = list[i] if (!it || it.role !== "user") continue const t = textOf(it) - if (t) return t + if (!t) continue + // A message that is nothing but a harness block is not a question; + // keep looking for the user's actual words. + const spoken = withoutHarness(t) + if (spoken) return spoken } return null } @@ -426,6 +470,17 @@ function lastQuestion(list) { // tool_result content and image payloads are the machinery, not the // question. function textOf(it) { + // Gemini: the text rides in parts[], and a part may be a thought + // (thought: true) — the model's own reasoning, not the user's words. + if (Array.isArray(it.parts)) { + let out = "" + for (const p of it.parts) { + if (!p || typeof p !== "object" || typeof p.text !== "string") continue + if (p.thought) continue + out += " " + p.text + } + return out.trim() || null + } if (typeof it.content === "string") return it.content if (!Array.isArray(it.content)) return null let out = "" @@ -441,6 +496,95 @@ function words(text) { return out } +// ── gemini ───────────────────────────────────────────────────────────── +// Gemini's history is contents[]: each turn has role user|model and parts[]. +// A part may be plain text, a thought (thought: true), a functionCall, a +// functionResponse (the tool output — the big one), inlineData (an image +// or a file) or fileData. +// +// What must not be touched: thoughts and anything carrying a +// thoughtSignature (the provider checks the signature over the payload), +// functionCall (the call half of a functionResponse), and the sources — +// inlineData of a document or a video is a payload, not history prose. +// +// A functionResponse keeps its id, its name and its keys; only the strings +// inside the response object are thinned. + +function slimGemini(c, cfg) { + if (!c || typeof c !== "object" || !Array.isArray(c.parts)) return false + let changed = false + for (const p of c.parts) { + if (!p || typeof p !== "object") continue + if (p.thought || p.thoughtSignature) continue + if (p.functionCall) continue + if (p.functionResponse) { + changed = slimFunctionResponse(p, cfg) || changed + } else if (typeof p.text === "string") { + const u = cut(p.text, cfg.textHead, cfg.textTail, "text", cfg) + if (u !== p.text) { p.text = u; changed = true } + } else if (cfg.dropImages && p.inlineData && typeof p.inlineData.mimeType === "string" && p.inlineData.mimeType.startsWith("image/")) { + p.text = "[" + MARK + ": old image omitted]" + delete p.inlineData + changed = true + } + } + return changed +} + +function collapseGemini(c, cfg) { + if (!c || typeof c !== "object" || !Array.isArray(c.parts)) return false + let changed = false + for (const p of c.parts) { + if (!p || typeof p !== "object") continue + if (p.thought || p.thoughtSignature) continue + if (p.functionCall) continue + if (p.functionResponse) { + changed = collapseFunctionResponse(p, cfg) || changed + } else if (typeof p.text === "string") { + changed = collapseText(p, "text", cfg) || changed + } else if (p.inlineData && typeof p.inlineData.mimeType === "string" && p.inlineData.mimeType.startsWith("image/")) { + p.text = NOTE + delete p.inlineData + changed = true + } + } + return changed +} + +// The response object is free-form: walk it and cut its long strings, +// leaving the keys and the shape alone. +function slimFunctionResponse(part, cfg) { + const r = part.functionResponse && part.functionResponse.response + if (!r || typeof r !== "object") return false + let changed = false + for (const key of Object.keys(r)) { + const v = r[key] + if (typeof v === "string") { + const u = cut(v, cfg.toolHead, cfg.toolTail, "tool output", cfg) + if (u !== v) { r[key] = u; changed = true } + } else if (v && typeof v === "object") { + changed = slimFunctionResponse({ functionResponse: { response: v } }, cfg) || changed + } + } + return changed +} + +function collapseFunctionResponse(part, cfg) { + const r = part.functionResponse && part.functionResponse.response + if (!r || typeof r !== "object") return false + let changed = false + for (const key of Object.keys(r)) { + const v = r[key] + if (typeof v === "string") { + const holder = { value: v } + if (collapseText(holder, "value", cfg)) { r[key] = holder.value; changed = true } + } else if (v && typeof v === "object") { + changed = collapseFunctionResponse({ functionResponse: { response: v } }, cfg) || changed + } + } + return changed +} + // ── shared ───────────────────────────────────────────────────────────── function isImage(type) { diff --git a/packages/fovea/fovea.protocols.test.mjs b/packages/fovea/fovea.protocols.test.mjs new file mode 100644 index 000000000..8a430d711 --- /dev/null +++ b/packages/fovea/fovea.protocols.test.mjs @@ -0,0 +1,359 @@ +// One request per client protocol magpie's gateway serves (ctx.protocol: +// anthropic, chat, responses, gemini), each carrying the block and item +// types real clients send beyond the common ones: server tools, redacted +// thinking, citations, search results, legacy functions, reasoning_details, +// audio and files, custom and computer tools, item references, Gemini +// thought signatures and function responses. +// +// The window is set so every old item is in reach (cut_step 1, two kept): +// these check what fovea does to each kind of item, and what it must leave +// alone. fovea.probe.test.mjs checks the same contracts over whole sessions +// with the shipped defaults. +// +// Caching, per protocol: Anthropic's breakpoints are cache_control objects +// on blocks (their place and ttl are the agent's); Chat and Responses cache +// on the prefix by themselves and are steered by top-level fields +// (prompt_cache_key, prompt_cache_retention, previous_response_id); Gemini +// by cachedContent. So every top-level field stays byte-identical, and +// every cache_control stays where it was, with its value. +import { test, expect } from "bun:test" +import { readFileSync } from "node:fs" +import { join } from "node:path" +import { onRequest } from "./fovea.middleware.js" + +const DEFAULTS = JSON.parse(readFileSync(join(import.meta.dir, "package.json"), "utf8")).magpie.options +const EDGE = { ...DEFAULTS, threshold: 0, keep_recent: 2, keep_recent_tokens: 0, cut_step: 1 } +const MODES = ["off", "always"] + +const json = (v) => JSON.stringify(v) +const same = (a, b) => expect(json(a)).toBe(json(b)) +const send = (body, options) => onRequest(structuredClone(body), { options }) ?? body + +const TASK = "Fix the flaky scheduler test in internal/sched: the retry timing assertion fails under load. " + "context ".repeat(40) +const LONG = "An explanation of what was found and why. ".repeat(90) +const BIG = "line of tool output 😀 with text\n".repeat(200) +const IMAGE_NOTE = "[fovea: old image omitted]" +const TOPIC_NOTE = "[fovea: prior topic not carried]" + +function topLevelSame(before, after, listKey) { + expect(Object.keys(after)).toEqual(Object.keys(before)) + for (const k of Object.keys(before)) if (k !== listKey) same(after[k], before[k]) +} + +function cachePaths(v, path = "$", out = []) { + if (Array.isArray(v)) v.forEach((x, i) => cachePaths(x, `${path}[${i}]`, out)) + else if (v && typeof v === "object") { + for (const k of Object.keys(v)) { + if (k === "cache_control") out.push(`${path}.cache_control=${json(v[k])}`) + else cachePaths(v[k], `${path}.${k}`, out) + } + } + return out +} + +const LONE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const raw = anthropic() + const sent = send(raw, { ...EDGE, saccade }) + common(raw, sent, "messages") + const was = raw.messages, now = sent.messages + // a saccade drops the old topic entirely — the task message goes with it + if (saccade === "off") same(now[0], was[0]) + // the signer's and the provider's blocks + for (const k of [0, 1, 2, 3, 5]) same(now[1].content[k], was[1].content[k]) + // sources are not history: whole + same(now[2].content[1], was[2].content[1]) + same(now[2].content[2], was[2].content[2]) + // a tool result keeps its id, its error flag and its own breakpoint + expect(now[2].content[0].tool_use_id).toBe("toolu_1") + expect(now[2].content[0].is_error).toBe(true) + same(now[4].content[0].cache_control, { type: "ephemeral" }) + // an old image is a note that keeps the 1h breakpoint + same(now[2].content[0].content[1], { type: "text", text: saccade === "off" ? IMAGE_NOTE : TOPIC_NOTE, cache_control: { type: "ephemeral", ttl: "1h" } }) + // prose and tool output are thinned; the text keeps its citations + same(now[1].content[4].citations, was[1].content[4].citations) + const thinned = [now[1].content[4].text, now[2].content[0].content[0].text, now[4].content[0].content] + for (const t of thinned) expect(t).toContain("[fovea:") + }) +} + +// ── chat ─────────────────────────────────────────────────────────────── + +function chat() { + return { + model: "chat-test", stream: true, stream_options: { include_usage: true }, prompt_cache_key: "session-42", + reasoning_effort: "high", tool_choice: "auto", parallel_tool_calls: true, user: "u", store: false, + tools: [{ type: "function", function: { name: "read", parameters: { type: "object" } } }], + messages: [ + { role: "system", content: "S".repeat(2000) }, + // OpenRouter, Qwen and others take Anthropic-style breakpoints in Chat parts + { role: "developer", content: [{ type: "text", text: "D".repeat(2000), cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ type: "text", text: TASK }] }, + { role: "assistant", content: null, reasoning_content: LONG, + reasoning_details: [{ type: "reasoning.text", text: LONG, signature: "sig", format: "anthropic-claude-v1", index: 0 }, { type: "reasoning.encrypted", data: "enc" + "e".repeat(800), index: 1 }], + tool_calls: [ + { id: "call_1", type: "function", function: { name: "read", arguments: "{\"path\":\"a.go\"}" } }, + { id: "call_2", type: "function", function: { name: "read", arguments: "{\"path\":\"b.go\"}" } }, + ] }, + { role: "tool", tool_call_id: "call_1", content: BIG }, + { role: "tool", tool_call_id: "call_2", content: [{ type: "text", text: BIG }] }, + { role: "user", content: [ + { type: "text", text: LONG, cache_control: { type: "ephemeral" } }, + { type: "image_url", image_url: { url: "data:image/png;base64," + "A".repeat(3000), detail: "high" } }, + { type: "input_audio", input_audio: { data: "U".repeat(3000), format: "wav" } }, + { type: "file", file: { filename: "spec.pdf", file_data: "data:application/pdf;base64," + "P".repeat(3000) } }, + ] }, + // the legacy functions API, still sent by older clients + { role: "assistant", content: null, function_call: { name: "read", arguments: "{}" } }, + { role: "function", name: "read", content: BIG }, + { role: "assistant", content: LONG, refusal: null, name: "agent" }, + { role: "user", content: "go on" }, + { role: "assistant", content: "ok" }, + ], + } +} + +for (const saccade of MODES) { + test(`chat (saccade ${saccade}): parallel tool calls, reasoning_details, Chat breakpoints, audio, files and legacy functions`, () => { + const raw = chat() + const sent = send(raw, { ...EDGE, saccade }) + common(raw, sent, "messages") + const was = raw.messages, now = sent.messages + if (saccade === "off") for (const i of [0, 1, 2, 3, 7]) same(now[i], was[i]) // system, developer, task, the calls + for (let i = 0; i < was.length; i++) { + expect(now[i].role).toBe(was[i].role) + expect(now[i].tool_call_id).toBe(was[i].tool_call_id) + expect(now[i].name).toBe(was[i].name) + } + // audio and files are sources, like documents: whole + same(now[6].content[2], was[6].content[2]) + same(now[6].content[3], was[6].content[3]) + // an old image_url becomes a Chat text part + same(now[6].content[1], { type: "text", text: saccade === "off" ? IMAGE_NOTE : TOPIC_NOTE }) + expect(now[6].content[0].cache_control).toEqual({ type: "ephemeral" }) + expect(now[9].refusal).toBe(null) + const thinned = [now[4].content, now[5].content[0].text, now[6].content[0].text, now[8].content, now[9].content] + for (const t of thinned) expect(t).toContain("[fovea:") + }) +} + +// ── responses ────────────────────────────────────────────────────────── + +const OPAQUE = ["reasoning", "function_call", "custom_tool_call", "computer_call", "computer_call_output", "web_search_call", "local_shell_call", "local_shell_call_output", "item_reference"] + +function responses() { + return { + model: "gpt-test", stream: true, store: false, prompt_cache_key: "thread-7", prompt_cache_retention: "24h", + include: ["reasoning.encrypted_content"], reasoning: { effort: "high", summary: "auto" }, text: { verbosity: "low" }, + instructions: "codex base instructions " + "I".repeat(2000), tool_choice: "auto", parallel_tool_calls: false, + tools: [{ type: "function", name: "shell", parameters: { type: "object" } }, { type: "custom", name: "apply_patch" }, { type: "web_search" }], + input: [ + { role: "developer", content: "sandbox policy " + "P".repeat(500) }, + { role: "user", content: [{ type: "input_text", text: "# AGENTS.md instructions\n" + "A".repeat(3000) }] }, + { type: "message", role: "user", content: [{ type: "input_text", text: TASK }] }, + { type: "reasoning", id: "rs_1", summary: [{ type: "summary_text", text: LONG }], encrypted_content: "gAAAA" + "e".repeat(1500) }, + { type: "message", role: "assistant", id: "msg_1", status: "completed", content: [{ type: "output_text", text: LONG, annotations: [] }] }, + { type: "function_call", id: "fc_1", call_id: "call_1", name: "shell", arguments: "{\"command\":[\"go\",\"test\"]}" }, + { type: "function_call_output", call_id: "call_1", output: [ + { type: "input_text", text: BIG }, + { type: "input_image", image_url: "data:image/png;base64," + "A".repeat(3000), detail: "auto" }, + ] }, + { type: "custom_tool_call", id: "ctc_1", call_id: "call_2", name: "apply_patch", input: "*** Begin Patch\n" + "+x\n".repeat(900) + "*** End Patch" }, + { type: "custom_tool_call_output", call_id: "call_2", output: BIG }, + { type: "computer_call", id: "cu_1", call_id: "call_3", action: { type: "click", x: 1, y: 2 }, pending_safety_checks: [], status: "completed" }, + { type: "computer_call_output", call_id: "call_3", output: { type: "computer_screenshot", image_url: "data:image/png;base64," + "C".repeat(3000) } }, + { type: "web_search_call", id: "ws_1", status: "completed", action: { type: "search", query: "go retry" } }, + { type: "local_shell_call", id: "ls_1", call_id: "call_4", action: { type: "exec", command: ["ls"], env: {} }, status: "completed" }, + { type: "local_shell_call_output", id: "call_4", output: BIG }, + { type: "item_reference", id: "msg_0" }, + { type: "message", role: "user", content: [ + { type: "input_text", text: LONG }, + { type: "input_image", image_url: "data:image/png;base64," + "B".repeat(3000), detail: "high" }, + { type: "input_file", filename: "spec.pdf", file_data: "data:application/pdf;base64," + "P".repeat(3000) }, + ] }, + { type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }, + { type: "message", role: "user", content: [{ type: "input_text", text: "go on" }] }, + ], + } +} + +for (const saccade of MODES) { + test(`responses (saccade ${saccade}): reasoning, custom, computer, shell and web-search items, item references, files and cache fields`, () => { + const raw = responses() + const sent = send(raw, { ...EDGE, saccade }) + common(raw, sent, "input") + const was = raw.input, now = sent.input + if (saccade === "off") for (const i of [0, 1, 2]) same(now[i], was[i]) // the preamble + for (let i = 0; i < was.length; i++) { + if (OPAQUE.includes(was[i].type)) same(now[i], was[i]) + expect(now[i].call_id).toBe(was[i].call_id) + } + // an assistant message keeps its id, status and annotations + expect(now[4].id).toBe("msg_1") + expect(now[4].status).toBe("completed") + same(now[4].content[0].annotations, []) + // files are sources: whole; old images become input_text + same(now[15].content[2], was[15].content[2]) + const note = saccade === "off" ? IMAGE_NOTE : TOPIC_NOTE + same(now[6].output[1], { type: "input_text", text: note }) + same(now[15].content[1], { type: "input_text", text: note }) + const thinned = [now[4].content[0].text, now[6].output[0].text, now[8].output, now[15].content[0].text] + for (const t of thinned) expect(t).toContain("[fovea:") + }) +} + +test("responses with previous_response_id: the history lives on the server, the request carries only new items, nothing is touched", () => { + const raw = { model: "gpt-test", previous_response_id: "resp_9", prompt_cache_key: "thread-7", + input: [{ type: "function_call_output", call_id: "call_9", output: BIG.repeat(30) }] } + for (const options of [DEFAULTS, EDGE]) expect(onRequest(structuredClone(raw), { options })).toBeUndefined() +}) + +test("responses: a custom_tool_call is an answer — the preamble ends there, even without a reasoning item", () => { + const raw = responses() + raw.input.splice(3, 2) // no reasoning, no assistant message: the first answer is the function_call + raw.input.splice(3, 0, { type: "custom_tool_call", id: "ctc_0", call_id: "call_0", name: "apply_patch", input: "patch" }, { type: "custom_tool_call_output", call_id: "call_0", output: BIG }) + const sent = send(raw, EDGE) + same(sent.input[2], raw.input[2]) + expect(sent.input[4].output).toContain("[fovea:") +}) + +// ── gemini ───────────────────────────────────────────────────────────── + +function gemini() { + return { + systemInstruction: { parts: [{ text: "S".repeat(2000) }] }, + tools: [{ functionDeclarations: [{ name: "read_file", parameters: { type: "OBJECT" } }] }], + toolConfig: { functionCallingConfig: { mode: "AUTO" } }, + generationConfig: { temperature: 0, thinkingConfig: { includeThoughts: true, thinkingBudget: -1 } }, + cachedContent: "cachedContents/abc123", + contents: [ + { role: "user", parts: [{ text: TASK }] }, + { role: "model", parts: [ + { text: LONG, thought: true }, + { text: LONG }, + { functionCall: { id: "fc_1", name: "read_file", args: { path: "a.go" } }, thoughtSignature: "sig" + "g".repeat(300) }, + ] }, + { role: "user", parts: [{ functionResponse: { id: "fc_1", name: "read_file", response: { output: BIG } } }] }, + { role: "user", parts: [ + { text: LONG }, + { inlineData: { mimeType: "image/png", data: "A".repeat(3000) } }, + { inlineData: { mimeType: "application/pdf", data: "P".repeat(3000) } }, + { fileData: { mimeType: "video/mp4", fileUri: "https://generativelanguage.googleapis.com/v1beta/files/x" } }, + ] }, + { role: "model", parts: [ + { text: LONG, thoughtSignature: "sig2" }, + { functionCall: { id: "fc_2", name: "read_file", args: {} } }, + ] }, + { role: "user", parts: [{ functionResponse: { id: "fc_2", name: "read_file", response: { error: BIG, exit_code: 1 } } }] }, + { role: "model", parts: [{ text: "ok" }] }, + { role: "user", parts: [{ text: "go on" }] }, + ], + } +} + +for (const saccade of MODES) { + test(`gemini (saccade ${saccade}): thoughts, thought signatures, function calls and responses, images, files and cachedContent`, () => { + const raw = gemini() + const sent = send(raw, { ...EDGE, saccade }) + common(raw, sent, "contents") + const was = raw.contents, now = sent.contents + if (saccade === "off") same(now[0], was[0]) // the task + for (let i = 0; i < was.length; i++) { + expect(now[i].role).toBe(was[i].role) + expect(now[i].parts.length).toBe(was[i].parts.length) + } + // thoughts, anything a signature covers, function calls, files: whole + for (const [i, k] of [[1, 0], [1, 2], [3, 2], [3, 3], [4, 0], [4, 1]]) same(now[i].parts[k], was[i].parts[k]) + // a function response keeps its id, name and keys; its strings are thinned + for (const i of [2, 5]) { + expect(now[i].parts[0].functionResponse.id).toBe(was[i].parts[0].functionResponse.id) + expect(now[i].parts[0].functionResponse.name).toBe("read_file") + expect(Object.keys(now[i].parts[0].functionResponse.response)).toEqual(Object.keys(was[i].parts[0].functionResponse.response)) + } + expect(now[5].parts[0].functionResponse.response.exit_code).toBe(1) + same(now[3].parts[1], { text: saccade === "off" ? IMAGE_NOTE : TOPIC_NOTE }) + const thinned = [now[1].parts[1].text, now[2].parts[0].functionResponse.response.output, now[3].parts[0].text, now[5].parts[0].functionResponse.response.error] + for (const t of thinned) expect(t).toContain("[fovea:") + }) +} + +test("gemini, saccade auto: a follow-up to the task keeps the old context; a new topic collapses it", () => { + const options = { ...EDGE, saccade: "auto" } + const follow = gemini() + follow.contents.at(-1).parts[0].text = "the scheduler retry timing assertion in internal/sched still fails under load" + expect(json(send(follow, options))).not.toContain(TOPIC_NOTE) + const shift = gemini() + shift.contents.at(-1).parts[0].text = "unrelated: draft a release announcement email aimed at marketing newsletter readers" + expect(json(send(shift, options))).toContain(TOPIC_NOTE) +}) + +test("gemini, state_marker: a state block in a text part stays whole and lifts the task exemption", () => { + const marker = "" + const raw = gemini() + // the state block is the agent's own message: in Gemini that is the model + // turn, and the marker has to be at the head of the text (0.2.4 rule) + raw.contents[4].parts[0].text = marker + "\n" + LONG + // the task must be long enough to be worth cutting once the state block + // lifts its exemption + raw.contents[0].parts[0].text = TASK + " " + "ctx ".repeat(400) + const sent = send(raw, { ...EDGE, state_marker: marker }) + same(sent.contents[4].parts[0], raw.contents[4].parts[0]) + expect(sent.contents[0].parts[0].text).not.toBe(raw.contents[0].parts[0].text) + // a marker in a user turn is somebody else's text: it exempts nothing + const forged = gemini() + forged.contents[3].parts[0].text = marker + "\n" + LONG + forged.contents[0].parts[0].text = TASK + " " + "ctx ".repeat(400) + const sent2 = send(forged, { ...EDGE, state_marker: marker }) + expect(sent2.contents[3].parts[0].text).not.toBe(forged.contents[3].parts[0].text) +}) diff --git a/packages/fovea/package.json b/packages/fovea/package.json index b95df479d..49cb34c2a 100644 --- a/packages/fovea/package.json +++ b/packages/fovea/package.json @@ -1,6 +1,6 @@ { "name": "@magpie-community/middleware-fovea", - "version": "0.2.4", + "version": "0.2.5", "description": "magpie gateway middleware: keep the recent window sharp, thin old history to head and tail, and on a saccade drop the old context on a new topic — with an evidence page", "type": "module", "files": [ @@ -11,6 +11,7 @@ "build.mjs", "cases.json", "fovea.middleware.js", + "fovea.protocols.test.mjs", "fovea.test.mjs", "page/*", "preview.mjs", diff --git a/packages/fovea/page/measure.js b/packages/fovea/page/measure.js index 9544c6783..54a43bd96 100644 --- a/packages/fovea/page/measure.js +++ b/packages/fovea/page/measure.js @@ -86,7 +86,7 @@ function onRequest(body, ctx) { // carries body.input. Size is measured by walking, not by // stringifying: a 5MB body would otherwise allocate 5MB of string on // every request, on the request path. - const list = Array.isArray(body.messages) ? body.messages : Array.isArray(body.input) ? body.input : null + const list = Array.isArray(body.messages) ? body.messages : Array.isArray(body.input) ? body.input : Array.isArray(body.contents) ? body.contents : null if (!list) return if (cfg.threshold > 0 && sizeOf(body) < cfg.threshold) return @@ -115,6 +115,7 @@ function onRequest(body, ctx) { const item = list[i] if (cfg.stateMarker && carriesState(item, cfg.stateMarker)) continue if (list === body.input) changed = (saccade ? collapseInput : slimInput)(item, cfg) || changed + else if (list === body.contents) changed = (saccade ? collapseGemini : slimGemini)(item, cfg) || changed else changed = (saccade ? collapseMessage : slimMessage)(item, cfg) || changed } return changed ? body : undefined @@ -147,12 +148,28 @@ function windowStart(list, keepCount, keepTokens) { // assistant reply, a function_call or a reasoning item, whichever comes // first — is that preamble and stays whole; the history proper begins at // the answer. +// The Responses API marks an answer with the item's type, not the role: +// a function_call, a custom_tool_call, a computer_call, a local_shell_call +// and their outputs are all "the model spoke". Leaving them out made the +// preamble exemption swallow the whole history on a body whose first +// answer was a custom_tool_call (2026-11-05). +const ANSWER_ITEMS = new Set([ + "function_call", "function_call_output", + "custom_tool_call", "custom_tool_call_output", + "computer_call", "computer_call_output", + "local_shell_call", "local_shell_call_output", + "web_search_call", "reasoning", +]) + function leadingUserEnd(list, start) { for (let i = 0; i < start; i++) { const it = list[i] if (!it || typeof it !== "object") continue - if (it.role === "assistant" || it.role === "tool" || it.type === "function_call" || - it.type === "reasoning" || it.type === "function_call_output") return i + // Gemini's model turns carry role "model"; its function responses ride + // in a user turn's parts, so the role alone cannot mark the answer. + if (it.role === "assistant" || it.role === "model" || it.role === "tool") return i + if (typeof it.type === "string" && ANSWER_ITEMS.has(it.type)) return i + if (Array.isArray(it.parts) && it.parts.some((p) => p && p.functionCall)) return i } return start } @@ -175,6 +192,13 @@ function hasStateBlock(list, marker) { function carriesState(item, marker) { if (!item || typeof item !== "object" || !marker) return false + // Gemini's model turn is the agent's own message, like an assistant one. + if (item.role === "model" && Array.isArray(item.parts)) { + for (const p of item.parts) { + if (p && typeof p === "object" && typeof p.text === "string" && startsWithMarker(p.text, marker)) return true + } + return false + } if (item.role !== "assistant") return false if (item.type !== undefined && item.type !== "message") return false if (typeof item.content === "string") return startsWithMarker(item.content, marker) @@ -195,7 +219,11 @@ function startsWithMarker(text, marker) { function slimMessage(m, cfg) { if (!m || typeof m !== "object") return false if (m.role === "system" || m.role === "developer") return false - if (m.role === "tool") return slimContent(m, "content", cfg.toolHead, cfg.toolTail, cfg) + // role "function" is the legacy functions API's answer half (older + // clients). Its content is the tool output and just as big as a + // role "tool" message — leaving it whole meant the oldest clients got + // none of the saving. + if (m.role === "tool" || m.role === "function") return slimContent(m, "content", cfg.toolHead, cfg.toolTail, cfg) if (m.role === "assistant") { // tool_calls are the pairing half of the tool results below; thinking // blocks carry signatures their next turn checks. Neither is ours. @@ -303,7 +331,8 @@ function slimContent(holder, key, head, tail, cfg) { function collapseMessage(m, cfg) { if (!m || typeof m !== "object") return false if (m.role === "system" || m.role === "developer") return false - if (m.role === "tool") return collapseContent(m, "content", cfg) + // role "function": the legacy functions API's answer half — see slimMessage. + if (m.role === "tool" || m.role === "function") return collapseContent(m, "content", cfg) if (m.role === "assistant") { let changed = false if (typeof m.content === "string") changed = collapseText(m, "content", cfg) @@ -400,24 +429,39 @@ function firstTask(list) { const it = list[i] if (it && it.role === "user") { const t = textOf(it) - if (t) return t + if (t) return withoutHarness(t) } } for (const it of list) { if (it && it.role === "user") { const t = textOf(it) - if (t) return t + if (t) return withoutHarness(t) } } return null } +// Harness-injected blocks, not the user's words. Claude Code appends +// as its own user message — token budgets, tool guidance, +// file-change notices. Reading one as "the newest question" made every +// follow-up look like a new topic: the reminder shares no words with the +// task, so the overlap test said "shifted" and the history collapsed. +const HARNESS_BLOCKS = /<(system-reminder|system-reminder-sudo)>[\s\S]*?<\/\1>/gi + +function withoutHarness(text) { + return String(text).replace(HARNESS_BLOCKS, " ").replace(/\s+/g, " ").trim() +} + function lastQuestion(list) { for (let i = list.length - 1; i >= 0; i--) { const it = list[i] if (!it || it.role !== "user") continue const t = textOf(it) - if (t) return t + if (!t) continue + // A message that is nothing but a harness block is not a question; + // keep looking for the user's actual words. + const spoken = withoutHarness(t) + if (spoken) return spoken } return null } @@ -426,6 +470,17 @@ function lastQuestion(list) { // tool_result content and image payloads are the machinery, not the // question. function textOf(it) { + // Gemini: the text rides in parts[], and a part may be a thought + // (thought: true) — the model's own reasoning, not the user's words. + if (Array.isArray(it.parts)) { + let out = "" + for (const p of it.parts) { + if (!p || typeof p !== "object" || typeof p.text !== "string") continue + if (p.thought) continue + out += " " + p.text + } + return out.trim() || null + } if (typeof it.content === "string") return it.content if (!Array.isArray(it.content)) return null let out = "" @@ -441,6 +496,95 @@ function words(text) { return out } +// ── gemini ───────────────────────────────────────────────────────────── +// Gemini's history is contents[]: each turn has role user|model and parts[]. +// A part may be plain text, a thought (thought: true), a functionCall, a +// functionResponse (the tool output — the big one), inlineData (an image +// or a file) or fileData. +// +// What must not be touched: thoughts and anything carrying a +// thoughtSignature (the provider checks the signature over the payload), +// functionCall (the call half of a functionResponse), and the sources — +// inlineData of a document or a video is a payload, not history prose. +// +// A functionResponse keeps its id, its name and its keys; only the strings +// inside the response object are thinned. + +function slimGemini(c, cfg) { + if (!c || typeof c !== "object" || !Array.isArray(c.parts)) return false + let changed = false + for (const p of c.parts) { + if (!p || typeof p !== "object") continue + if (p.thought || p.thoughtSignature) continue + if (p.functionCall) continue + if (p.functionResponse) { + changed = slimFunctionResponse(p, cfg) || changed + } else if (typeof p.text === "string") { + const u = cut(p.text, cfg.textHead, cfg.textTail, "text", cfg) + if (u !== p.text) { p.text = u; changed = true } + } else if (cfg.dropImages && p.inlineData && typeof p.inlineData.mimeType === "string" && p.inlineData.mimeType.startsWith("image/")) { + p.text = "[" + MARK + ": old image omitted]" + delete p.inlineData + changed = true + } + } + return changed +} + +function collapseGemini(c, cfg) { + if (!c || typeof c !== "object" || !Array.isArray(c.parts)) return false + let changed = false + for (const p of c.parts) { + if (!p || typeof p !== "object") continue + if (p.thought || p.thoughtSignature) continue + if (p.functionCall) continue + if (p.functionResponse) { + changed = collapseFunctionResponse(p, cfg) || changed + } else if (typeof p.text === "string") { + changed = collapseText(p, "text", cfg) || changed + } else if (p.inlineData && typeof p.inlineData.mimeType === "string" && p.inlineData.mimeType.startsWith("image/")) { + p.text = NOTE + delete p.inlineData + changed = true + } + } + return changed +} + +// The response object is free-form: walk it and cut its long strings, +// leaving the keys and the shape alone. +function slimFunctionResponse(part, cfg) { + const r = part.functionResponse && part.functionResponse.response + if (!r || typeof r !== "object") return false + let changed = false + for (const key of Object.keys(r)) { + const v = r[key] + if (typeof v === "string") { + const u = cut(v, cfg.toolHead, cfg.toolTail, "tool output", cfg) + if (u !== v) { r[key] = u; changed = true } + } else if (v && typeof v === "object") { + changed = slimFunctionResponse({ functionResponse: { response: v } }, cfg) || changed + } + } + return changed +} + +function collapseFunctionResponse(part, cfg) { + const r = part.functionResponse && part.functionResponse.response + if (!r || typeof r !== "object") return false + let changed = false + for (const key of Object.keys(r)) { + const v = r[key] + if (typeof v === "string") { + const holder = { value: v } + if (collapseText(holder, "value", cfg)) { r[key] = holder.value; changed = true } + } else if (v && typeof v === "object") { + changed = collapseFunctionResponse({ functionResponse: { response: v } }, cfg) || changed + } + } + return changed +} + // ── shared ───────────────────────────────────────────────────────────── function isImage(type) {