chore: merge v2

Author: usrnk1

packages/core/src/config/plugin/command.ts

@@ -206,7 +206,7 @@ function evaluateTemplate(
       if (position === last) return args.slice(argIndex).join(" ")
       return args[argIndex]
     })
-    const withArguments = expanded.replaceAll("$ARGUMENTS", input)
+    const withArguments = expanded.replaceAll("$ARGUMENTS", () => input)
     const text =
       placeholders.length === 0 && !template.includes("$ARGUMENTS") && input.trim()
         ? `${withArguments}\n\n${input}`.trim()

packages/core/src/tool/html-markdown.ts

@@ -217,13 +217,20 @@ export function convertHTMLToMarkdown(html: string) {
     }
     if (code.inline) {
       const fence = "`".repeat(Math.max(1, backticks + 1))
-      const padding = /^ | $/.test(code.text) && !/^ +$/.test(code.text) ? " " : ""
       flushSpace()
       prefixQuote()
-      const wrapper = encoder.encode(`${fence}${padding}${padding}${fence}`).byteLength
-      appendRaw(
-        `${fence}${padding}${sliceBytes(code.text, Math.max(0, CONTENT_BYTES - outputBytes - wrapper))}${padding}${fence}`,
-      )
+      const available = Math.max(0, CONTENT_BYTES - outputBytes - fence.length * 2)
+      let payload = sliceBytes(code.text, available)
+      while (payload) {
+        const padding = /^[ `]|[ `]$/.test(payload) && !/^ +$/.test(payload) ? " " : ""
+        const bytes = encoder.encode(payload).byteLength
+        if (bytes + padding.length * 2 <= available) {
+          appendRaw(`${fence}${padding}${payload}${padding}${fence}`)
+          return
+        }
+        // Padding costs at most two bytes, so at most two whole-code-point trims are needed.
+        payload = sliceBytes(payload, bytes - 1)
+      }
       return
     }
     if (activeCell) {
@@ -246,6 +253,8 @@ export function convertHTMLToMarkdown(html: string) {
         block()
         return
       }
+      // No amount of trimming can make the empty fenced block fit.
+      if (!payload) return
       const excess = valueBytes - Math.max(0, CONTENT_BYTES - outputBytes)
       payload = sliceBytes(payload, Math.max(0, encoder.encode(payload).byteLength - Math.ceil(excess)))
     }

packages/core/test/config/command.test.ts

@@ -71,6 +71,73 @@ const it = testEffect(
 const decode = Schema.decodeUnknownSync(Info)
 
 describe("ConfigCommandPlugin.Plugin", () => {
+  for (const item of [
+    ...["$&", "$$", "$`", "$'"].flatMap((input) => [
+      { template: "Explain $ARGUMENTS.", input, expected: `Explain ${input}.` },
+      { template: "Explain $1.", input: `"${input}"`, expected: `Explain ${input}.` },
+      { template: "Explain.", input, expected: `Explain.\n\n${input}` },
+    ]),
+    ...["abc", "", "alpha beta", '"alpha beta"', "$1", "$<name>"].map((input) => ({
+      template: "Explain $ARGUMENTS.",
+      input,
+      expected: `Explain ${input}.`,
+    })),
+    {
+      template: "First $1. Rest $2.",
+      input: '"alpha beta" gamma delta',
+      expected: "First alpha beta. Rest gamma delta.",
+    },
+    { template: "$ARGUMENTS / $ARGUMENTS", input: "$& $$", expected: "$& $$ / $& $$" },
+  ]) {
+    it.live(`interpolates ${JSON.stringify(item.template)} with literal input ${JSON.stringify(item.input)}`, () =>
+      Effect.gen(function* () {
+        const command = yield* Command.Service
+        const prompts: { text: string; delivery?: string }[] = []
+        yield* ConfigCommandPlugin.Plugin.effect(
+          host({
+            command: {
+              list: () => Effect.die(new Error("unused command.list")),
+              transform: command.transform,
+              reload: command.reload,
+            },
+            session: {
+              prompt: (input) =>
+                Effect.sync(() => {
+                  prompts.push({ text: input.text, delivery: input.delivery })
+                  return SessionInbox.User.make({
+                    id: SessionMessage.ID.make("msg_test"),
+                    sessionID: input.sessionID,
+                    timeCreated: DateTime.makeUnsafe(0),
+                    type: "user",
+                    payload: { text: input.text },
+                    delivery: input.delivery ?? "steer",
+                  })
+                }),
+            },
+          }),
+        ).pipe(
+          Effect.provide(
+            Config.testLayer([
+              new Document({
+                type: "document",
+                info: decode({ commands: { explain: { template: item.template } } }),
+              }),
+            ]),
+          ),
+        )
+        yield* command.execute({
+          name: "explain",
+          invocation: {
+            sessionID: Session.ID.make("ses_test"),
+            prompt: { text: item.input },
+            delivery: "queue",
+          },
+        })
+        expect(prompts).toEqual([{ text: item.expected, delivery: "queue" }])
+      }),
+    )
+  }
+
   it.live("loads inline and file-based commands in config order", () =>
     Effect.acquireDisposable(Effect.promise(() => tmpdir())).pipe(
       Effect.flatMap((tmp) =>

packages/core/test/fixture/html-markdown.ts

@@ -0,0 +1,7 @@
+import { writeSync } from "node:fs"
+import { convertHTMLToMarkdown } from "../../src/tool/html-markdown"
+
+const html = await Bun.stdin.text()
+// Flush readiness before entering a conversion that may block the child event loop.
+writeSync(1, "ready\n")
+await Bun.write(Bun.stdout, convertHTMLToMarkdown(html))

packages/core/test/tool-html-markdown-budget.test.ts

@@ -0,0 +1,69 @@
+import { expect, test } from "bun:test"
+import { fileURLToPath } from "node:url"
+import { MAX_MARKDOWN_BYTES } from "../src/tool/html-markdown"
+
+const budget = MAX_MARKDOWN_BYTES - 64 * 1024
+
+test.each([
+  ["exhausted budget", budget, "x", "", false],
+  ["one byte short of an empty fence", budget - 13, "x", "", false],
+  ["small fitting block", 64, "x", "\n\n```\nx\n```", false],
+  ["last payload byte fits", budget - 15, "x", "\n\n```\nx\n```", false],
+  ["only an empty fence fits", budget - 14, "x", "\n\n```\n\n```", false],
+  ["Unicode payload truncates at a code point", budget - 18, "😀é", "\n\n```\n😀\n```", false],
+  ["quoted payload fits", budget - 23, "x", "\n\n> ```\n> x\n> ```", true],
+  ["only an empty quoted fence fits", budget - 22, "x", "\n\n> ```\n> \n> ```", true],
+  ["one byte short of an empty quoted fence", budget - 21, "x", "", true],
+] as const)(
+  "finishes bounded code conversion: %s",
+  async (_name, count, payload, suffix, quoted) => {
+    const code = `<pre>${payload}</pre>`
+    const html = `<p>${"x".repeat(count)}</p>${quoted ? `<blockquote>${code}</blockquote>` : code}`
+    expect(Buffer.byteLength(html)).toBeLessThanOrEqual(MAX_MARKDOWN_BYTES)
+    const child = Bun.spawn({
+      cmd: [process.execPath, fileURLToPath(new URL("./fixture/html-markdown.ts", import.meta.url))],
+      stdin: new Blob([html]),
+      stdout: "pipe",
+      stderr: "pipe",
+    })
+    let ready = false
+    let timeout: "startup" | "conversion" | undefined
+    let timer = setTimeout(() => {
+      timeout = "startup"
+      child.kill("SIGKILL")
+    }, 10_000)
+    const stdout = (async () => {
+      let output = ""
+      for await (const chunk of child.stdout.pipeThrough(new TextDecoderStream())) {
+        output += chunk
+        if (ready || !output.startsWith("ready\n")) continue
+        ready = true
+        clearTimeout(timer)
+        // This watchdog runs outside the possibly stuck synchronous converter.
+        timer = setTimeout(() => {
+          timeout = "conversion"
+          child.kill("SIGKILL")
+        }, 3_000)
+      }
+      return output.slice("ready\n".length)
+    })()
+    const stderr = new Response(child.stderr).text()
+    try {
+      const exitCode = await child.exited
+      const output = await stdout
+      expect({ ready, timeout, exitCode, stderr: await stderr }).toEqual({
+        ready: true,
+        timeout: undefined,
+        exitCode: 0,
+        stderr: "",
+      })
+      expect(Buffer.byteLength(output)).toBeLessThanOrEqual(MAX_MARKDOWN_BYTES)
+      expect(output).toBe("x".repeat(Math.min(count, budget - 2)) + suffix)
+    } finally {
+      clearTimeout(timer)
+      if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL")
+      await child.exited
+    }
+  },
+  15_000,
+)

packages/core/test/tool-webfetch.test.ts

@@ -90,6 +90,53 @@ describe("WebFetchTool helpers", () => {
     )
   })
 
+  test.each([
+    ["`x`", "`` `x` ``"],
+    ["`x", "`` `x ``"],
+    ["x`", "`` x` ``"],
+    ["`", "`` ` ``"],
+    ["``", "``` `` ```"],
+    ["``x`", "``` ``x` ```"],
+    ["say(`x`)", "``say(`x`)``"],
+    ["a``b`c", "```a``b`c```"],
+    ["x", "`x`"],
+    [" x ", "`  x  `"],
+    [" x", "`  x `"],
+    ["x ", "` x  `"],
+    ["   ", "`   `"],
+    [" ` ", "``  `  ``"],
+  ])("preserves inline code boundaries for %j", (content, expected) => {
+    expect(WebFetchTool.convertHTMLToMarkdown(`<p>Use <code>${content}</code>.</p>`)).toBe(`Use ${expected}.`)
+  })
+
+  test.each([
+    ["discarded trailing backtick after ASCII", "x`", 7, "``x``"],
+    ["discarded trailing backtick after Unicode", "😀`", 10, "``😀``"],
+    ["discarded trailing backtick with spare room", "x`", 9, "``x``"],
+    ["retained trailing backtick", "x`", 10, "`` x` ``"],
+    ["new trailing backtick from an internal run", "x`y", 8, "``x``"],
+    ["leading backtick without padding room", "`x", 8, ""],
+    ["leading backtick alone fits", "`x", 9, "`` ` ``"],
+    ["leading backtick with payload fits", "`x", 10, "`` `x ``"],
+    ["all backticks truncated", "``", 11, "``` ` ```"],
+    ["all backticks fit", "``", 12, "``` `` ```"],
+    ["mixed internal runs truncated", "a``b`c", 11, "```a```"],
+    ["Unicode code point cannot fit", "😀`", 9, ""],
+    ["ordinary payload cannot fit", "x", 3, ""],
+    ["spaces cannot fit", "   ", 4, ""],
+    ["space-only prefix fits", "   ", 5, "` `"],
+    ["truncated prefix becomes space-only", " x", 6, "` `"],
+    ["discarded trailing space", "x ", 5, "`x`"],
+  ] as const)("fits inline code to its emitted boundaries: %s", (_name, content, spare, expected) => {
+    const prefix = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024 - spare)
+    const html = `<p>${prefix}<code>${content}</code></p>`
+    expect(Buffer.byteLength(html)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
+    const output = WebFetchTool.convertHTMLToMarkdown(html)
+    expect(output.slice(0, prefix.length)).toBe(prefix)
+    expect(output.slice(prefix.length)).toBe(expected)
+    expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
+  })
+
   test("keeps nested ordered and unordered lists structurally readable", () => {
     const html = `<ol start="3"><li>alpha<ul><li>nested <strong>item</strong></li></ul></li><li><p>beta first</p><p>beta second</p></li></ol>`
     expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(