diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index b04be8c47..0b5a6a76b 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -160,6 +160,41 @@ public void ParseArguments_MarkdownFencesNoLanguage_Stripped() Assert.Equal("NY", result["city"]?.ToString()); } + [Fact] + public void ParseArguments_MarkdownFencesWithSurroundingWhitespace_Stripped() + { + var result = ToolDispatch.ParseArguments(" \n```json \n {\"city\": \"NY\"} \n ``` \n"); + Assert.Equal("NY", result["city"]?.ToString()); + + result = ToolDispatch.ParseArguments("```{\"city\": \"NY\"}```"); + Assert.Equal("NY", result["city"]?.ToString()); + } + + [Fact] + public void ParseArguments_UnclosedFence_FallsThroughToBlockExtraction() + { + var result = ToolDispatch.ParseArguments("```json\n{\"city\": \"NY\"}"); + Assert.Equal("NY", result["city"]?.ToString()); + + Assert.True(ToolDispatch.ParseArguments("```json\n```").ContainsKey("_error")); + Assert.True(ToolDispatch.ParseArguments("``````").ContainsKey("_error")); + } + + [Theory] + [InlineData(' ')] + [InlineData('\n')] + public void ParseArguments_UnclosedFenceFollowedByWhitespace_DoesNotBacktrack(char filler) + { + var arguments = "```json" + new string(filler, 20000) + "x"; + + var started = System.Diagnostics.Stopwatch.StartNew(); + var result = ToolDispatch.ParseArguments(arguments); + started.Stop(); + + Assert.True(result.ContainsKey("_error")); + Assert.True(started.ElapsedMilliseconds < 1000, $"took {started.ElapsedMilliseconds} ms"); + } + [Fact] public void ParseArguments_JsonInProse_Extracted() { diff --git a/runtime/csharp/Prompty.Core/ToolDispatch.cs b/runtime/csharp/Prompty.Core/ToolDispatch.cs index 2a1f58858..f76ffaeb7 100644 --- a/runtime/csharp/Prompty.Core/ToolDispatch.cs +++ b/runtime/csharp/Prompty.Core/ToolDispatch.cs @@ -260,10 +260,9 @@ private static bool ResolveBindings( if (direct is not null) return direct; // Strategy 2: Strip markdown code fences - var fenceMatch = Regex.Match(arguments, @"^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$"); - if (fenceMatch.Success) + var stripped = StripCodeFence(arguments); + if (stripped is not null) { - var stripped = fenceMatch.Groups[1].Value; var fenced = TryParseJson(stripped); if (fenced is not null) { @@ -342,6 +341,23 @@ private static bool ResolveBindings( /// /// Try to parse a JSON string into a dictionary. Returns null on failure. /// + /// + /// Returns the text inside a surrounding markdown code fence, or null when the input is not fenced. + /// String operations rather than a regex: the obvious ^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$ + /// backtracks cubically on an opening fence that never closes, and tool arguments are model output. + /// + private static string? StripCodeFence(string arguments) + { + var text = arguments.Trim(); + if (text.Length < 6 || !text.StartsWith("```", StringComparison.Ordinal) || !text.EndsWith("```", StringComparison.Ordinal)) + return null; + + var inner = text[3..^3]; + if (inner.StartsWith("json", StringComparison.Ordinal)) + inner = inner[4..]; + return inner.Trim(); + } + private static Dictionary? TryParseJson(string json) { try diff --git a/runtime/java/prompty/src/main/java/com/microsoft/prompty/ToolDispatch.java b/runtime/java/prompty/src/main/java/com/microsoft/prompty/ToolDispatch.java index 8f8adf90a..2134e92c9 100644 --- a/runtime/java/prompty/src/main/java/com/microsoft/prompty/ToolDispatch.java +++ b/runtime/java/prompty/src/main/java/com/microsoft/prompty/ToolDispatch.java @@ -8,7 +8,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Matcher; import java.util.regex.Pattern; /** @@ -33,9 +32,6 @@ public final class ToolDispatch { /** Longest prefix of unparseable arguments quoted back in an error. */ private static final int ERROR_EXCERPT_CHARS = 200; - private static final Pattern CODE_FENCE = - Pattern.compile("^\\s*```(?:json)?\\s*\\n?(.*?)\\n?\\s*```\\s*$", Pattern.DOTALL); - private static final Pattern TRAILING_COMMA = Pattern.compile(",\\s*([}\\]])"); private ToolDispatch() {} @@ -177,14 +173,11 @@ public static Object parseArguments(String raw) { return direct; } - Matcher fenced = CODE_FENCE.matcher(text); - if (fenced.matches()) { - String stripped = fenced.group(1); - if (!stripped.equals(text)) { - Object parsed = tryParse(stripped); - if (parsed != NOT_JSON) { - return parsed; - } + String stripped = stripCodeFence(text); + if (stripped != null && !stripped.equals(text)) { + Object parsed = tryParse(stripped); + if (parsed != NOT_JSON) { + return parsed; } } @@ -220,6 +213,45 @@ private static Object tryParse(String text) { } } + /** + * Returns the text inside a surrounding markdown code fence, or {@code null} when the input is + * not fenced. + * + *

Plain index arithmetic rather than a regex: the obvious {@code + * ^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$} backtracks cubically on an opening fence that never + * closes, and tool arguments are model output. Whitespace is the ASCII set a regex {@code \s} + * matches, so results are unchanged. + */ + private static String stripCodeFence(String text) { + int start = 0; + int end = text.length(); + while (start < end && isSpace(text.charAt(start))) { + start++; + } + while (end > start && isSpace(text.charAt(end - 1))) { + end--; + } + if (end - start < 6 || !text.startsWith("```", start) || !text.startsWith("```", end - 3)) { + return null; + } + int from = start + 3; + int to = end - 3; + if (text.startsWith("json", from) && from + 4 <= to) { + from += 4; + } + while (from < to && isSpace(text.charAt(from))) { + from++; + } + while (to > from && isSpace(text.charAt(to - 1))) { + to--; + } + return text.substring(from, to); + } + + private static boolean isSpace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f' || c == '\r'; + } + /** Extract the first balanced {@code {...}} block, respecting strings and escapes. */ private static String firstJsonObject(String text) { int start = text.indexOf('{'); diff --git a/runtime/java/prompty/src/test/java/com/microsoft/prompty/RuntimeTest.java b/runtime/java/prompty/src/test/java/com/microsoft/prompty/RuntimeTest.java index 405bd3bed..a74663289 100644 --- a/runtime/java/prompty/src/test/java/com/microsoft/prompty/RuntimeTest.java +++ b/runtime/java/prompty/src/test/java/com/microsoft/prompty/RuntimeTest.java @@ -453,6 +453,33 @@ void plainTextRoundsRecordTheAssistantText() { assertTrue(result.toolRequests.isEmpty()); } + @Test + void parseArgumentsStripsFencesWithSurroundingWhitespace() { + assertEquals( + Map.of("a", 1), ToolDispatch.parseArguments(" \n```json \n {\"a\": 1} \n ``` \n")); + assertEquals(Map.of("a", 1), ToolDispatch.parseArguments("```{\"a\": 1}```")); + } + + @Test + void parseArgumentsFallsThroughWhenTheFenceNeverCloses() { + assertEquals(Map.of("a", 1), ToolDispatch.parseArguments("```json\n{\"a\": 1}")); + assertThrows(IllegalArgumentException.class, () -> ToolDispatch.parseArguments("```json\n```")); + assertThrows(IllegalArgumentException.class, () -> ToolDispatch.parseArguments("``````")); + } + + @Test + void parseArgumentsDoesNotBacktrackOnAnUnclosedFenceFollowedByWhitespace() { + for (char filler : new char[] {' ', '\n'}) { + String arguments = "```json" + String.valueOf(filler).repeat(20000) + "x"; + + long started = System.nanoTime(); + assertThrows(IllegalArgumentException.class, () -> ToolDispatch.parseArguments(arguments)); + long elapsedMillis = (System.nanoTime() - started) / 1_000_000; + + assertTrue(elapsedMillis < 1000, "took " + elapsedMillis + " ms"); + } + } + private static Agent agentWithInputs(Map kindsByName) { List inputs = new ArrayList<>(); kindsByName.forEach((name, kind) -> inputs.add(Map.of("name", name, "kind", kind))); diff --git a/runtime/typescript/packages/core/src/core/tool-dispatch.ts b/runtime/typescript/packages/core/src/core/tool-dispatch.ts index 71ce0167f..1f19dd181 100644 --- a/runtime/typescript/packages/core/src/core/tool-dispatch.ts +++ b/runtime/typescript/packages/core/src/core/tool-dispatch.ts @@ -238,6 +238,21 @@ export function extractFirstJsonBlock(text: string): string | null { return null; } +/** + * Return the text inside a surrounding markdown code fence, or null when `raw` + * is not fenced. String operations rather than a regex: the obvious + * `^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$` backtracks cubically on an + * opening fence that never closes, and tool arguments are model output. + */ +function stripCodeFence(raw: string): string | null { + const text = raw.trim(); + if (text.length < 6 || !text.startsWith("```") || !text.endsWith("```")) { + return null; + } + const inner = text.slice(3, -3); + return (inner.startsWith("json") ? inner.slice(4) : inner).trim(); +} + /** * Parse JSON with fallback strategies per spec §9.8. * Returns parsed value on success, or null if all strategies fail. @@ -256,10 +271,10 @@ export function resilientJsonParse( } // Strategy 2: Strip markdown code fences - const fenceMatch = raw.match(/^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$/); - if (fenceMatch) { + const fenced = stripCodeFence(raw); + if (fenced !== null) { try { - const result = JSON.parse(fenceMatch[1]); + const result = JSON.parse(fenced); console.warn( "[prompty] Parsed tool arguments after stripping markdown fences", ); diff --git a/runtime/typescript/packages/core/tests/resilience.test.ts b/runtime/typescript/packages/core/tests/resilience.test.ts index d4e92bbfb..99feb6296 100644 --- a/runtime/typescript/packages/core/tests/resilience.test.ts +++ b/runtime/typescript/packages/core/tests/resilience.test.ts @@ -46,6 +46,25 @@ describe("Resilient JSON Parsing (§9.8)", () => { expect(resilientJsonParse(raw)).toEqual({ temp: 72 }); }); + it("tolerates whitespace around the fences", () => { + expect(resilientJsonParse(' \n```json \n {"a": 1} \n ``` \n')).toEqual({ a: 1 }); + expect(resilientJsonParse('```{"a": 1}```')).toEqual({ a: 1 }); + }); + + it("falls through when the fence never closes", () => { + expect(resilientJsonParse('```json\n{"a": 1}')).toEqual({ a: 1 }); + expect(resilientJsonParse("```json\n```")).toBeNull(); + expect(resilientJsonParse("``````")).toBeNull(); + }); + + it("does not backtrack on an unclosed fence followed by whitespace", () => { + for (const filler of [" ", "\n"]) { + const started = performance.now(); + expect(resilientJsonParse("```json" + filler.repeat(3000) + "x")).toBeNull(); + expect(performance.now() - started).toBeLessThan(1000); + } + }); + it("extracts JSON block from prose", () => { const raw = 'Here is the result: {"city": "NY"} enjoy!'; expect(resilientJsonParse(raw)).toEqual({ city: "NY" }); diff --git a/spec/spec.md b/spec/spec.md index 2bbffa218..f1d812d5e 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -2506,10 +2506,9 @@ function resilient_json_parse(raw_arguments) → dict: return json_parse(raw_arguments) catch: pass - // Strategy 2: Strip markdown code fences - stripped = regex_replace(raw_arguments, - /^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$/s, "$1") - if stripped != raw_arguments: + // Strategy 2: Strip markdown code fences (string operations, see below) + stripped = strip_code_fence(raw_arguments) // null when not fenced + if stripped is not null: try: return json_parse(stripped) catch: pass @@ -2531,6 +2530,26 @@ function resilient_json_parse(raw_arguments) → dict: return null // caller MUST convert to error tool result ``` +`strip_code_fence` is defined with string operations, not a regular expression: + +``` +function strip_code_fence(raw) → string | null: + t = trim_whitespace(raw) + if length(t) < 6 or not starts_with(t, "```") or not ends_with(t, "```"): + return null // not fenced + inner = t[3 : length(t) - 3] + if starts_with(inner, "json"): + inner = inner[4:] // optional language tag + return trim_whitespace(inner) +``` + +The tempting single-regex form, `` /^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$/s ``, returns the +same text but backtracks cubically when a fence opens and never closes (for +example an opening fence and `json` followed by 10,000 spaces and no closing +fence), and the arguments string is model output. Implementations MUST NOT +use a backtracking regex for this step. Linear-time engines (Rust's `regex`, Go's +`regexp`) are not affected. + **Requirements:** - Implementations SHOULD attempt all four strategies in order. diff --git a/web/src/content/docs/specification/agent-loop.mdx b/web/src/content/docs/specification/agent-loop.mdx index 224f990fc..587830d72 100644 --- a/web/src/content/docs/specification/agent-loop.mdx +++ b/web/src/content/docs/specification/agent-loop.mdx @@ -269,10 +269,9 @@ function resilient_json_parse(raw_arguments) → dict: return json_parse(raw_arguments) catch: pass - // Strategy 2: Strip markdown code fences - stripped = regex_replace(raw_arguments, - /^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$/s, "$1") - if stripped != raw_arguments: + // Strategy 2: Strip markdown code fences (string operations, see below) + stripped = strip_code_fence(raw_arguments) // null when not fenced + if stripped is not null: try: return json_parse(stripped) catch: pass @@ -294,6 +293,26 @@ function resilient_json_parse(raw_arguments) → dict: return null // caller MUST convert to error tool result ``` +`strip_code_fence` is defined with string operations, not a regular expression: + +``` +function strip_code_fence(raw) → string | null: + t = trim_whitespace(raw) + if length(t) < 6 or not starts_with(t, "```") or not ends_with(t, "```"): + return null // not fenced + inner = t[3 : length(t) - 3] + if starts_with(inner, "json"): + inner = inner[4:] // optional language tag + return trim_whitespace(inner) +``` + +The tempting single-regex form, `` /^\s*```(?:json)?\s*\n?(.*?)\n?\s*```\s*$/s ``, returns the +same text but backtracks cubically when a fence opens and never closes (for +example an opening fence and `json` followed by 10,000 spaces and no closing +fence), and the arguments string is model output. Implementations MUST NOT +use a backtracking regex for this step. Linear-time engines (Rust's `regex`, Go's +`regexp`) are not affected. + **Requirements:** - Implementations SHOULD attempt all four strategies in order.