From a30252192131ef8214e1e62ed727f1f1c0dc41bd Mon Sep 17 00:00:00 2001 From: "shuwen.wu" Date: Fri, 25 Sep 2026 16:07:05 +0800 Subject: [PATCH 1/2] fix: CodeTokenizer: handle spaces inside quoted named-arg values When parsing templates like {{func key='hello world'}}, the tokenizer incorrectly included spaces inside quoted named-arg values because the VALUE state did not check for a closing delimiter when a space was encountered. Two changes: 1. A space in VALUE mode now only ends the value when followed by a non-alphanumeric, non-$-, non-quote character (i.e., a genuine token separator, not part of the quoted text). 2. The closing delimiter check now additionally verifies that the character after the quote is not alphanumeric, which correctly handles quoted values adjacent to punctuation like }}. Fixes #14460. --- .../template_engine/code_tokenizer.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/python/semantic_kernel/template_engine/code_tokenizer.py b/python/semantic_kernel/template_engine/code_tokenizer.py index 7b4ebb9c4a2a..d924e1fbeed8 100644 --- a/python/semantic_kernel/template_engine/code_tokenizer.py +++ b/python/semantic_kernel/template_engine/code_tokenizer.py @@ -85,10 +85,29 @@ def tokenize(text: str) -> list[Block]: skip_next_char = True continue + # A space inside a quoted value ends the value only when the + # following character is not a valid value continuation. + # Valid continuations: alphanumeric, quote, $, _, - + if current_char == Symbols.SPACE and next_char not in ( + Symbols.SGL_QUOTE, + Symbols.DBL_QUOTE, + Symbols.VAR_PREFIX, + ) and not next_char.isalnum() and next_char not in ("_", "-"): + # Space ends the quoted value + blocks.append(ValBlock(content="".join(current_token_content))) + current_token_content.clear() + current_token_type = None + space_separator_found = True + continue + current_token_content.append(current_char) - # When we reach the end of the value, we add the block - if current_char == text_value_delimiter: + # When we reach the end of the value, we add the block. + # The closing delimiter must not be followed by alphanumeric chars + # (which would mean the delimiter is inside the quoted value). + if current_char == text_value_delimiter and ( + index + 2 >= len(text) or not text[index + 2].isalnum() + ): blocks.append(ValBlock(content="".join(current_token_content))) current_token_content.clear() current_token_type = None From 5592924daf28be4384455ccbdb77f271213090e9 Mon Sep 17 00:00:00 2001 From: SK Fix Bot Date: Thu, 8 Oct 2026 13:54:57 +0800 Subject: [PATCH 2/2] fix: properly handle spaces in quoted named-arg values The previous fix had two issues: 1. It added a broken space-handling branch that incorrectly terminated ValBlocks on spaces, causing tokens after quoted values to be consumed into the ValBlock. 2. The closing-delimiter check used text[index+2] (off-by-one), preventing legitimate closing quotes (e.g. in escape sequences like 'f\\'xy') from being detected. 3. It never added the state transition from FUNCTION_ID to NAMED_ARG when the opening quote appears mid-token after '=', which is the root cause of the original bug. The root cause was that when parsing 'key='hello world'', the opening quote appears after '=' (still mid-FUNCTION_ID token), not after a space separator. The code never switched into quoted-value mode, so the space inside 'hello world' incorrectly terminated the FUNCTION_ID token. Fix: - Remove the broken space-handling and incorrect index+2 check. - When in FUNCTION_ID mode and the current char is a quote and '=' is already present in the token content, switch to NAMED_ARG mode (setting text_value_delimiter) so that spaces inside the quoted value are preserved and the closing quote ends the NamedArgBlock. - When closing a quoted block, create NamedArgBlock (not ValBlock) if in NAMED_ARG mode. - Handle NAMED_ARG in the end-of-loop final-capture. - Add regression test for issue #14460. --- .../template_engine/code_tokenizer.py | 42 +++++++++---------- .../template_engine/test_code_tokenizer.py | 22 ++++++++++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/python/semantic_kernel/template_engine/code_tokenizer.py b/python/semantic_kernel/template_engine/code_tokenizer.py index d924e1fbeed8..f95119477d0e 100644 --- a/python/semantic_kernel/template_engine/code_tokenizer.py +++ b/python/semantic_kernel/template_engine/code_tokenizer.py @@ -70,7 +70,7 @@ def tokenize(text: str) -> list[Block]: current_token_content.append(current_char) continue - # While reading values between quotes + # While reading values between quotes (positional value or named-arg value) if current_token_type in (BlockTypes.VALUE, BlockTypes.NAMED_ARG): # If the current char is escaping the next special char we: # - skip the current char (escape char) @@ -85,30 +85,14 @@ def tokenize(text: str) -> list[Block]: skip_next_char = True continue - # A space inside a quoted value ends the value only when the - # following character is not a valid value continuation. - # Valid continuations: alphanumeric, quote, $, _, - - if current_char == Symbols.SPACE and next_char not in ( - Symbols.SGL_QUOTE, - Symbols.DBL_QUOTE, - Symbols.VAR_PREFIX, - ) and not next_char.isalnum() and next_char not in ("_", "-"): - # Space ends the quoted value - blocks.append(ValBlock(content="".join(current_token_content))) - current_token_content.clear() - current_token_type = None - space_separator_found = True - continue - current_token_content.append(current_char) - # When we reach the end of the value, we add the block. - # The closing delimiter must not be followed by alphanumeric chars - # (which would mean the delimiter is inside the quoted value). - if current_char == text_value_delimiter and ( - index + 2 >= len(text) or not text[index + 2].isalnum() - ): - blocks.append(ValBlock(content="".join(current_token_content))) + # When we reach the end of the value, we add the block + if current_char == text_value_delimiter: + if current_token_type == BlockTypes.NAMED_ARG: + blocks.append(NamedArgBlock(content="".join(current_token_content))) + else: + blocks.append(ValBlock(content="".join(current_token_content))) current_token_content.clear() current_token_type = None space_separator_found = False @@ -155,6 +139,16 @@ def tokenize(text: str) -> list[Block]: else: # A function id starts here current_token_type = BlockTypes.FUNCTION_ID + elif ( + current_token_type == BlockTypes.FUNCTION_ID + and current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE) + and Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content + ): + # We just appended the opening quote of a named-arg value (e.g. key='value'). + # Switch to NAMED_ARG quoted-value mode so spaces inside the value + # are not treated as token separators. + current_token_type = BlockTypes.NAMED_ARG + text_value_delimiter = current_char # end of main for loop @@ -163,6 +157,8 @@ def tokenize(text: str) -> list[Block]: if current_token_type == BlockTypes.VALUE: blocks.append(ValBlock(content="".join(current_token_content))) + elif current_token_type == BlockTypes.NAMED_ARG: + blocks.append(NamedArgBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.VARIABLE: blocks.append(VarBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.FUNCTION_ID: diff --git a/python/tests/unit/template_engine/test_code_tokenizer.py b/python/tests/unit/template_engine/test_code_tokenizer.py index 42024c6120ad..ab5bb18bb87a 100644 --- a/python/tests/unit/template_engine/test_code_tokenizer.py +++ b/python/tests/unit/template_engine/test_code_tokenizer.py @@ -128,3 +128,25 @@ def test_named_args(): assert blocks[1].content == '"direct"' assert blocks[2].content == "arg1=$arg1" assert blocks[3].content == 'arg2="arg2"' + + +def test_named_args_with_spaces_in_quoted_value(): + """Regression test for issue #14460: spaces inside quoted named-arg values should not split tokens.""" + blocks = CodeTokenizer.tokenize("func key='hello world'") + assert len(blocks) == 2 + assert blocks[0].content == "func" + assert blocks[0].type == BlockTypes.FUNCTION_ID + assert blocks[1].content == "key='hello world'" + assert blocks[1].type == BlockTypes.NAMED_ARG + + blocks = CodeTokenizer.tokenize('func key="hello world"') + assert len(blocks) == 2 + assert blocks[0].content == "func" + assert blocks[1].content == 'key="hello world"' + assert blocks[1].type == BlockTypes.NAMED_ARG + + blocks = CodeTokenizer.tokenize("plugin.func greeting='hello world' name='john doe'") + assert len(blocks) == 3 + assert blocks[0].content == "plugin.func" + assert blocks[1].content == "greeting='hello world'" + assert blocks[2].content == "name='john doe'"