diff --git a/python/semantic_kernel/template_engine/code_tokenizer.py b/python/semantic_kernel/template_engine/code_tokenizer.py index 7b4ebb9c4a2a..f95119477d0e 100644 --- a/python/semantic_kernel/template_engine/code_tokenizer.py +++ b/python/semantic_kernel/template_engine/code_tokenizer.py @@ -70,7 +70,7 @@ def tokenize(text: str) -> list[Block]: current_token_content.append(current_char) continue - # While reading values between quotes + # While reading values between quotes (positional value or named-arg value) if current_token_type in (BlockTypes.VALUE, BlockTypes.NAMED_ARG): # If the current char is escaping the next special char we: # - skip the current char (escape char) @@ -89,7 +89,10 @@ def tokenize(text: str) -> list[Block]: # When we reach the end of the value, we add the block if current_char == text_value_delimiter: - blocks.append(ValBlock(content="".join(current_token_content))) + if current_token_type == BlockTypes.NAMED_ARG: + blocks.append(NamedArgBlock(content="".join(current_token_content))) + else: + blocks.append(ValBlock(content="".join(current_token_content))) current_token_content.clear() current_token_type = None space_separator_found = False @@ -136,6 +139,16 @@ def tokenize(text: str) -> list[Block]: else: # A function id starts here current_token_type = BlockTypes.FUNCTION_ID + elif ( + current_token_type == BlockTypes.FUNCTION_ID + and current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE) + and Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content + ): + # We just appended the opening quote of a named-arg value (e.g. key='value'). + # Switch to NAMED_ARG quoted-value mode so spaces inside the value + # are not treated as token separators. + current_token_type = BlockTypes.NAMED_ARG + text_value_delimiter = current_char # end of main for loop @@ -144,6 +157,8 @@ def tokenize(text: str) -> list[Block]: if current_token_type == BlockTypes.VALUE: blocks.append(ValBlock(content="".join(current_token_content))) + elif current_token_type == BlockTypes.NAMED_ARG: + blocks.append(NamedArgBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.VARIABLE: blocks.append(VarBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.FUNCTION_ID: diff --git a/python/tests/unit/template_engine/test_code_tokenizer.py b/python/tests/unit/template_engine/test_code_tokenizer.py index 42024c6120ad..ab5bb18bb87a 100644 --- a/python/tests/unit/template_engine/test_code_tokenizer.py +++ b/python/tests/unit/template_engine/test_code_tokenizer.py @@ -128,3 +128,25 @@ def test_named_args(): assert blocks[1].content == '"direct"' assert blocks[2].content == "arg1=$arg1" assert blocks[3].content == 'arg2="arg2"' + + +def test_named_args_with_spaces_in_quoted_value(): + """Regression test for issue #14460: spaces inside quoted named-arg values should not split tokens.""" + blocks = CodeTokenizer.tokenize("func key='hello world'") + assert len(blocks) == 2 + assert blocks[0].content == "func" + assert blocks[0].type == BlockTypes.FUNCTION_ID + assert blocks[1].content == "key='hello world'" + assert blocks[1].type == BlockTypes.NAMED_ARG + + blocks = CodeTokenizer.tokenize('func key="hello world"') + assert len(blocks) == 2 + assert blocks[0].content == "func" + assert blocks[1].content == 'key="hello world"' + assert blocks[1].type == BlockTypes.NAMED_ARG + + blocks = CodeTokenizer.tokenize("plugin.func greeting='hello world' name='john doe'") + assert len(blocks) == 3 + assert blocks[0].content == "plugin.func" + assert blocks[1].content == "greeting='hello world'" + assert blocks[2].content == "name='john doe'"