From cb3b74b0ad849ce181dd40b1ee2967de1216bfc3 Mon Sep 17 00:00:00 2001 From: xThreeh Date: Wed, 7 Oct 2026 15:59:24 -0300 Subject: [PATCH] Python: Fix text chunker merging the last paragraph past max_tokens _split_text_paragraph merges a short last paragraph into the previous one, but the merge check counted words (split(" ")) instead of calling token_counter. A word is usually more than one token, so the merged paragraph could exceed max_tokens. Measure the merged text with token_counter and only merge when it fits. The merged text itself is unchanged. Eight existing tests expected a last chunk above max_token_per_line (16 tokens with a limit of 15, 18 with 17); their last chunk is now split instead. A new test checks that no paragraph exceeds the limit. Fixes #14566 --- python/semantic_kernel/text/text_chunker.py | 14 +++----- python/tests/unit/text/test_text_chunker.py | 39 ++++++++++++++++----- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/python/semantic_kernel/text/text_chunker.py b/python/semantic_kernel/text/text_chunker.py index f79796399b6d..fa6bde5c14ed 100644 --- a/python/semantic_kernel/text/text_chunker.py +++ b/python/semantic_kernel/text/text_chunker.py @@ -137,16 +137,10 @@ def _split_text_paragraph(text: list[str], max_tokens: int, token_counter: Calla sec_last_para = paragraphs[-2] if token_counter(last_para) < max_tokens / 4: - last_para_tokens = last_para.split(" ") - sec_last_para_tokens = sec_last_para.split(" ") - last_para_token_count = len(last_para_tokens) - sec_last_para_token_count = len(sec_last_para_tokens) - - if last_para_token_count + sec_last_para_token_count <= max_tokens: - sec_last_para = " ".join(sec_last_para_tokens) + NEWLINE - last_para = " ".join(last_para_tokens) - new_sec_last_para = sec_last_para + last_para - paragraphs[-2] = new_sec_last_para.strip() + new_sec_last_para = f"{sec_last_para}{NEWLINE}{last_para}".strip() + + if token_counter(new_sec_last_para) <= max_tokens: + paragraphs[-2] = new_sec_last_para paragraphs.pop() return paragraphs diff --git a/python/tests/unit/text/test_text_chunker.py b/python/tests/unit/text/test_text_chunker.py index 648f84b752c5..e3e8cc73e495 100644 --- a/python/tests/unit/text/test_text_chunker.py +++ b/python/tests/unit/text/test_text_chunker.py @@ -160,12 +160,28 @@ def test_split_text_paragraph_evenly(): "This is only a test.", "We repeat, this is only a test. A unit test.", "A small note. And another. And once again.", - f"Seriously, this is the end. We're finished. All set. Bye.{NEWLINE}Done.", + "Seriously, this is the end. We're finished. All set. Bye.", + "Done.", ] split = split_plaintext_paragraph(text, max_token_per_line) assert expected == split +def test_split_text_paragraph_evenly_respects_max_tokens(): + """Test split_paragraph() does not merge the last paragraph past max_tokens""" + text = [ + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test.", + "A small note. And another. And once again. Seriously, this is the end. " + "We're finished. All set. Bye.", + "Done.", + ] + + max_token_per_line = 15 + + split = split_plaintext_paragraph(text, max_token_per_line) + assert all(len(paragraph) // 4 <= max_token_per_line for paragraph in split) + + def test_split_text_paragraph_evenly_2(): """Test split_paragraph() with evenly split input""" text = [ @@ -187,7 +203,8 @@ def test_split_text_paragraph_evenly_2(): "The sun set over the horizon peacefully, the beautiful star.", f"Cats love boxes.{NEWLINE}That is something. Incredible news that is.", f"What a beautiful day to be alive.{NEWLINE}Seriously, this is the end.", - f"We're finished once of for all. All set. Ok.{NEWLINE}Done.{NEWLINE}" + f"Or is it?{NEWLINE}Surprise!", + f"We're finished once of for all. All set. Ok.{NEWLINE}Done.{NEWLINE}" + "Or is it?", + "Surprise!", ] split = split_plaintext_paragraph(text, max_token_per_line) assert expected == split @@ -247,7 +264,8 @@ def test_split_paragraph_semicolon(): "This is only a test", "We repeat; this is only a test; A unit test", "A small note; And another; And once again;", - f"Seriously, this is the end; We're finished; All set; Bye.{NEWLINE}Done.", + "Seriously, this is the end; We're finished; All set; Bye.", + "Done.", ] max_token_per_line = 15 split = split_plaintext_paragraph(text, max_token_per_line) @@ -267,7 +285,8 @@ def test_split_paragraph_colon(): "This is only a test", "We repeat: this is only a test: A unit test", "A small note: And another: And once again:", - f"Seriously, this is the end: We're finished: All set: Bye.{NEWLINE}Done.", + "Seriously, this is the end: We're finished: All set: Bye.", + "Done.", ] max_token_per_line = 15 split = split_plaintext_paragraph(text, max_token_per_line) @@ -388,7 +407,8 @@ def test_split_md_on_dot(): "This\n is only a test", "We repeat. this is only a test. A unit test", "A small note. And another. And once again.", - f"Seriously, this is the end. We're finished. All set. Bye.{NEWLINE}Done.", + "Seriously, this is the end. We're finished. All set. Bye.", + "Done.", ] max_token_per_line = 15 split = split_markdown_paragraph(text, max_token_per_line) @@ -408,7 +428,8 @@ def test_split_md_on_colon(): "This is only a test", "We repeat: this is only a test: A unit test", "A small note: And another: And once again:", - f"Seriously, this is the end: We're finished: All set: Bye.{NEWLINE}Done.", + "Seriously, this is the end: We're finished: All set: Bye.", + "Done.", ] max_token_per_line = 15 split = split_markdown_paragraph(text, max_token_per_line) @@ -428,7 +449,8 @@ def test_split_md_on_punctuation(): "This\n is only a test", "We repeat? this is only a test! A unit test", "A small note? And another! And once again?", - f"Seriously, this is the end! We're finished! All set! Bye.{NEWLINE}Done.", + "Seriously, this is the end! We're finished! All set! Bye.", + "Done.", ] max_token_per_line = 15 split = split_markdown_paragraph(text, max_token_per_line) @@ -448,7 +470,8 @@ def test_split_md_on_semicolon(): "This is only a test", "We repeat; this is only a test; A unit test", "A small note; And another; And once again;", - f"Seriously, this is the end; We're finished; All set; Bye.{NEWLINE}Done.", + "Seriously, this is the end; We're finished; All set; Bye.", + "Done.", ] max_token_per_line = 15 split = split_markdown_paragraph(text, max_token_per_line)