Skip to content

Commit 1ed803e

Browse files
author
SqlRush
committed
Keep chunked emoji tag sequences together
1 parent fde3d5d commit 1ed803e

5 files changed

Lines changed: 25 additions & 3 deletions

File tree

docs/cc-100-roadmap.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1006,7 +1006,7 @@ M7 补充:prompt history `LogEntry` 读取现在接受 `sessionID`/`session`/`
10061006

10071007
本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
10081008

1009-
本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 base emoji + tag chars + cancel tag 作为单个宽 grapheme,避免 wrap/pad/cursor pipeline 拆分视觉 emoji。
1009+
本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 black-flag base + tag chars + cancel tag 作为单个宽 grapheme,完整输入以及跨 `Feed()` 边界切在 base emoji 或 tag char 后时都不会拆分视觉 emoji。
10101010

10111011
本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji keycap sequence,`1️⃣`/`2⃣` 这类 keycap 在完整输入和跨 `Feed()` 边界切在 base 或 variation selector 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
10121012

docs/claude-code-go-rewrite-plan.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -571,7 +571,7 @@ test/parity/ # golden tests against TS/official behavior
571571
- 本轮补充:terminal parser 跟踪 OSC 8 hyperlink start/end 状态,暴露当前 `inLink``linkUrl`,reset 时清空链接状态,贴近官方 parser 的 link range 状态语义。
572572
- 本轮补充:terminal parser 的 text grapheme 基础分段补齐 combining mark、variation selector、emoji modifier、ZWJ emoji 序列和 regional indicator flag pair;宽度计算现在让 base+combining-mark cluster 保持 base glyph 宽度,emoji presentation/ZWJ/flag 仍按宽 grapheme 处理,完整 Unicode UAX #29 分段仍未宣称完成。
573573
- 本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
574-
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 base emoji + tag chars + cancel tag 作为单个宽 grapheme,避免 wrap/pad/cursor pipeline 拆分视觉 emoji。
574+
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 black-flag base + tag chars + cancel tag 作为单个宽 grapheme,完整输入以及跨 `Feed()` 边界切在 base emoji 或 tag char 后时都不会拆分视觉 emoji。
575575
- 本轮补充:terminal CSI parser 现在对 tokenizer flush 出来的非 final-byte incomplete CSI 返回 unknown action,而不是丢弃,贴近官方 `parseCSI` 对 flushed partial sequence 的 fallback 行为。
576576
- 本轮补充:terminal sequence dispatcher 对 tokenizer flush 出来的 OSC partial sequence 使用 `ParseOSCContent` fallback,允许无 BEL/ST terminator 的 title/link/tab-status content 按官方 parser 语义产出 action。
577577
- 本轮补充:terminal tokenizer 增加明确的 output/input 构造器,output 路径默认不吞 `CSI M` 后续字节,input 路径默认开启 X10 mouse payload 边界消费,避免调用方误用布尔选项导致 output parser 吞文本或 stdin mouse payload 泄漏。

docs/first-second-parity-audit.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,7 @@ M7 progress now includes:
428428
- `internal/tui`: terminal sequence dispatch now also parses modified SS3 application cursor sequences such as `ESC O 1;2A`, `ESC O 1;5B`, and `ESC O 1;16D` as structured cursor move actions instead of unknown SS3 sequences.
429429
- `internal/tui`: terminal tokenizer now keeps SS3 parameter bytes buffered until a final byte, so modified SS3 cursor sequences such as `ESC O 1;5D` can cross chunk boundaries and still reach the dispatcher as one sequence token.
430430
- `internal/tui`: terminal grapheme width now keeps base+combining-mark clusters at the base glyph width while preserving wide treatment for emoji presentation, ZWJ, regional-indicator, and tag sequences.
431+
- `internal/tui`: terminal grapheme segmentation now keeps emoji tag sequences such as subdivision flags as one wide grapheme across full input and streaming chunks split after the black-flag base or tag characters.
431432
- `internal/tui`: terminal grapheme segmentation now treats emoji keycap sequences such as `1️⃣` and `2⃣` as single wide graphemes, including streaming chunks split after the keycap base or variation selector.
432433
- `internal/tui`: terminal grapheme segmentation now applies Hangul L/V/T jamo joining, keeping decomposed syllables such as `한` as one wide grapheme across full input and streaming chunk boundaries.
433434
- `internal/tui`: terminal grapheme segmentation now keeps CRLF as a single zero-width line-break grapheme across full input and streaming chunks split between `\r` and `\n`, and wrap/trim/render width paths share the same line-break check.

internal/tui/terminal_parser.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func terminalGraphemeMayContinueAtChunkBoundary(value string) bool {
274274
return true
275275
}
276276
last, _ := utf8.DecodeLastRuneInString(value)
277-
if last == 0x200d || isTerminalVirama(last) || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeMayContinueHangul(value) {
277+
if last == 0x200d || isTerminalVirama(last) || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeCanStartEmojiTagSequence(value) || terminalGraphemeMayContinueHangul(value) {
278278
return true
279279
}
280280
regionalCount := 0
@@ -527,6 +527,11 @@ func terminalGraphemeCanStartKeycapSequence(value string) bool {
527527
return len(runes) == 2 && isTerminalEmojiKeycapBase(runes[0]) && runes[1] == 0xfe0f
528528
}
529529

530+
func terminalGraphemeCanStartEmojiTagSequence(value string) bool {
531+
runes := []rune(value)
532+
return len(runes) == 1 && isTerminalEmojiTagBase(runes[0])
533+
}
534+
530535
func isTerminalLineBreakGrapheme(value string) bool {
531536
return value == "\n" || value == "\r" || value == "\r\n"
532537
}
@@ -703,6 +708,10 @@ func isTerminalEmojiTag(r rune) bool {
703708
return (r >= 0xe0020 && r <= 0xe007e) || r == 0xe007f
704709
}
705710

711+
func isTerminalEmojiTagBase(r rune) bool {
712+
return r == 0x1f3f4
713+
}
714+
706715
func isTerminalRegionalIndicator(r rune) bool {
707716
return r >= 0x1f1e0 && r <= 0x1f1ff
708717
}

internal/tui/terminal_parser_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,18 @@ func TestTerminalParserKeepsChunkedGraphemeClustersTogether(t *testing.T) {
317317
t.Fatalf("regional width = %d", width)
318318
}
319319

320+
parser = NewTerminalParser()
321+
if actions := parser.Feed("\U0001f3f4"); len(actions) != 0 {
322+
t.Fatalf("partial emoji tag base actions = %#v", actions)
323+
}
324+
actions = parser.Feed("\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f!")
325+
if len(actions) != 1 || len(actions[0].Graphemes) != 2 {
326+
t.Fatalf("emoji tag actions = %#v", actions)
327+
}
328+
if got := actions[0].Graphemes[0]; got.Value != "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f" || got.Width != 2 {
329+
t.Fatalf("emoji tag grapheme = %#v", got)
330+
}
331+
320332
parser = NewTerminalParser()
321333
if actions := parser.Feed("1"); len(actions) != 0 {
322334
t.Fatalf("partial keycap base actions = %#v", actions)

0 commit comments

Comments
 (0)