Skip to content

Commit 42d1a8e

Browse files
author
SqlRush
committed
Keep chunked emoji modifier sequences together
1 parent 1ed803e commit 42d1a8e

5 files changed

Lines changed: 57 additions & 3 deletions

File tree

‎docs/cc-100-roadmap.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1004,7 +1004,7 @@ M7 补充:prompt history `LogEntry` 读取现在接受 `sessionID`/`session`/`
10041004

10051005
本轮补充:terminal parser 的 text grapheme 基础分段补齐 combining mark、variation selector、emoji modifier、ZWJ emoji 序列和 regional indicator flag pair;宽度计算现在让 base+combining-mark cluster 保持 base glyph 宽度,emoji presentation/ZWJ/flag 仍按宽 grapheme 处理,完整 Unicode UAX #29 分段仍未宣称完成。
10061006

1007-
本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
1007+
本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、emoji modifier sequence、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
10081008

10091009
本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 black-flag base + tag chars + cancel tag 作为单个宽 grapheme,完整输入以及跨 `Feed()` 边界切在 base emoji 或 tag char 后时都不会拆分视觉 emoji。
10101010

‎docs/claude-code-go-rewrite-plan.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,7 @@ test/parity/ # golden tests against TS/official behavior
570570
- 本轮补充:terminal parser 增加轻量 ANSI action pipeline,串接 tokenizer、CSI/OSC/ESC dispatcher 和 SGR style state,输出 text/bell/cursor/erase/scroll/mode/title/link/tabStatus/reset/unknown action,文本宽度覆盖 ASCII、emoji 和 East Asian wide;完整 grapheme cluster segmentation 与 renderer style 应用仍继续推进。
571571
- 本轮补充:terminal parser 跟踪 OSC 8 hyperlink start/end 状态,暴露当前 `inLink` 和 `linkUrl`,reset 时清空链接状态,贴近官方 parser 的 link range 状态语义。
572572
- 本轮补充:terminal parser 的 text grapheme 基础分段补齐 combining mark、variation selector、emoji modifier、ZWJ emoji 序列和 regional indicator flag pair;宽度计算现在让 base+combining-mark cluster 保持 base glyph 宽度,emoji presentation/ZWJ/flag 仍按宽 grapheme 处理,完整 Unicode UAX #29 分段仍未宣称完成。
573-
- 本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
573+
- 本轮补充:terminal parser 的 streaming text action 现在会暂存可能跨 chunk 延续的末尾 grapheme;ZWJ emoji、emoji modifier sequence、regional indicator flag 和未完成 emoji tag sequence 跨 `Feed()` 边界时不会被拆成两个宽字符,遇到控制序列或 `Flush()` 会先落地 pending text。
574574
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji tag sequence,把 subdivision flag 这类 black-flag base + tag chars + cancel tag 作为单个宽 grapheme,完整输入以及跨 `Feed()` 边界切在 base emoji 或 tag char 后时都不会拆分视觉 emoji。
575575
- 本轮补充:terminal CSI parser 现在对 tokenizer flush 出来的非 final-byte incomplete CSI 返回 unknown action,而不是丢弃,贴近官方 `parseCSI` 对 flushed partial sequence 的 fallback 行为。
576576
- 本轮补充:terminal sequence dispatcher 对 tokenizer flush 出来的 OSC partial sequence 使用 `ParseOSCContent` fallback,允许无 BEL/ST terminator 的 title/link/tab-status content 按官方 parser 语义产出 action。

‎docs/first-second-parity-audit.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,7 @@ M7 progress now includes:
429429
- `internal/tui`: terminal tokenizer now keeps SS3 parameter bytes buffered until a final byte, so modified SS3 cursor sequences such as `ESC O 1;5D` can cross chunk boundaries and still reach the dispatcher as one sequence token.
430430
- `internal/tui`: terminal grapheme width now keeps base+combining-mark clusters at the base glyph width while preserving wide treatment for emoji presentation, ZWJ, regional-indicator, and tag sequences.
431431
- `internal/tui`: terminal grapheme segmentation now keeps emoji tag sequences such as subdivision flags as one wide grapheme across full input and streaming chunks split after the black-flag base or tag characters.
432+
- `internal/tui`: terminal grapheme segmentation now keeps emoji modifier sequences such as `👋🏽` as one wide grapheme across full input and streaming chunks split after the modifier base.
432433
- `internal/tui`: terminal grapheme segmentation now treats emoji keycap sequences such as `1️⃣` and `2⃣` as single wide graphemes, including streaming chunks split after the keycap base or variation selector.
433434
- `internal/tui`: terminal grapheme segmentation now applies Hangul L/V/T jamo joining, keeping decomposed syllables such as `한` as one wide grapheme across full input and streaming chunk boundaries.
434435
- `internal/tui`: terminal grapheme segmentation now keeps CRLF as a single zero-width line-break grapheme across full input and streaming chunks split between `\r` and `\n`, and wrap/trim/render width paths share the same line-break check.

‎internal/tui/terminal_parser.go‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func terminalGraphemeMayContinueAtChunkBoundary(value string) bool {
274274
return true
275275
}
276276
last, _ := utf8.DecodeLastRuneInString(value)
277-
if last == 0x200d || isTerminalVirama(last) || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeCanStartEmojiTagSequence(value) || terminalGraphemeMayContinueHangul(value) {
277+
if last == 0x200d || isTerminalVirama(last) || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeCanStartEmojiModifierSequence(value) || terminalGraphemeCanStartEmojiTagSequence(value) || terminalGraphemeMayContinueHangul(value) {
278278
return true
279279
}
280280
regionalCount := 0
@@ -532,6 +532,14 @@ func terminalGraphemeCanStartEmojiTagSequence(value string) bool {
532532
return len(runes) == 1 && isTerminalEmojiTagBase(runes[0])
533533
}
534534

535+
func terminalGraphemeCanStartEmojiModifierSequence(value string) bool {
536+
runes := []rune(value)
537+
if len(runes) == 1 {
538+
return isTerminalEmojiModifierBase(runes[0])
539+
}
540+
return len(runes) == 2 && isTerminalEmojiModifierBase(runes[0]) && runes[1] == 0xfe0f
541+
}
542+
535543
func isTerminalLineBreakGrapheme(value string) bool {
536544
return value == "\n" || value == "\r" || value == "\r\n"
537545
}
@@ -696,6 +704,39 @@ func isTerminalEmojiModifier(r rune) bool {
696704
return r >= 0x1f3fb && r <= 0x1f3ff
697705
}
698706

707+
func isTerminalEmojiModifierBase(r rune) bool {
708+
switch {
709+
case r == 0x261d || r == 0x26f9:
710+
return true
711+
case r >= 0x270a && r <= 0x270d:
712+
return true
713+
case r == 0x1f385 || (r >= 0x1f3c2 && r <= 0x1f3c4) || r == 0x1f3c7 || (r >= 0x1f3ca && r <= 0x1f3cc):
714+
return true
715+
case (r >= 0x1f442 && r <= 0x1f443) || (r >= 0x1f446 && r <= 0x1f450):
716+
return true
717+
case (r >= 0x1f466 && r <= 0x1f478) || r == 0x1f47c || (r >= 0x1f481 && r <= 0x1f483):
718+
return true
719+
case (r >= 0x1f485 && r <= 0x1f487) || r == 0x1f4aa:
720+
return true
721+
case (r >= 0x1f574 && r <= 0x1f575) || r == 0x1f57a || r == 0x1f590 || (r >= 0x1f595 && r <= 0x1f596):
722+
return true
723+
case (r >= 0x1f645 && r <= 0x1f647) || (r >= 0x1f64b && r <= 0x1f64f):
724+
return true
725+
case r == 0x1f6a3 || (r >= 0x1f6b4 && r <= 0x1f6b6) || r == 0x1f6c0 || r == 0x1f6cc:
726+
return true
727+
case (r >= 0x1f90c && r <= 0x1f91f) || r == 0x1f926 || (r >= 0x1f930 && r <= 0x1f939):
728+
return true
729+
case (r >= 0x1f93c && r <= 0x1f93e) || (r >= 0x1f9b5 && r <= 0x1f9b6) || (r >= 0x1f9b8 && r <= 0x1f9b9):
730+
return true
731+
case r == 0x1f9bb || (r >= 0x1f9cd && r <= 0x1f9cf) || (r >= 0x1f9d1 && r <= 0x1f9dd):
732+
return true
733+
case r >= 0x1faf0 && r <= 0x1faf8:
734+
return true
735+
default:
736+
return false
737+
}
738+
}
739+
699740
func isTerminalEmojiKeycapBase(r rune) bool {
700741
return r == '#' || r == '*' || (r >= '0' && r <= '9')
701742
}

‎internal/tui/terminal_parser_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,18 @@ func TestTerminalParserKeepsChunkedGraphemeClustersTogether(t *testing.T) {
329329
t.Fatalf("emoji tag grapheme = %#v", got)
330330
}
331331

332+
parser = NewTerminalParser()
333+
if actions := parser.Feed("\U0001f44b"); len(actions) != 0 {
334+
t.Fatalf("partial emoji modifier base actions = %#v", actions)
335+
}
336+
actions = parser.Feed("\U0001f3fd!")
337+
if len(actions) != 1 || len(actions[0].Graphemes) != 2 {
338+
t.Fatalf("emoji modifier actions = %#v", actions)
339+
}
340+
if got := actions[0].Graphemes[0]; got.Value != "\U0001f44b\U0001f3fd" || got.Width != 2 {
341+
t.Fatalf("emoji modifier grapheme = %#v", got)
342+
}
343+
332344
parser = NewTerminalParser()
333345
if actions := parser.Feed("1"); len(actions) != 0 {
334346
t.Fatalf("partial keycap base actions = %#v", actions)

0 commit comments

Comments
 (0)