Skip to content

Commit fde3d5d

Browse files
author
SqlRush
committed
Keep Indic virama graphemes together
1 parent 12b1d25 commit fde3d5d

5 files changed

Lines changed: 49 additions & 2 deletions

File tree

docs/cc-100-roadmap.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,6 +1016,8 @@ M7 补充:prompt history `LogEntry` 读取现在接受 `sessionID`/`session`/`
10161016

10171017
本轮补充:terminal parser 的 text grapheme 分段继续补齐 Unicode mark category 和 Prepend 规则,nonspacing/enclosing/spacing mark 会归入前一 cluster,`का` 这类 spacing mark cluster 不再被拆宽,Arabic prepend mark 加 base text 会保持同一 grapheme,单独 prepend mark flush 时按零宽处理;完整 Unicode UAX #29 分段仍未宣称完成。
10181018

1019+
本轮补充:terminal parser 的 text grapheme 分段继续补齐常见 Indic virama conjunct,`क्ष` 这类 consonant + virama + consonant cluster 在完整输入和跨 `Feed()` 边界切在 virama 后时都会保持单个窄 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
1020+
10191021
本轮补充:terminal CSI parser 现在对 tokenizer flush 出来的非 final-byte incomplete CSI 返回 unknown action,而不是丢弃,贴近官方 `parseCSI` 对 flushed partial sequence 的 fallback 行为。
10201022

10211023
本轮补充:terminal sequence dispatcher 对 tokenizer flush 出来的 OSC partial sequence 使用 `ParseOSCContent` fallback,允许无 BEL/ST terminator 的 title/link/tab-status content 按官方 parser 语义产出 action。

docs/claude-code-go-rewrite-plan.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,7 @@ test/parity/ # golden tests against TS/official behavior
478478
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 Hangul L/V/T jamo 连接规则,decomposed `한` 这类音节在完整输入以及跨 `Feed()` 边界切在 leading/vowel jamo 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
479479
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 CRLF line-break cluster,完整输入和跨 `Feed()` 边界切在 `\r`/`\n` 中间时都会保持单个零宽 line-break grapheme,wrap/trim/render 宽度路径也复用同一 line-break 判断;完整 Unicode UAX #29 分段仍未宣称完成。
480480
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 Unicode mark category 和 Prepend 规则,nonspacing/enclosing/spacing mark 会归入前一 cluster,`का` 这类 spacing mark cluster 不再被拆宽,Arabic prepend mark 加 base text 会保持同一 grapheme,单独 prepend mark flush 时按零宽处理;完整 Unicode UAX #29 分段仍未宣称完成。
481+
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐常见 Indic virama conjunct,`क्ष` 这类 consonant + virama + consonant cluster 在完整输入和跨 `Feed()` 边界切在 virama 后时都会保持单个窄 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
481482
- 本轮补充:prompt history 写入按官方 `history.ts` 跳过 image pasted content,避免把 image base64/filename/mediaType 存入 `history.jsonl`,读取路径仍兼容旧 image metadata。
482483
- 本轮补充:paste-cache 增加按 cutoff mtime 清理旧 `.txt` paste 文件的 best-effort 入口,忽略缺失目录、非 `.txt` 文件和单文件错误,和官方 `cleanupOldPastes` 语义对齐。
483484
- 本轮补充:Buffered prompt history writer 支持撤销最近 pending entry,覆盖官方 `removeLastFromHistory` 在异步 flush 前直接从 pending buffer 移除的 fast path。

docs/first-second-parity-audit.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,7 @@ M7 progress now includes:
432432
- `internal/tui`: terminal grapheme segmentation now applies Hangul L/V/T jamo joining, keeping decomposed syllables such as `한` as one wide grapheme across full input and streaming chunk boundaries.
433433
- `internal/tui`: terminal grapheme segmentation now keeps CRLF as a single zero-width line-break grapheme across full input and streaming chunks split between `\r` and `\n`, and wrap/trim/render width paths share the same line-break check.
434434
- `internal/tui`: terminal grapheme segmentation now uses Unicode mark categories for nonspacing/enclosing/spacing marks and handles Prepend characters, keeping clusters such as Devanagari `का` and Arabic prepend-mark plus base text together while treating a lone prepend mark as zero width.
435+
- `internal/tui`: terminal grapheme segmentation now keeps common Indic virama conjuncts such as Devanagari `क्ष` as one narrow grapheme across full input and streaming chunks split after the virama.
435436
- `internal/tui`: image hint parsing now accepts OSC ST terminators and base64 `name=` filenames while preserving prompt pasted-content metadata.
436437
- `internal/session`: prompt history writing now skips image pasted-content records like official `history.ts`, while the reader still accepts older image metadata entries.
437438
- `internal/session`: paste-cache now has a best-effort cutoff-mtime cleanup helper for old `.txt` paste files, matching official `cleanupOldPastes` behavior.

internal/tui/terminal_parser.go

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func terminalGraphemeMayContinueAtChunkBoundary(value string) bool {
274274
return true
275275
}
276276
last, _ := utf8.DecodeLastRuneInString(value)
277-
if last == 0x200d || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeMayContinueHangul(value) {
277+
if last == 0x200d || isTerminalVirama(last) || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeMayContinueHangul(value) {
278278
return true
279279
}
280280
regionalCount := 0
@@ -407,6 +407,7 @@ func nextTerminalGrapheme(text string) (string, int) {
407407
}
408408
prependPrefix := isTerminalPrepend(first)
409409
previousWasZWJ := false
410+
previousWasVirama := false
410411
regionalCount := 0
411412
if isTerminalRegionalIndicator(first) {
412413
regionalCount = 1
@@ -427,21 +428,30 @@ func nextTerminalGrapheme(text string) (string, int) {
427428
}
428429
}
429430
previousWasZWJ = false
431+
previousWasVirama = false
430432
continue
431433
}
432434
if isTerminalCombiningMark(r) || isTerminalSpacingMark(r) || isTerminalVariationSelector(r) || isTerminalEmojiModifier(r) || isTerminalEmojiTag(r) {
433435
end += nextSize
434436
previousWasZWJ = false
437+
previousWasVirama = isTerminalVirama(r)
435438
continue
436439
}
437440
if r == 0x200d {
438441
end += nextSize
439442
previousWasZWJ = true
443+
previousWasVirama = false
440444
continue
441445
}
442446
if previousWasZWJ {
443447
end += nextSize
444448
previousWasZWJ = false
449+
previousWasVirama = false
450+
continue
451+
}
452+
if previousWasVirama && unicode.IsLetter(r) {
453+
end += nextSize
454+
previousWasVirama = false
445455
continue
446456
}
447457
if regionalCount == 1 && isTerminalRegionalIndicator(r) {
@@ -453,6 +463,7 @@ func nextTerminalGrapheme(text string) (string, int) {
453463
end += nextSize
454464
hangulClass = terminalHangulJoinedClass(hangulClass, nextHangulClass)
455465
previousWasZWJ = false
466+
previousWasVirama = false
456467
continue
457468
}
458469
break
@@ -646,6 +657,21 @@ func isTerminalSpacingMark(r rune) bool {
646657
return unicode.Is(unicode.Mc, r)
647658
}
648659

660+
func isTerminalVirama(r rune) bool {
661+
switch r {
662+
case 0x094d, 0x09cd, 0x0a4d, 0x0acd, 0x0b4d, 0x0bcd, 0x0c4d, 0x0ccd, 0x0d4d, 0x0dca,
663+
0x0e3a, 0x0eba, 0x0f84, 0x1039, 0x103a, 0x1714, 0x1734, 0x17d2, 0x1a60, 0x1bab,
664+
0x1bf2, 0x1bf3, 0x2d7f, 0xa806, 0xa82c, 0xa8c4, 0xa953, 0xa9c0, 0xaaf6, 0xabed,
665+
0x10a3f, 0x11046, 0x11070, 0x1107f, 0x110b9, 0x11133, 0x11134, 0x111c0, 0x11235, 0x112e9,
666+
0x112ea, 0x1134d, 0x11442, 0x114c2, 0x115bf, 0x1163f, 0x116b6, 0x1172b, 0x11839, 0x1193d,
667+
0x1193e, 0x119e0, 0x11a34, 0x11a47, 0x11a99, 0x11c3f, 0x11d44, 0x11d45, 0x11d97, 0x11f41,
668+
0x11f42:
669+
return true
670+
default:
671+
return false
672+
}
673+
}
674+
649675
func isTerminalPrepend(r rune) bool {
650676
return (r >= 0x0600 && r <= 0x0605) ||
651677
r == 0x06dd ||

internal/tui/terminal_parser_test.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ func TestTerminalParserDispatchesStringControlActions(t *testing.T) {
239239

240240
func TestTerminalParserSegmentsCommonGraphemeClusters(t *testing.T) {
241241
parser := NewTerminalParser()
242-
actions := parser.Feed("e\u0301 \u2764\ufe0f 1\ufe0f\u20e3 2\u20e3 \u1112\u1161\u11ab \U0001f44b\U0001f3fd \U0001f469\u200d\U0001f4bb \U0001f1fa\U0001f1f8 \U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f")
242+
actions := parser.Feed("e\u0301 \u2764\ufe0f 1\ufe0f\u20e3 2\u20e3 \u1112\u1161\u11ab \u0915\u094d\u0937 \U0001f44b\U0001f3fd \U0001f469\u200d\U0001f4bb \U0001f1fa\U0001f1f8 \U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f")
243243
if len(actions) != 1 || actions[0].Type != TerminalActionText {
244244
t.Fatalf("actions = %#v", actions)
245245
}
@@ -254,6 +254,8 @@ func TestTerminalParserSegmentsCommonGraphemeClusters(t *testing.T) {
254254
{Value: " ", Width: 1},
255255
{Value: "\u1112\u1161\u11ab", Width: 2},
256256
{Value: " ", Width: 1},
257+
{Value: "\u0915\u094d\u0937", Width: 1},
258+
{Value: " ", Width: 1},
257259
{Value: "\U0001f44b\U0001f3fd", Width: 2},
258260
{Value: " ", Width: 1},
259261
{Value: "\U0001f469\u200d\U0001f4bb", Width: 2},
@@ -378,6 +380,21 @@ func TestTerminalParserKeepsChunkedGraphemeClustersTogether(t *testing.T) {
378380
t.Fatalf("hangul trailing grapheme = %#v", got)
379381
}
380382

383+
parser = NewTerminalParser()
384+
if actions := parser.Feed("\u0915\u094d"); len(actions) != 0 {
385+
t.Fatalf("partial virama actions = %#v", actions)
386+
}
387+
actions = parser.Feed("\u0937!")
388+
if len(actions) != 1 || len(actions[0].Graphemes) != 2 {
389+
t.Fatalf("virama actions = %#v", actions)
390+
}
391+
if got := actions[0].Graphemes[0]; got.Value != "\u0915\u094d\u0937" || got.Width != 1 {
392+
t.Fatalf("virama grapheme = %#v", got)
393+
}
394+
if width := TerminalActionsVisibleWidth(actions); width != 2 {
395+
t.Fatalf("virama width = %d", width)
396+
}
397+
381398
parser = NewTerminalParser()
382399
actions = parser.Feed("\U0001f1fa" + CSISequence(31, "m") + "red")
383400
if len(actions) != 2 {

0 commit comments

Comments
 (0)