Skip to content

Commit a2a403a

Browse files
author
SqlRush
committed
Support terminal Hangul graphemes
1 parent 0e7033a commit a2a403a

5 files changed

Lines changed: 123 additions & 3 deletions

File tree

docs/cc-100-roadmap.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,8 @@ M7 补充:prompt history `LogEntry` 读取现在接受 `sessionID`/`session`/`
902902

903903
本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji keycap sequence,`1️⃣`/`2⃣` 这类 keycap 在完整输入和跨 `Feed()` 边界切在 base 或 variation selector 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
904904

905+
本轮补充:terminal parser 的 text grapheme 分段继续补齐 Hangul L/V/T jamo 连接规则,decomposed `한` 这类音节在完整输入以及跨 `Feed()` 边界切在 leading/vowel jamo 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
906+
905907
本轮补充:terminal CSI parser 现在对 tokenizer flush 出来的非 final-byte incomplete CSI 返回 unknown action,而不是丢弃,贴近官方 `parseCSI` 对 flushed partial sequence 的 fallback 行为。
906908

907909
本轮补充:terminal sequence dispatcher 对 tokenizer flush 出来的 OSC partial sequence 使用 `ParseOSCContent` fallback,允许无 BEL/ST terminator 的 title/link/tab-status content 按官方 parser 语义产出 action。

docs/claude-code-go-rewrite-plan.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ test/parity/ # golden tests against TS/official behavior
409409
- 本轮补充:terminal CSI parser 按 ANSI 默认参数解析 scroll-region (`CSI r`/`CSI ;10r`),缺失 top 默认为 1,缺失 bottom 保持 0 表示 reset/full-height,避免 reset scroll region 被误判为单行区域。
410410
- 本轮补充:terminal CSI parser 把 DECSTR soft reset (`CSI !p`) 归入 reset action,并在 terminal parser 中清理 SGR/link 状态。
411411
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 emoji keycap sequence,`1️⃣`/`2⃣` 这类 keycap 在完整输入和跨 `Feed()` 边界切在 base 或 variation selector 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
412+
- 本轮补充:terminal parser 的 text grapheme 分段继续补齐 Hangul L/V/T jamo 连接规则,decomposed `한` 这类音节在完整输入以及跨 `Feed()` 边界切在 leading/vowel jamo 后时都会保持单个宽 grapheme;完整 Unicode UAX #29 分段仍未宣称完成。
412413
- 本轮补充:prompt history 写入按官方 `history.ts` 跳过 image pasted content,避免把 image base64/filename/mediaType 存入 `history.jsonl`,读取路径仍兼容旧 image metadata。
413414
- 本轮补充:paste-cache 增加按 cutoff mtime 清理旧 `.txt` paste 文件的 best-effort 入口,忽略缺失目录、非 `.txt` 文件和单文件错误,和官方 `cleanupOldPastes` 语义对齐。
414415
- 本轮补充:Buffered prompt history writer 支持撤销最近 pending entry,覆盖官方 `removeLastFromHistory` 在异步 flush 前直接从 pending buffer 移除的 fast path。

docs/first-second-parity-audit.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ M7 progress now includes:
348348
- `internal/tui`: terminal sequence dispatch now parses SS3 application cursor sequences such as `ESC OA`/`OB`/`OC`/`OD` as structured cursor move actions.
349349
- `internal/tui`: terminal grapheme width now keeps base+combining-mark clusters at the base glyph width while preserving wide treatment for emoji presentation, ZWJ, regional-indicator, and tag sequences.
350350
- `internal/tui`: terminal grapheme segmentation now treats emoji keycap sequences such as `1️⃣` and `2⃣` as single wide graphemes, including streaming chunks split after the keycap base or variation selector.
351+
- `internal/tui`: terminal grapheme segmentation now applies Hangul L/V/T jamo joining, keeping decomposed syllables such as `한` as one wide grapheme across full input and streaming chunk boundaries.
351352
- `internal/tui`: image hint parsing now accepts OSC ST terminators and base64 `name=` filenames while preserving prompt pasted-content metadata.
352353
- `internal/session`: prompt history writing now skips image pasted-content records like official `history.ts`, while the reader still accepts older image metadata entries.
353354
- `internal/session`: paste-cache now has a best-effort cutoff-mtime cleanup helper for old `.txt` paste files, matching official `cleanupOldPastes` behavior.

internal/tui/terminal_parser.go

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ func terminalGraphemeMayContinueAtChunkBoundary(value string) bool {
265265
return false
266266
}
267267
last, _ := utf8.DecodeLastRuneInString(value)
268-
if last == 0x200d || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) {
268+
if last == 0x200d || isTerminalEmojiModifier(last) || terminalGraphemeCanStartKeycapSequence(value) || terminalGraphemeMayContinueHangul(value) {
269269
return true
270270
}
271271
regionalCount := 0
@@ -388,6 +388,7 @@ func nextTerminalGrapheme(text string) (string, int) {
388388
if isTerminalRegionalIndicator(first) {
389389
regionalCount = 1
390390
}
391+
hangulClass := terminalHangulClassOf(first)
391392
for end < len(text) {
392393
r, nextSize := utf8.DecodeRuneInString(text[end:])
393394
if r == utf8.RuneError && nextSize == 0 {
@@ -413,6 +414,12 @@ func nextTerminalGrapheme(text string) (string, int) {
413414
regionalCount++
414415
continue
415416
}
417+
if nextHangulClass := terminalHangulClassOf(r); terminalHangulCanJoin(hangulClass, nextHangulClass) {
418+
end += nextSize
419+
hangulClass = terminalHangulJoinedClass(hangulClass, nextHangulClass)
420+
previousWasZWJ = false
421+
continue
422+
}
416423
break
417424
}
418425
return text[:end], end
@@ -425,7 +432,7 @@ func terminalGraphemeStringWidth(grapheme string) int {
425432
for _, r := range grapheme {
426433
if !hasBase {
427434
hasBase = true
428-
if isTerminalEmoji(r) || isTerminalEastAsianWide(r) {
435+
if isTerminalEmoji(r) || isTerminalEastAsianWide(r) || isTerminalHangulWideBase(r) {
429436
baseWidth = 2
430437
}
431438
continue
@@ -468,6 +475,86 @@ func terminalGraphemeCanStartKeycapSequence(value string) bool {
468475
return len(runes) == 2 && isTerminalEmojiKeycapBase(runes[0]) && runes[1] == 0xfe0f
469476
}
470477

478+
func terminalGraphemeMayContinueHangul(value string) bool {
479+
class := terminalHangulNone
480+
for _, r := range value {
481+
if next := terminalHangulClassOf(r); next != terminalHangulNone {
482+
class = next
483+
}
484+
}
485+
return class != terminalHangulNone
486+
}
487+
488+
type terminalHangulClass uint8
489+
490+
const (
491+
terminalHangulNone terminalHangulClass = iota
492+
terminalHangulL
493+
terminalHangulV
494+
terminalHangulT
495+
terminalHangulLV
496+
terminalHangulLVT
497+
)
498+
499+
func terminalHangulClassOf(r rune) terminalHangulClass {
500+
switch {
501+
case (r >= 0x1100 && r <= 0x115f) || (r >= 0xa960 && r <= 0xa97c):
502+
return terminalHangulL
503+
case (r >= 0x1160 && r <= 0x11a7) || (r >= 0xd7b0 && r <= 0xd7c6):
504+
return terminalHangulV
505+
case (r >= 0x11a8 && r <= 0x11ff) || (r >= 0xd7cb && r <= 0xd7fb):
506+
return terminalHangulT
507+
case r >= 0xac00 && r <= 0xd7a3:
508+
if (r-0xac00)%28 == 0 {
509+
return terminalHangulLV
510+
}
511+
return terminalHangulLVT
512+
default:
513+
return terminalHangulNone
514+
}
515+
}
516+
517+
func terminalHangulCanJoin(left, right terminalHangulClass) bool {
518+
switch left {
519+
case terminalHangulL:
520+
return right == terminalHangulL || right == terminalHangulV || right == terminalHangulLV || right == terminalHangulLVT
521+
case terminalHangulV, terminalHangulLV:
522+
return right == terminalHangulV || right == terminalHangulT
523+
case terminalHangulT, terminalHangulLVT:
524+
return right == terminalHangulT
525+
default:
526+
return false
527+
}
528+
}
529+
530+
func terminalHangulJoinedClass(left, right terminalHangulClass) terminalHangulClass {
531+
switch {
532+
case left == terminalHangulL && right == terminalHangulL:
533+
return terminalHangulL
534+
case left == terminalHangulL && right == terminalHangulV:
535+
return terminalHangulV
536+
case left == terminalHangulL && right == terminalHangulLV:
537+
return terminalHangulLV
538+
case left == terminalHangulL && right == terminalHangulLVT:
539+
return terminalHangulLVT
540+
case (left == terminalHangulV || left == terminalHangulLV) && right == terminalHangulV:
541+
return terminalHangulV
542+
case (left == terminalHangulV || left == terminalHangulLV) && right == terminalHangulT:
543+
return terminalHangulT
544+
case (left == terminalHangulT || left == terminalHangulLVT) && right == terminalHangulT:
545+
return terminalHangulT
546+
case right != terminalHangulNone:
547+
return right
548+
default:
549+
return left
550+
}
551+
}
552+
553+
func isTerminalHangulWideBase(r rune) bool {
554+
class := terminalHangulClassOf(r)
555+
return class == terminalHangulL || class == terminalHangulLV || class == terminalHangulLVT
556+
}
557+
471558
func isTerminalEmoji(r rune) bool {
472559
return (r >= 0x2600 && r <= 0x26ff) ||
473560
(r >= 0x2700 && r <= 0x27bf) ||

internal/tui/terminal_parser_test.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func TestTerminalParserDispatchesStringControlActions(t *testing.T) {
131131

132132
func TestTerminalParserSegmentsCommonGraphemeClusters(t *testing.T) {
133133
parser := NewTerminalParser()
134-
actions := parser.Feed("e\u0301 \u2764\ufe0f 1\ufe0f\u20e3 2\u20e3 \U0001f44b\U0001f3fd \U0001f469\u200d\U0001f4bb \U0001f1fa\U0001f1f8 \U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f")
134+
actions := parser.Feed("e\u0301 \u2764\ufe0f 1\ufe0f\u20e3 2\u20e3 \u1112\u1161\u11ab \U0001f44b\U0001f3fd \U0001f469\u200d\U0001f4bb \U0001f1fa\U0001f1f8 \U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f")
135135
if len(actions) != 1 || actions[0].Type != TerminalActionText {
136136
t.Fatalf("actions = %#v", actions)
137137
}
@@ -144,6 +144,8 @@ func TestTerminalParserSegmentsCommonGraphemeClusters(t *testing.T) {
144144
{Value: " ", Width: 1},
145145
{Value: "2\u20e3", Width: 2},
146146
{Value: " ", Width: 1},
147+
{Value: "\u1112\u1161\u11ab", Width: 2},
148+
{Value: " ", Width: 1},
147149
{Value: "\U0001f44b\U0001f3fd", Width: 2},
148150
{Value: " ", Width: 1},
149151
{Value: "\U0001f469\u200d\U0001f4bb", Width: 2},
@@ -241,6 +243,33 @@ func TestTerminalParserKeepsChunkedGraphemeClustersTogether(t *testing.T) {
241243
t.Fatalf("flush keycap base actions = %#v", actions)
242244
}
243245

246+
parser = NewTerminalParser()
247+
if actions := parser.Feed("\u1112"); len(actions) != 0 {
248+
t.Fatalf("partial hangul leading actions = %#v", actions)
249+
}
250+
actions = parser.Feed("\u1161\u11ab!")
251+
if len(actions) != 1 || len(actions[0].Graphemes) != 2 {
252+
t.Fatalf("hangul actions = %#v", actions)
253+
}
254+
if got := actions[0].Graphemes[0]; got.Value != "\u1112\u1161\u11ab" || got.Width != 2 {
255+
t.Fatalf("hangul grapheme = %#v", got)
256+
}
257+
if width := TerminalActionsVisibleWidth(actions); width != 3 {
258+
t.Fatalf("hangul width = %d", width)
259+
}
260+
261+
parser = NewTerminalParser()
262+
if actions := parser.Feed("\u1112\u1161"); len(actions) != 0 {
263+
t.Fatalf("partial hangul vowel actions = %#v", actions)
264+
}
265+
actions = parser.Feed("\u11ab!")
266+
if len(actions) != 1 || len(actions[0].Graphemes) != 2 {
267+
t.Fatalf("hangul trailing actions = %#v", actions)
268+
}
269+
if got := actions[0].Graphemes[0]; got.Value != "\u1112\u1161\u11ab" || got.Width != 2 {
270+
t.Fatalf("hangul trailing grapheme = %#v", got)
271+
}
272+
244273
parser = NewTerminalParser()
245274
actions = parser.Feed("\U0001f1fa" + CSISequence(31, "m") + "red")
246275
if len(actions) != 2 {

0 commit comments

Comments
 (0)