Tokenize modified SS3 cursor sequences

Author: SqlRush

docs/cc-100-roadmap.md

@@ -996,6 +996,8 @@ M7 补充：prompt history `LogEntry` 读取现在接受 `sessionID`/`session`/`
 
 本轮补充：terminal tokenizer 增加官方 streaming escape boundary 状态机，支持跨 chunk buffer/flush/reset、CSI/SS3/OSC/DCS/APC 序列边界、OSC BEL/ST terminator、ESC intermediate charset 序列、invalid CSI text fallback 和 opt-in X10 mouse payload 消费；完整 text grapheme action parser 仍继续推进。
 
+本轮补充：terminal tokenizer 的 SS3 状态现在会消费参数字节后再等待 final byte，`ESC O 1;5D` 这类 modified SS3 cursor 序列可跨 chunk 作为完整 sequence token 进入 dispatcher。
+
 本轮补充：terminal parser 增加轻量 ANSI action pipeline，串接 tokenizer、CSI/OSC/ESC dispatcher 和 SGR style state，输出 text/bell/cursor/erase/scroll/mode/title/link/tabStatus/reset/unknown action，文本宽度覆盖 ASCII、emoji 和 East Asian wide；完整 grapheme cluster segmentation 和 renderer style parity 仍继续推进。
 
 本轮补充：terminal parser 跟踪 OSC 8 hyperlink start/end 状态，暴露当前 `inLink` 和 `linkUrl`，reset 时清空链接状态，贴近官方 parser 的 link range 状态语义。

docs/claude-code-go-rewrite-plan.md

@@ -565,6 +565,7 @@ test/parity/                 # golden tests against TS/official behavior
 - 本轮补充：terminal SGR helper 增加官方 `TextStyle` 状态解析基础，覆盖 reset、bold/dim/italic/underline/blink/inverse/hidden/strikethrough/overline、普通/亮色命名色、256 色、RGB 色、underline color、分号和冒号参数格式；完整 ANSI parser/render style 应用仍继续推进。
 - 本轮补充：terminal sequence dispatcher 增加官方 `identifySequence` 等价分流，按 CSI/OSC/ESC/SS3/unknown 识别并委派现有 parser，SS3 按官方 output parser 作为 unknown action；streaming tokenizer 和 text grapheme action 仍继续推进。
 - 本轮补充：terminal tokenizer 增加官方 streaming escape boundary 状态机，支持跨 chunk buffer/flush/reset、CSI/SS3/OSC/DCS/APC 序列边界、OSC BEL/ST terminator、ESC intermediate charset 序列、invalid CSI text fallback 和 opt-in X10 mouse payload 消费；完整 text grapheme action parser 仍继续推进。
+- 本轮补充：terminal tokenizer 的 SS3 状态现在会消费参数字节后再等待 final byte，`ESC O 1;5D` 这类 modified SS3 cursor 序列可跨 chunk 作为完整 sequence token 进入 dispatcher。
 - 本轮补充：terminal parser 增加轻量 ANSI action pipeline，串接 tokenizer、CSI/OSC/ESC dispatcher 和 SGR style state，输出 text/bell/cursor/erase/scroll/mode/title/link/tabStatus/reset/unknown action，文本宽度覆盖 ASCII、emoji 和 East Asian wide；完整 grapheme cluster segmentation 与 renderer style 应用仍继续推进。
 - 本轮补充：terminal parser 跟踪 OSC 8 hyperlink start/end 状态，暴露当前 `inLink` 和 `linkUrl`，reset 时清空链接状态，贴近官方 parser 的 link range 状态语义。
 - 本轮补充：terminal parser 的 text grapheme 基础分段补齐 combining mark、variation selector、emoji modifier、ZWJ emoji 序列和 regional indicator flag pair；宽度计算现在让 base+combining-mark cluster 保持 base glyph 宽度，emoji presentation/ZWJ/flag 仍按宽 grapheme 处理，完整 Unicode UAX #29 分段仍未宣称完成。

docs/first-second-parity-audit.md

@@ -426,6 +426,7 @@ M7 progress now includes:
 - `internal/tui`: terminal CSI parsing now emits reset actions for DECSTR soft reset `CSI !p`, and the terminal parser clears SGR/link state through the existing reset path.
 - `internal/tui`: terminal sequence dispatch now parses SS3 application cursor sequences such as `ESC OA`/`OB`/`OC`/`OD` as structured cursor move actions.
 - `internal/tui`: terminal sequence dispatch now also parses modified SS3 application cursor sequences such as `ESC O 1;2A`, `ESC O 1;5B`, and `ESC O 1;16D` as structured cursor move actions instead of unknown SS3 sequences.
+- `internal/tui`: terminal tokenizer now keeps SS3 parameter bytes buffered until a final byte, so modified SS3 cursor sequences such as `ESC O 1;5D` can cross chunk boundaries and still reach the dispatcher as one sequence token.
 - `internal/tui`: terminal grapheme width now keeps base+combining-mark clusters at the base glyph width while preserving wide treatment for emoji presentation, ZWJ, regional-indicator, and tag sequences.
 - `internal/tui`: terminal grapheme segmentation now treats emoji keycap sequences such as `1️⃣` and `2⃣` as single wide graphemes, including streaming chunks split after the keycap base or variation selector.
 - `internal/tui`: terminal grapheme segmentation now applies Hangul L/V/T jamo joining, keeping decomposed syllables such as `한` as one wide grapheme across full input and streaming chunk boundaries.

internal/tui/terminal_tokenizer.go

@@ -198,7 +198,9 @@ func tokenizeTerminal(input string, initialState TerminalTokenizerState, initial
 				textStart = seqStart
 			}
 		case terminalTokenizerSS3:
-			if code >= 0x40 && code <= 0x7e {
+			if IsCSIParam(code) {
+				i++
+			} else if IsCSIFinal(code) {
 				i++
 				emitSequence(data[seqStart:i])
 			} else {

internal/tui/terminal_tokenizer_test.go

@@ -81,10 +81,11 @@ func TestTerminalTokenizerHandlesOSCAndStringControls(t *testing.T) {
 
 func TestTerminalTokenizerHandlesESCIntermediateSS3AndInvalidSequences(t *testing.T) {
 	tokenizer := NewTerminalTokenizer(TerminalTokenizerOptions{})
-	tokens := tokenizer.Feed("\x1b(B\x1bOA\x1b[31x")
+	tokens := tokenizer.Feed("\x1b(B\x1bOA\x1bO1;2A\x1b[31x")
 	want := []TerminalToken{
 		{Type: TerminalTokenSequence, Value: "\x1b(B"},
 		{Type: TerminalTokenSequence, Value: "\x1bOA"},
+		{Type: TerminalTokenSequence, Value: "\x1bO1;2A"},
 		{Type: TerminalTokenSequence, Value: "\x1b[31x"},
 	}
 	if !reflect.DeepEqual(tokens, want) {
@@ -99,6 +100,16 @@ func TestTerminalTokenizerHandlesESCIntermediateSS3AndInvalidSequences(t *testin
 	if !reflect.DeepEqual(tokens, want) {
 		t.Fatalf("invalid csi tokens = %#v", tokens)
 	}
+
+	tokens = tokenizer.Feed("\x1bO1;")
+	if len(tokens) != 0 || tokenizer.Buffer() != "\x1bO1;" {
+		t.Fatalf("partial modified ss3 tokens=%#v buffer=%q", tokens, tokenizer.Buffer())
+	}
+	tokens = tokenizer.Feed("5D")
+	want = []TerminalToken{{Type: TerminalTokenSequence, Value: "\x1bO1;5D"}}
+	if !reflect.DeepEqual(tokens, want) || tokenizer.Buffer() != "" {
+		t.Fatalf("completed modified ss3 tokens=%#v buffer=%q", tokens, tokenizer.Buffer())
+	}
 }
 
 func TestTerminalTokenizerX10MouseOption(t *testing.T) {