From 426b65a85dcd4e1c1fba65465cd47dac7259dd11 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 6 May 2026 07:29:59 +0200 Subject: [PATCH] fix(parser)!: track byte offsets instead of char counts in source locations SourcePosition.offset now tracks byte offsets rather than UTF-8 character counts. Previously, any multi-byte character caused the reported index to drift by (len_utf8() - 1), breaking downstream consumers that use the index to slice the original source (diagnostics, syntax highlighting, LSP, etc.). SourcePosition.index is removed to force downstreams to be aware of the change. Closes #1127 --- brush-core/src/callstack.rs | 4 +- brush-interactive/src/highlighting.rs | 8 +- brush-parser/src/ast.rs | 80 +++++++++++++++++-- brush-parser/src/parser/peg.rs | 6 +- brush-parser/src/parser/tests/mod.rs | 2 +- ...tests__complex__parse_nested_if_while.snap | 4 +- ...tests__compound_commands__parse_until.snap | 4 +- ...tests__compound_commands__parse_while.snap | 4 +- ...pound_commands__parse_while_multiline.snap | 4 +- ..._parser__parser__tests__parse_program.snap | 32 ++++---- ...pelines__parse_negated_timed_pipeline.snap | 4 +- ...ests__pipelines__parse_timed_pipeline.snap | 4 +- ...pipelines__parse_timed_pipeline_posix.snap | 4 +- ...tests__tokenize_arithmetic_expression.snap | 8 +- ...ize_arithmetic_expression_with_parens.snap | 4 +- ...nize_arithmetic_expression_with_space.snap | 4 +- ...tests__tokenize_backquote_with_escape.snap | 8 +- ...tokenize_braced_parameter_expansion-2.snap | 4 +- ...__tokenize_braced_parameter_expansion.snap | 4 +- ...ced_parameter_expansion_with_escaping.snap | 4 +- ..._tests__tokenize_command_substitution.snap | 8 +- ...mmand_substitution_containing_extglob.snap | 8 +- ...ze_command_substitution_with_subshell.snap | 4 +- ...r__tokenizer__tests__tokenize_comment.snap | 8 +- ...nizer__tests__tokenize_comment_at_eof.snap | 4 +- ...lex_here_docs_in_command_substitution.snap | 8 +- ...kenizer__tests__tokenize_double_quote.snap | 4 +- ...e_double_quoted_arithmetic_expression.snap | 4 +- ...ze_double_quoted_command_substitution.snap | 4 +- ...nizer__tests__tokenize_empty_here_doc.snap | 24 +++--- ...r__tests__tokenize_escaped_whitespace.snap | 8 +- ...tokenizer__tests__tokenize_here_doc-2.snap | 24 +++--- ...tokenizer__tests__tokenize_here_doc-3.snap | 28 +++---- ...tokenizer__tests__tokenize_here_doc-4.snap | 24 +++--- ...__tokenizer__tests__tokenize_here_doc.snap | 36 ++++----- ...nize_here_doc_in_command_substitution.snap | 8 +- ...in_double_quoted_command_substitution.snap | 8 +- ...uoted_command_substitution_with_space.snap | 8 +- ...__tokenize_here_doc_with_other_tokens.snap | 36 ++++----- ...s__tokenize_here_doc_with_tab_removal.snap | 24 +++--- ...er__tests__tokenize_line_continuation.snap | 4 +- ...r__tests__tokenize_multiple_here_docs.snap | 52 ++++++------ ..._tokenizer__tests__tokenize_operators.snap | 12 +-- ...zer__tests__tokenize_simple_backquote.snap | 8 +- ...kenizer__tests__tokenize_single_quote.snap | 4 +- ..._tests__tokenize_special_parameters-2.snap | 4 +- ..._tests__tokenize_special_parameters-3.snap | 4 +- ..._tests__tokenize_special_parameters-4.snap | 4 +- ..._tests__tokenize_special_parameters-5.snap | 4 +- ...r__tests__tokenize_special_parameters.snap | 4 +- ...kenize_unbraced_parameter_expansion-2.snap | 4 +- ...tokenize_unbraced_parameter_expansion.snap | 4 +- ...tokenizer__tests__tokenize_whitespace.snap | 12 +-- brush-parser/src/source.rs | 18 ++--- brush-parser/src/tokenizer.rs | 6 +- 55 files changed, 344 insertions(+), 276 deletions(-) diff --git a/brush-core/src/callstack.rs b/brush-core/src/callstack.rs index 33b9bf7c5..02e76bff5 100644 --- a/brush-core/src/callstack.rs +++ b/brush-core/src/callstack.rs @@ -191,7 +191,7 @@ impl Frame { let mut new_start = if let Some(existing_start) = &self.source_info.start { if let Some(current) = pos { Some(Arc::new(crate::SourcePosition { - index: existing_start.index + current.index, + offset: existing_start.offset + current.offset, line: existing_start.line + (current.line - 1), column: if current.line <= 1 { existing_start.column + (current.column - 1) @@ -214,7 +214,7 @@ impl Frame { Some(Arc::new(pos)) } else { Some(Arc::new(crate::SourcePosition { - index: 0, + offset: 0, line: self.current_line_offset + 1, column: 1, })) diff --git a/brush-interactive/src/highlighting.rs b/brush-interactive/src/highlighting.rs index cb23a45c3..bc24c50f3 100644 --- a/brush-interactive/src/highlighting.rs +++ b/brush-interactive/src/highlighting.rs @@ -188,13 +188,13 @@ impl<'a, SE: brush_core::ShellExtensions> Highlighter<'a, SE> { for token in tokens { match token { brush_parser::Token::Operator(_op, token_location) => { - let start = global_offset + byte_offset(token_location.start.index); - let end = global_offset + byte_offset(token_location.end.index); + let start = global_offset + byte_offset(token_location.start.offset); + let end = global_offset + byte_offset(token_location.end.offset); self.append_span(HighlightKind::Operator, start..end); } brush_parser::Token::Word(w, token_location) => { - let start_byte = byte_offset(token_location.start.index); - let end_byte = byte_offset(token_location.end.index); + let start_byte = byte_offset(token_location.start.offset); + let end_byte = byte_offset(token_location.end.offset); // Parse the raw slice from `line`, not `w.as_str()`: the tokenizer may // drop chars from `w` (e.g. `\` continuations), so offsets into diff --git a/brush-parser/src/ast.rs b/brush-parser/src/ast.rs index c41a261bd..112536451 100644 --- a/brush-parser/src/ast.rs +++ b/brush-parser/src/ast.rs @@ -2368,7 +2368,7 @@ echo there SourcePosition { line: 1, column: 1, - index: 0 + offset: 0 } ); assert_eq!( @@ -2376,7 +2376,7 @@ echo there SourcePosition { line: 2, column: 11, - index: 18 + offset: 18 } ); } @@ -2404,7 +2404,7 @@ my_func SourcePosition { line: 1, column: 1, - index: 0 + offset: 0 } ); assert_eq!( @@ -2412,7 +2412,7 @@ my_func SourcePosition { line: 4, column: 2, - index: 36 + offset: 36 } ); } @@ -2435,7 +2435,7 @@ my_func SourcePosition { line: 1, column: 1, - index: 0 + offset: 0 } ); assert_eq!( @@ -2443,8 +2443,76 @@ my_func SourcePosition { line: 1, column: 28, - index: 27 + offset: 27 } ); } + + #[test] + fn simple_cmd_loc_after_multibyte_comment() { + // Line 1: "# café\n" — 'é' is 2 bytes (0xC3 0xA9), total 8 bytes + // Line 2: "ls\n" — 'l' is at byte offset 8, total 3 bytes + let input = "# café\nls\n"; + + let program = parse(input); + + let Command::Simple(cmd) = &program.complete_commands[0].0[0].0.first.seq[0] else { + panic!("expected simple command"); + }; + + let loc = cmd.location().unwrap(); + + assert_eq!( + *(loc.start), + SourcePosition { + line: 2, + column: 1, + offset: 8 + } + ); + assert_eq!( + *(loc.end), + SourcePosition { + line: 2, + column: 3, + offset: 10 + } + ); + + assert_eq!(input.get(loc.start.offset..loc.end.offset), Some("ls")); + } + + #[test] + fn simple_cmd_loc_after_multiple_multibyte_chars() { + // "# café 日本語\n" = 18 bytes (é=2, 日=3, 本=3, 語=3 + ASCII) + // "ls" starts at byte offset 18 + let input = "# café 日本語\nls\n"; + + let program = parse(input); + + let Command::Simple(cmd) = &program.complete_commands[0].0[0].0.first.seq[0] else { + panic!("expected simple command"); + }; + + let loc = cmd.location().unwrap(); + + assert_eq!( + *(loc.start), + SourcePosition { + line: 2, + column: 1, + offset: 18 + } + ); + assert_eq!( + *(loc.end), + SourcePosition { + line: 2, + column: 3, + offset: 20 + } + ); + + assert_eq!(input.get(loc.start.offset..loc.end.offset), Some("ls")); + } } diff --git a/brush-parser/src/parser/peg.rs b/brush-parser/src/parser/peg.rs index 75ad8d569..766bde8bd 100644 --- a/brush-parser/src/parser/peg.rs +++ b/brush-parser/src/parser/peg.rs @@ -753,7 +753,7 @@ fn add_pipe_extension_redirection(c: &mut ast::Command) -> Result<(), &'static s #[inline] fn locations_are_contiguous(loc_left: &crate::SourceSpan, loc_right: &crate::SourceSpan) -> bool { - loc_left.end.index == loc_right.start.index + loc_left.end.offset == loc_right.start.offset } impl peg::Parse for Tokens<'_> { @@ -811,13 +811,13 @@ impl<'a> peg::ParseSlice<'a> for Tokens<'a> { let loc = token.location(); if let Some(prev_end) = prev_end_index { - if loc.start.index > prev_end { + if loc.start.offset > prev_end { result.push(' '); } } result.push_str(token.to_str()); - prev_end_index = Some(loc.end.index); + prev_end_index = Some(loc.end.offset); } result diff --git a/brush-parser/src/parser/tests/mod.rs b/brush-parser/src/parser/tests/mod.rs index 37ee4184f..dc92649db 100644 --- a/brush-parser/src/parser/tests/mod.rs +++ b/brush-parser/src/parser/tests/mod.rs @@ -96,7 +96,7 @@ fn is_source_span(value: &Value) -> bool { fn normalize_source_span(value: &mut Value) { if let Value::Object(map) = value { let placeholder_pos = serde_json::json!({ - "index": 0, + "offset": 0, "line": 0, "column": 0 }); diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__complex__parse_nested_if_while.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__complex__parse_nested_if_while.snap index 3207377b5..b974b7f76 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__complex__parse_nested_if_while.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__complex__parse_nested_if_while.snap @@ -72,12 +72,12 @@ ParseResult( loc: "[location]", ), SourceSpan( start: SourcePosition( - index: 18, + offset: 18, line: 2, column: 5, ), end: SourcePosition( - index: 67, + offset: 67, line: 4, column: 9, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_until.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_until.snap index 5f3e05285..77218587d 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_until.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_until.snap @@ -47,12 +47,12 @@ ParseResult( loc: "[location]", ), SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 31, + offset: 31, line: 1, column: 32, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while.snap index e3e7b9675..ee7d7e169 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while.snap @@ -47,12 +47,12 @@ ParseResult( loc: "[location]", ), SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 30, + offset: 30, line: 1, column: 31, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while_multiline.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while_multiline.snap index df570c7d5..e5f78c439 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while_multiline.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while_multiline.snap @@ -59,12 +59,12 @@ ParseResult( loc: "[location]", ), SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 42, + offset: 42, line: 5, column: 5, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__parse_program.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__parse_program.snap index 27a8f7947..55577cb30 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__parse_program.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__parse_program.snap @@ -17,12 +17,12 @@ ParseResult( value: "A", loc: Some(SourceSpan( start: SourcePosition( - index: 32, + offset: 32, line: 5, column: 10, ), end: SourcePosition( - index: 33, + offset: 33, line: 5, column: 11, ), @@ -32,12 +32,12 @@ ParseResult( value: "B", loc: Some(SourceSpan( start: SourcePosition( - index: 34, + offset: 34, line: 5, column: 12, ), end: SourcePosition( - index: 35, + offset: 35, line: 5, column: 13, ), @@ -47,12 +47,12 @@ ParseResult( value: "C", loc: Some(SourceSpan( start: SourcePosition( - index: 36, + offset: 36, line: 5, column: 14, ), end: SourcePosition( - index: 37, + offset: 37, line: 5, column: 15, ), @@ -69,12 +69,12 @@ ParseResult( value: "echo", loc: Some(SourceSpan( start: SourcePosition( - index: 60, + offset: 60, line: 8, column: 5, ), end: SourcePosition( - index: 64, + offset: 64, line: 8, column: 9, ), @@ -85,12 +85,12 @@ ParseResult( value: "\"${f@L}\"", loc: Some(SourceSpan( start: SourcePosition( - index: 65, + offset: 65, line: 8, column: 10, ), end: SourcePosition( - index: 73, + offset: 73, line: 8, column: 18, ), @@ -100,12 +100,12 @@ ParseResult( value: "2", loc: Some(SourceSpan( start: SourcePosition( - index: 76, + offset: 76, line: 8, column: 21, ), end: SourcePosition( - index: 77, + offset: 77, line: 8, column: 22, ), @@ -119,12 +119,12 @@ ParseResult( ]), loc: SourceSpan( start: SourcePosition( - index: 39, + offset: 39, line: 5, column: 17, ), end: SourcePosition( - index: 86, + offset: 86, line: 10, column: 8, ), @@ -132,12 +132,12 @@ ParseResult( ), loc: SourceSpan( start: SourcePosition( - index: 23, + offset: 23, line: 5, column: 1, ), end: SourcePosition( - index: 86, + offset: 86, line: 10, column: 8, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_negated_timed_pipeline.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_negated_timed_pipeline.snap index bb592be77..34f43e3c5 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_negated_timed_pipeline.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_negated_timed_pipeline.snap @@ -11,12 +11,12 @@ ParseResult( first: Pipeline( timed: Some(Timed(SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline.snap index b8d248d26..f73ccdac0 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline.snap @@ -11,12 +11,12 @@ ParseResult( first: Pipeline( timed: Some(Timed(SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), diff --git a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline_posix.snap b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline_posix.snap index a4c0c9105..b895a6030 100644 --- a/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline_posix.snap +++ b/brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__pipelines__parse_timed_pipeline_posix.snap @@ -11,12 +11,12 @@ ParseResult( first: Pipeline( timed: Some(TimedWithPosixOutput(SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 7, + offset: 7, line: 1, column: 8, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap index 0d8235198..cf33ea84f 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("a$((1+2))b", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 10, + offset: 10, line: 1, column: 11, ), )), Word("c", SourceSpan( start: SourcePosition( - index: 11, + offset: 11, line: 1, column: 12, ), end: SourcePosition( - index: 12, + offset: 12, line: 1, column: 13, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap index e19b70bf0..52616a906 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$(( (0) ))", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 10, + offset: 10, line: 1, column: 11, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap index 686294cb9..af5fe8fb7 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$(( 1 ))", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 8, + offset: 8, line: 1, column: 9, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap index 64e861c8a..107d10c27 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("echo", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), )), Word("`echo\\`hi`", SourceSpan( start: SourcePosition( - index: 5, + offset: 5, line: 1, column: 6, ), end: SourcePosition( - index: 15, + offset: 15, line: 1, column: 16, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap index 4d5765eae..6d6019905 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("a${x}b", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 6, + offset: 6, line: 1, column: 7, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap index d351246d4..0a11fb164 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("${x}", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap index b02f5a710..00e5a41ae 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("a${x\\}}b", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 8, + offset: 8, line: 1, column: 9, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap index cccd129c2..2f7465260 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("a$(echo hi)b", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 12, + offset: 12, line: 1, column: 13, ), )), Word("c", SourceSpan( start: SourcePosition( - index: 13, + offset: 13, line: 1, column: 14, ), end: SourcePosition( - index: 14, + offset: 14, line: 1, column: 15, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap index df02257c0..211ed8e1e 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("echo", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), )), Word("$(echo !(x))", SourceSpan( start: SourcePosition( - index: 5, + offset: 5, line: 1, column: 6, ), end: SourcePosition( - index: 17, + offset: 17, line: 1, column: 18, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap index 26f495ab7..1f9c8feb1 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$( (:) )", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 8, + offset: 8, line: 1, column: 9, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap index a2ade98d1..b621d07ae 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("a", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 1, + offset: 1, line: 1, column: 2, ), )), Operator("\n", SourceSpan( start: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), end: SourcePosition( - index: 11, + offset: 11, line: 2, column: 1, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap index 82f249d2f..d540326b0 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("a", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 1, + offset: 1, line: 1, column: 2, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap index 5507f7f51..3d8d7bf4e 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("echo", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), )), Word("$(cat <>", SourceSpan( start: SourcePosition( - index: 1, + offset: 1, line: 1, column: 2, ), end: SourcePosition( - index: 3, + offset: 3, line: 1, column: 4, ), )), Word("b", SourceSpan( start: SourcePosition( - index: 3, + offset: 3, line: 1, column: 4, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap index f29c9ef64..4cb43e871 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap @@ -7,24 +7,24 @@ TokenizerResult( result: [ Word("echo", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), )), Word("`echo hi`", SourceSpan( start: SourcePosition( - index: 5, + offset: 5, line: 1, column: 6, ), end: SourcePosition( - index: 14, + offset: 14, line: 1, column: 15, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap index e9c397ceb..5dcbba1c5 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("x\'a b\'y", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 7, + offset: 7, line: 1, column: 8, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap index 0fc4868e5..1817c5418 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$@", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap index d2b636b01..1d00a7089 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$!", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap index 01d525cfb..a6aa8bc29 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$?", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap index f2d7e2b3d..a8ce02890 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$*", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap index 784961762..4e7b7242c 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$$", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap index 41e88816a..b7a202fa0 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("a$x", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 3, + offset: 3, line: 1, column: 4, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap index d70450824..7836b29b0 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap @@ -7,12 +7,12 @@ TokenizerResult( result: [ Word("$x", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap index 5a39c2ecd..f0739627e 100644 --- a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap +++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap @@ -7,36 +7,36 @@ TokenizerResult( result: [ Word("1", SourceSpan( start: SourcePosition( - index: 0, + offset: 0, line: 1, column: 1, ), end: SourcePosition( - index: 1, + offset: 1, line: 1, column: 2, ), )), Word("2", SourceSpan( start: SourcePosition( - index: 2, + offset: 2, line: 1, column: 3, ), end: SourcePosition( - index: 3, + offset: 3, line: 1, column: 4, ), )), Word("3", SourceSpan( start: SourcePosition( - index: 4, + offset: 4, line: 1, column: 5, ), end: SourcePosition( - index: 5, + offset: 5, line: 1, column: 6, ), diff --git a/brush-parser/src/source.rs b/brush-parser/src/source.rs index f6279f43e..dd6bb7957 100644 --- a/brush-parser/src/source.rs +++ b/brush-parser/src/source.rs @@ -8,11 +8,11 @@ use std::{fmt::Display, sync::Arc}; derive(PartialEq, Eq, serde::Serialize, serde::Deserialize) )] pub struct SourcePosition { - /// The 0-based index of the character in the input stream. - pub index: usize, + /// The 0-based byte offset in the input stream. + pub offset: usize, /// The 1-based line number. pub line: usize, - /// The 1-based column number. + /// The 1-based column number (character count within the line). pub column: usize, } @@ -31,7 +31,7 @@ impl SourcePosition { #[must_use] pub const fn offset(&self, offset: &SourcePositionOffset) -> Self { Self { - index: self.index + offset.index, + offset: self.offset + offset.offset, line: self.line + offset.line, column: if offset.line == 0 { self.column + offset.column @@ -46,7 +46,7 @@ impl SourcePosition { impl From<&SourcePosition> for miette::SourceOffset { #[allow(clippy::cast_sign_loss)] fn from(position: &SourcePosition) -> Self { - position.index.into() + position.offset.into() } } @@ -58,8 +58,8 @@ impl From<&SourcePosition> for miette::SourceOffset { derive(PartialEq, Eq, serde::Serialize, serde::Deserialize) )] pub struct SourcePositionOffset { - /// The 0-based character offset. - pub index: usize, + /// The 0-based byte offset. + pub offset: usize, /// The 0-based line offset. pub line: usize, /// The 0-based column offset. @@ -81,9 +81,9 @@ pub struct SourceSpan { } impl SourceSpan { - /// Returns the length of the token in characters. + /// Returns the length of the span in bytes. pub fn length(&self) -> usize { - self.end.index - self.start.index + self.end.offset - self.start.offset } pub(crate) fn within(start: &Self, end: &Self) -> Self { Self { diff --git a/brush-parser/src/tokenizer.rs b/brush-parser/src/tokenizer.rs index ceee9e364..188b96d78 100644 --- a/brush-parser/src/tokenizer.rs +++ b/brush-parser/src/tokenizer.rs @@ -539,7 +539,7 @@ impl<'a, R: ?Sized + std::io::BufRead> Tokenizer<'a, R> { char_reader: reader.chars().peekable(), cross_state: CrossTokenParseState { cursor: SourcePosition { - index: 0, + offset: 0, line: 1, column: 1, }, @@ -570,7 +570,7 @@ impl<'a, R: ?Sized + std::io::BufRead> Tokenizer<'a, R> { } else { self.cross_state.cursor.column += 1; } - self.cross_state.cursor.index += 1; + self.cross_state.cursor.offset += ch.len_utf8(); } Ok(c) @@ -1133,7 +1133,7 @@ impl<'a, R: ?Sized + std::io::BufRead> Tokenizer<'a, R> { } else { // Make sure we don't include this char in the token range. state.start_position.column += 1; - state.start_position.index += 1; + state.start_position.offset += c.len_utf8(); } self.consume_char()?;