Skip to content

fix(parser)!: track byte offsets instead of char counts in source locations #1144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lu-zero wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(parser)!: track byte offsets instead of char counts in source locations #1144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions brush-core/src/callstack.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ impl Frame {
let mut new_start = if let Some(existing_start) = &self.source_info.start {
if let Some(current) = pos {
Some(Arc::new(crate::SourcePosition {
index: existing_start.index + current.index,
offset: existing_start.offset + current.offset,
line: existing_start.line + (current.line - 1),
column: if current.line <= 1 {
existing_start.column + (current.column - 1)
Expand All @@ -214,7 +214,7 @@ impl Frame {
Some(Arc::new(pos))
} else {
Some(Arc::new(crate::SourcePosition {
index: 0,
offset: 0,
line: self.current_line_offset + 1,
column: 1,
}))
Expand Down
8 changes: 4 additions & 4 deletions brush-interactive/src/highlighting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,13 @@ impl<'a, SE: brush_core::ShellExtensions> Highlighter<'a, SE> {
for token in tokens {
match token {
brush_parser::Token::Operator(_op, token_location) => {
let start = global_offset + byte_offset(token_location.start.index);
let end = global_offset + byte_offset(token_location.end.index);
let start = global_offset + byte_offset(token_location.start.offset);
let end = global_offset + byte_offset(token_location.end.offset);
self.append_span(HighlightKind::Operator, start..end);
}
brush_parser::Token::Word(w, token_location) => {
let start_byte = byte_offset(token_location.start.index);
let end_byte = byte_offset(token_location.end.index);
let start_byte = byte_offset(token_location.start.offset);
let end_byte = byte_offset(token_location.end.offset);

// Parse the raw slice from `line`, not `w.as_str()`: the tokenizer may
// drop chars from `w` (e.g. `\<newline>` continuations), so offsets into
Expand Down
80 changes: 74 additions & 6 deletions brush-parser/src/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2368,15 +2368,15 @@ echo there
SourcePosition {
line: 1,
column: 1,
index: 0
offset: 0
}
);
assert_eq!(
*(loc.end),
SourcePosition {
line: 2,
column: 11,
index: 18
offset: 18
}
);
}
Expand Down Expand Up @@ -2404,15 +2404,15 @@ my_func
SourcePosition {
line: 1,
column: 1,
index: 0
offset: 0
}
);
assert_eq!(
*(loc.end),
SourcePosition {
line: 4,
column: 2,
index: 36
offset: 36
}
);
}
Expand All @@ -2435,16 +2435,84 @@ my_func
SourcePosition {
line: 1,
column: 1,
index: 0
offset: 0
}
);
assert_eq!(
*(loc.end),
SourcePosition {
line: 1,
column: 28,
index: 27
offset: 27
}
);
}

#[test]
fn simple_cmd_loc_after_multibyte_comment() {
// Line 1: "# café\n" — 'é' is 2 bytes (0xC3 0xA9), total 8 bytes
// Line 2: "ls\n" — 'l' is at byte offset 8, total 3 bytes
let input = "# café\nls\n";

let program = parse(input);

let Command::Simple(cmd) = &program.complete_commands[0].0[0].0.first.seq[0] else {
panic!("expected simple command");
};

let loc = cmd.location().unwrap();

assert_eq!(
*(loc.start),
SourcePosition {
line: 2,
column: 1,
offset: 8
}
);
assert_eq!(
*(loc.end),
SourcePosition {
line: 2,
column: 3,
offset: 10
}
);

assert_eq!(input.get(loc.start.offset..loc.end.offset), Some("ls"));
}

#[test]
fn simple_cmd_loc_after_multiple_multibyte_chars() {
// "# café 日本語\n" = 18 bytes (é=2, 日=3, 本=3, 語=3 + ASCII)
// "ls" starts at byte offset 18
let input = "# café 日本語\nls\n";

let program = parse(input);

let Command::Simple(cmd) = &program.complete_commands[0].0[0].0.first.seq[0] else {
panic!("expected simple command");
};

let loc = cmd.location().unwrap();

assert_eq!(
*(loc.start),
SourcePosition {
line: 2,
column: 1,
offset: 18
}
);
assert_eq!(
*(loc.end),
SourcePosition {
line: 2,
column: 3,
offset: 20
}
);

assert_eq!(input.get(loc.start.offset..loc.end.offset), Some("ls"));
}
}
6 changes: 3 additions & 3 deletions brush-parser/src/parser/peg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,7 +753,7 @@ fn add_pipe_extension_redirection(c: &mut ast::Command) -> Result<(), &'static s

#[inline]
fn locations_are_contiguous(loc_left: &crate::SourceSpan, loc_right: &crate::SourceSpan) -> bool {
loc_left.end.index == loc_right.start.index
loc_left.end.offset == loc_right.start.offset
}

impl peg::Parse for Tokens<'_> {
Expand Down Expand Up @@ -811,13 +811,13 @@ impl<'a> peg::ParseSlice<'a> for Tokens<'a> {
let loc = token.location();

if let Some(prev_end) = prev_end_index {
if loc.start.index > prev_end {
if loc.start.offset > prev_end {
result.push(' ');
}
}

result.push_str(token.to_str());
prev_end_index = Some(loc.end.index);
prev_end_index = Some(loc.end.offset);
}

result
Expand Down
2 changes: 1 addition & 1 deletion brush-parser/src/parser/tests/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ fn is_source_span(value: &Value) -> bool {
fn normalize_source_span(value: &mut Value) {
if let Value::Object(map) = value {
let placeholder_pos = serde_json::json!({
"index": 0,
"offset": 0,
"line": 0,
"column": 0
});
Expand Down
4 changes: 2 additions & 2 deletions ...c/parser/tests/snapshots/brush_parser__parser__tests__complex__parse_nested_if_while.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,12 @@ ParseResult(
loc: "[location]",
), SourceSpan(
start: SourcePosition(
index: 18,
offset: 18,
line: 2,
column: 5,
),
end: SourcePosition(
index: 67,
offset: 67,
line: 4,
column: 9,
),
Expand Down
4 changes: 2 additions & 2 deletions ...c/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_until.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ ParseResult(
loc: "[location]",
), SourceSpan(
start: SourcePosition(
index: 0,
offset: 0,
line: 1,
column: 1,
),
end: SourcePosition(
index: 31,
offset: 31,
line: 1,
column: 32,
),
Expand Down
4 changes: 2 additions & 2 deletions ...c/parser/tests/snapshots/brush_parser__parser__tests__compound_commands__parse_while.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ ParseResult(
loc: "[location]",
), SourceSpan(
start: SourcePosition(
index: 0,
offset: 0,
line: 1,
column: 1,
),
end: SourcePosition(
index: 30,
offset: 30,
line: 1,
column: 31,
),
Expand Down
4 changes: 2 additions & 2 deletions ...ests/snapshots/brush_parser__parser__tests__compound_commands__parse_while_multiline.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,12 @@ ParseResult(
loc: "[location]",
), SourceSpan(
start: SourcePosition(
index: 0,
offset: 0,
line: 1,
column: 1,
),
end: SourcePosition(
index: 42,
offset: 42,
line: 5,
column: 5,
),
Expand Down
32 changes: 16 additions & 16 deletions brush-parser/src/parser/tests/snapshots/brush_parser__parser__tests__parse_program.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ ParseResult(
value: "A",
loc: Some(SourceSpan(
start: SourcePosition(
index: 32,
offset: 32,
line: 5,
column: 10,
),
end: SourcePosition(
index: 33,
offset: 33,
line: 5,
column: 11,
),
Expand All @@ -32,12 +32,12 @@ ParseResult(
value: "B",
loc: Some(SourceSpan(
start: SourcePosition(
index: 34,
offset: 34,
line: 5,
column: 12,
),
end: SourcePosition(
index: 35,
offset: 35,
line: 5,
column: 13,
),
Expand All @@ -47,12 +47,12 @@ ParseResult(
value: "C",
loc: Some(SourceSpan(
start: SourcePosition(
index: 36,
offset: 36,
line: 5,
column: 14,
),
end: SourcePosition(
index: 37,
offset: 37,
line: 5,
column: 15,
),
Expand All @@ -69,12 +69,12 @@ ParseResult(
value: "echo",
loc: Some(SourceSpan(
start: SourcePosition(
index: 60,
offset: 60,
line: 8,
column: 5,
),
end: SourcePosition(
index: 64,
offset: 64,
line: 8,
column: 9,
),
Expand All @@ -85,12 +85,12 @@ ParseResult(
value: "\"${f@L}\"",
loc: Some(SourceSpan(
start: SourcePosition(
index: 65,
offset: 65,
line: 8,
column: 10,
),
end: SourcePosition(
index: 73,
offset: 73,
line: 8,
column: 18,
),
Expand All @@ -100,12 +100,12 @@ ParseResult(
value: "2",
loc: Some(SourceSpan(
start: SourcePosition(
index: 76,
offset: 76,
line: 8,
column: 21,
),
end: SourcePosition(
index: 77,
offset: 77,
line: 8,
column: 22,
),
Expand All @@ -119,25 +119,25 @@ ParseResult(
]),
loc: SourceSpan(
start: SourcePosition(
index: 39,
offset: 39,
line: 5,
column: 17,
),
end: SourcePosition(
index: 86,
offset: 86,
line: 10,
column: 8,
),
),
),
loc: SourceSpan(
start: SourcePosition(
index: 23,
offset: 23,
line: 5,
column: 1,
),
end: SourcePosition(
index: 86,
offset: 86,
line: 10,
column: 8,
),
Expand Down
4 changes: 2 additions & 2 deletions ...tests/snapshots/brush_parser__parser__tests__pipelines__parse_negated_timed_pipeline.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ ParseResult(
first: Pipeline(
timed: Some(Timed(SourceSpan(
start: SourcePosition(
index: 0,
offset: 0,
line: 1,
column: 1,
),
end: SourcePosition(
index: 4,
offset: 4,
line: 1,
column: 5,
),
Expand Down
Loading
Loading