diff --git a/assets/default_newlines.packdump b/assets/default_newlines.packdump index 54977dc0..9e45d9bd 100644 Binary files a/assets/default_newlines.packdump and b/assets/default_newlines.packdump differ diff --git a/assets/default_nonewlines.packdump b/assets/default_nonewlines.packdump index 21e4c431..e6922f67 100644 Binary files a/assets/default_nonewlines.packdump and b/assets/default_nonewlines.packdump differ diff --git a/examples/syntest.rs b/examples/syntest.rs index f65ff25b..e0166647 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -278,6 +278,7 @@ fn get_line_assertion_details<'a>( fn process_assertions( assertion: &AssertionRange<'_>, test_against_line_scopes: &[ScopedText], + next_line_scopes: Option<&[ScopedText]>, ) -> Vec { // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" // and they are sometimes in the syntax test as ^^^-comment, for example @@ -301,16 +302,59 @@ fn process_assertions( }; results.push(result); } - // don't ignore assertions after the newline, they should be treated as though they are asserting against the newline + // Past-EOL columns: ST's `view.text_point(row, col)` overflows into the + // next row when `col` exceeds the current line's char count, so its + // syntax-test framework evaluates past-EOL assertions against the + // corresponding column on the line below. Mirror that here when we have + // the next line's scopes; otherwise fall back to the last char's scope. let last = test_against_line_scopes.last().unwrap(); - if last.char_start + last.text_len < assertion.end_char { - let match_value = selector.does_match(last.scope.as_slice()); - let result = RangeTestResult { - column_begin: max(last.char_start + last.text_len, assertion.begin_char), - column_end: assertion.end_char, - success: match_value.is_some(), - }; - results.push(result); + let last_end = last.char_start + last.text_len; + if last_end < assertion.end_char { + let past_eol_begin = max(last_end, assertion.begin_char); + let past_eol_end = assertion.end_char; + if let Some(next_scopes) = next_line_scopes.filter(|s| !s.is_empty()) { + // Wrap formula: position `col` past EOL of a line of total length + // `last_end` (chars including trailing `\n`) lands on column + // `col - last_end` of the next line. + let wrap_begin = past_eol_begin - last_end; + let wrap_end = past_eol_end - last_end; + let mut covered = wrap_begin; + for scoped_text in next_scopes + .iter() + .skip_while(|s| s.char_start + s.text_len <= wrap_begin) + .take_while(|s| s.char_start < wrap_end) + { + let next_begin = max(scoped_text.char_start, wrap_begin); + let next_end = min(scoped_text.char_start + scoped_text.text_len, wrap_end); + let match_value = selector.does_match(scoped_text.scope.as_slice()); + results.push(RangeTestResult { + column_begin: next_begin + last_end, + column_end: next_end + last_end, + success: match_value.is_some(), + }); + covered = next_end; + } + // Wrap target extends past the next line's content too — recursive + // wrap is not yet implemented; fall back to the next line's last + // scope so the assertion still gets a defined verdict instead of + // silently passing. + if covered < wrap_end { + let next_last = next_scopes.last().unwrap(); + let match_value = selector.does_match(next_last.scope.as_slice()); + results.push(RangeTestResult { + column_begin: covered + last_end, + column_end: past_eol_end, + success: match_value.is_some(), + }); + } + } else { + let match_value = selector.does_match(last.scope.as_slice()); + results.push(RangeTestResult { + column_begin: past_eol_begin, + column_end: past_eol_end, + success: match_value.is_some(), + }); + } } results } @@ -368,7 +412,14 @@ fn test_file( let mut current_line_number = 1; let mut test_against_line_number = 1; - let mut scopes_on_line_being_tested = Vec::new(); + let mut scopes_on_line_being_tested: Vec = Vec::new(); + // Scopes of the first line that follows the current target line. ST's + // syntax-test framework evaluates past-EOL assertion columns against the + // corresponding column on the next line (because `text_point(row, col)` + // overflows into the next row when `col` exceeds the row's length); we + // mirror that by remembering the next line's scopes once and feeding + // them into `process_assertions`. Reset whenever the target line changes. + let mut next_target_line_scopes: Option> = None; let mut previous_non_assertion_line = line.to_string(); let mut assertion_failures: usize = 0; @@ -383,61 +434,21 @@ fn test_file( loop { // over lines of file, starting with the header line - let mut line_only_has_assertion = false; - let mut line_has_assertion = false; - if let Some(assertion) = get_line_assertion_details(testtoken_start, testtoken_end, &line) { - // `@+` and `>` lines are annotation-only (reference labels / reference - // assertions). They must be recognised so they do not drive - // `test_against_line_number`, but we do not yet implement - // cross-line label lookups, so no scope checks run here. - let mut current_assertion_failures: usize = 0; - if !assertion.is_reference { - let result = process_assertions(&assertion, &scopes_on_line_being_tested); - total_assertions += assertion.end_char - assertion.begin_char; - for failure in result.iter().filter(|r| !r.success) { - let length = failure.column_end - failure.column_begin; - let text: String = previous_non_assertion_line - .chars() - .skip(failure.column_begin) - .take(length) - .collect(); - pending_messages.push(BufferedFailureMessage { - selector_text: assertion.scope_selector_text.trim().to_string(), - assertion_line_number: current_line_number, - test_against_line_number, - column_begin: failure.column_begin, - column_end: failure.column_end, - text, - scope: scopes_on_line_being_tested - .iter() - .find(|s| s.char_start + s.text_len > failure.column_begin) - .unwrap_or_else(|| scopes_on_line_being_tested.last().unwrap()) - .scope - .clone(), - }); - assertion_failures += failure.column_end - failure.column_begin; - current_assertion_failures += failure.column_end - failure.column_begin; - } - // Buffer this assertion for re-evaluation if backtracking replays the target line - if let Some(idx) = current_test_line_buffer_idx { - if let Some(ref mut data) = parsed_line_buffer[idx].non_assertion_data { - data.assertions.push(BufferedAssertion { - begin_char: assertion.begin_char, - end_char: assertion.end_char, - scope_selector_text: assertion.scope_selector_text.to_string(), - assertion_line_number: current_line_number, - }); - data.assertion_failures += current_assertion_failures; - } - } - } // end `if !assertion.is_reference` - line_only_has_assertion = assertion.is_pure_assertion_line; - line_has_assertion = true; - } + let assertion_opt = get_line_assertion_details(testtoken_start, testtoken_end, &line); + let line_has_assertion = assertion_opt.is_some(); + let line_only_has_assertion = assertion_opt + .as_ref() + .map(|a| a.is_pure_assertion_line) + .unwrap_or(false); + + // Parse first so the just-parsed scopes are available when the + // assertion runs immediately after — needed for past-EOL wrap on the + // first assertion line after a target. if !line_only_has_assertion || parse_test_lines { if !line_has_assertion { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against scopes_on_line_being_tested.clear(); + next_target_line_scopes = None; test_against_line_number = current_line_number; previous_non_assertion_line = line.to_string(); } @@ -447,7 +458,6 @@ fn test_file( current_line_number, stack, state ); } - let stack_before = stack.clone(); let output = match state.parse_line(&line, ss) { Ok(output) => output, Err(e) => { @@ -496,6 +506,19 @@ fn test_file( // Reset stack to the state before the first replayed line stack = parsed_line_buffer[start_idx].stack_before.clone(); + // Collect corrected baselines to apply post-loop, since the + // mutable record borrow inside the loop forbids touching + // sibling buffer entries. After applying replayed[i], + // `stack` is the corrected end-of-line for the buffered + // record at start_idx + i, i.e. the corrected + // start-of-line baseline for record at start_idx + i + 1. + // Overwriting that baseline prevents a future replay from + // resurrecting any meta_scope the prior replay had unwound + // (observed as meta.link.reference.def.markdown leak past + // back-to-back Markdown link reference definitions). + let buf_len = parsed_line_buffer.len(); + let mut corrected_baselines: Vec<(usize, ScopeStack)> = Vec::new(); + for (i, replayed_ops) in replayed.iter().enumerate() { let record = &mut parsed_line_buffer[start_idx + i]; let has_non_assertion = record.non_assertion_data.is_some(); @@ -516,6 +539,10 @@ fn test_file( col += len; } } + let next_idx = start_idx + i + 1; + if next_idx < buf_len { + corrected_baselines.push((next_idx, stack.clone())); + } if let Some(ref mut data) = record.non_assertion_data { if !data.assertions.is_empty() { @@ -532,7 +559,12 @@ fn test_file( // above, so replays never hit reference lines. is_reference: false, }; - let result = process_assertions(&temp_assertion, &new_scoped); + // Replay path: fall back to the previous + // past-EOL semantics by passing no next-line + // scopes. Replays are rare and per-target; + // recomputing the next line's scopes here + // would require also replaying its ops. + let result = process_assertions(&temp_assertion, &new_scoped, None); for failure in result.iter().filter(|r| !r.success) { let length = failure.column_end - failure.column_begin; let text: String = record @@ -574,6 +606,9 @@ fn test_file( } } } + for (idx, corrected) in corrected_baselines { + parsed_line_buffer[idx].stack_before = corrected; + } } if out_opts.debug && !line_only_has_assertion { @@ -583,6 +618,18 @@ fn test_file( debug_print_ops(&line, &ops); } } + // Snapshot the stack now (post-replays, pre-current-ops) — this + // becomes the buffered `stack_before` for the current line, so a + // future replay covering it resets to the corrected baseline + // rather than the stale pre-parse value. + let stack_before = stack.clone(); + // Build the just-parsed line's scopes. For non-assertion (target) + // lines they go into `scopes_on_line_being_tested`; for the FIRST + // assertion line that follows a target they go into a fresh vec + // that becomes `next_target_line_scopes` (used for past-EOL wrap). + // Subsequent assertion lines don't need their scopes captured. + let capture_for_next_target = line_has_assertion && next_target_line_scopes.is_none(); + let mut next_target_buffer: Vec = Vec::new(); let mut col: usize = 0; for (s, op) in ScopeRegionIterator::new(&ops, &line) { if let Err(_) = stack.apply(op) { @@ -599,17 +646,23 @@ fn test_file( // in this case we don't care about blank tokens continue; } + let len = s.chars().count(); + let scoped = ScopedText { + char_start: col, + text_len: len, + scope: stack.as_slice().to_vec(), + }; if !line_has_assertion { // if the line has no assertions on it, remember the scopes on the line so we can test against them later - let len = s.chars().count(); - scopes_on_line_being_tested.push(ScopedText { - char_start: col, - text_len: len, - scope: stack.as_slice().to_vec(), - }); - // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired - col += len; + scopes_on_line_being_tested.push(scoped); + } else if capture_for_next_target { + next_target_buffer.push(scoped); } + // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired + col += len; + } + if capture_for_next_target { + next_target_line_scopes = Some(next_target_buffer); } // Buffer this parsed line for potential future replay @@ -630,13 +683,67 @@ fn test_file( if !line_has_assertion { current_test_line_buffer_idx = Some(parsed_line_buffer.len() - 1); } + } - // Flush buffered failure messages once the parser commits - if !state.is_speculative() { - flush_pending_messages(&mut pending_messages, out_opts.summary); + // Process the assertion (after parsing, so past-EOL wrap can see the + // current line's scopes via `next_target_line_scopes`). + if let Some(assertion) = assertion_opt { + // `@+` and `>` lines are annotation-only (reference labels / + // reference assertions). They must be recognised so they do not + // drive `test_against_line_number`, but we do not yet implement + // cross-line label lookups, so no scope checks run here. + let mut current_assertion_failures: usize = 0; + if !assertion.is_reference { + let result = process_assertions( + &assertion, + &scopes_on_line_being_tested, + next_target_line_scopes.as_deref(), + ); + total_assertions += assertion.end_char - assertion.begin_char; + for failure in result.iter().filter(|r| !r.success) { + let length = failure.column_end - failure.column_begin; + let text: String = previous_non_assertion_line + .chars() + .skip(failure.column_begin) + .take(length) + .collect(); + pending_messages.push(BufferedFailureMessage { + selector_text: assertion.scope_selector_text.trim().to_string(), + assertion_line_number: current_line_number, + test_against_line_number, + column_begin: failure.column_begin, + column_end: failure.column_end, + text, + scope: scopes_on_line_being_tested + .iter() + .find(|s| s.char_start + s.text_len > failure.column_begin) + .unwrap_or_else(|| scopes_on_line_being_tested.last().unwrap()) + .scope + .clone(), + }); + assertion_failures += failure.column_end - failure.column_begin; + current_assertion_failures += failure.column_end - failure.column_begin; + } + // Buffer this assertion for re-evaluation if backtracking replays the target line + if let Some(idx) = current_test_line_buffer_idx { + if let Some(ref mut data) = parsed_line_buffer[idx].non_assertion_data { + data.assertions.push(BufferedAssertion { + begin_char: assertion.begin_char, + end_char: assertion.end_char, + scope_selector_text: assertion.scope_selector_text.to_string(), + assertion_line_number: current_line_number, + }); + data.assertion_failures += current_assertion_failures; + } + } } } + // Flush buffered failure messages once the parser commits + if !state.is_speculative() { + flush_pending_messages(&mut pending_messages, out_opts.summary); + } + line.clear(); current_line_number += 1; if reader.read_line(&mut line).unwrap() == 0 { @@ -923,4 +1030,104 @@ mod tests { let a = details(CC, None, "// @@@").unwrap(); assert!(a.is_reference); } + + /// Build a `ScopedText` from a string of space-separated scope atoms. + fn st(char_start: usize, text_len: usize, scopes: &str) -> ScopedText { + ScopedText { + char_start, + text_len, + scope: scopes + .split_whitespace() + .map(|s| Scope::new(s).unwrap()) + .collect(), + } + } + fn assertion_range(begin_char: usize, end_char: usize, sel: &str) -> AssertionRange<'_> { + AssertionRange { + begin_char, + end_char, + scope_selector_text: sel, + is_pure_assertion_line: true, + is_reference: false, + } + } + + #[test] + fn past_eol_negative_assertion_uses_next_line_wrap() { + // git_config-shaped: the consumed `\n` of the target line still + // carries the parent meta_scope (`meta.section`), but the next line + // is a comment whose scope does not include `meta.section`. + // Negative past-EOL assertion `- meta.section` must pass. + let target = vec![ + st(0, 1, "text meta.section punctuation.section.brackets.begin"), + st( + 28, + 1, + "text meta.section meta.brackets invalid.illegal.unexpected.eol", + ), + ]; + let next = vec![ + st(0, 1, "text comment.line punctuation.definition.comment"), + st(1, 1, "text comment.line"), + ]; + let a = assertion_range(29, 30, "- meta.section"); + let r = process_assertions(&a, &target, Some(&next)); + assert_eq!(r.len(), 1); + assert!(r[0].success, "expected pass via wrap to next line: {r:?}"); + } + + #[test] + fn past_eol_positive_assertion_matches_next_line_scope() { + // Past-EOL assertion expecting `comment.line` finds it on the next + // line — both git_config (lines 554-555) and Clojure (line 33) work + // this way in ST. + let target = vec![ + st( + 0, + 1, + "text meta.mapping.value string.quoted.double punctuation.definition.string.end", + ), + st(81, 1, "text"), + ]; + let next = vec![ + st(0, 1, "text comment.line punctuation.definition.comment"), + st(1, 14, "text comment.line"), + ]; + let a = assertion_range(82, 97, "comment.line"); + let r = process_assertions(&a, &target, Some(&next)); + assert!( + r.iter().all(|res| res.success), + "expected all wrap segments to pass: {r:?}", + ); + } + + #[test] + fn past_eol_falls_back_when_next_line_unavailable() { + // Without next-line scopes (end-of-file or replay path), keep the + // previous behaviour of testing against the last char's scope. + let target = vec![st(0, 5, "text constant.numeric"), st(5, 1, "text")]; + let a_pass = assertion_range(6, 7, "- constant.numeric"); + let r_pass = process_assertions(&a_pass, &target, None); + assert_eq!(r_pass.len(), 1); + assert!(r_pass[0].success); + let a_fail = assertion_range(6, 7, "constant.numeric"); + let r_fail = process_assertions(&a_fail, &target, None); + assert_eq!(r_fail.len(), 1); + assert!(!r_fail[0].success); + } + + #[test] + fn past_eol_wrap_overshooting_next_line_uses_next_line_last_scope() { + // When the wrap target extends past the next line's content too, + // fall back to the next line's last scope (recursive wrap is a v2 + // concern). The verdict must still be defined, not silently skipped. + let target = vec![st(0, 1, "text"), st(1, 1, "text")]; + let next = vec![st(0, 3, "text source.x")]; + // Wrap range: cols [2, 8) on target → cols [0, 6) on next. + // Cols [3, 6) on next overshoot the 3-char `next` and should fall + // back to the next-line last scope (`text source.x`). + let a = assertion_range(2, 8, "source.x"); + let r = process_assertions(&a, &target, Some(&next)); + assert!(r.iter().all(|res| res.success), "got: {r:?}"); + } } diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs index 9d317bd2..e1a9dbef 100644 --- a/src/parsing/parser.rs +++ b/src/parsing/parser.rs @@ -90,15 +90,101 @@ pub struct ParseState { /// cross-line `fail` replay. Only the strings are stored; the ops are /// returned to callers immediately (same as before). pending_lines: Vec, + /// Snapshot of `shadow` at the start of each buffered line in + /// `pending_lines`. Used by the cross-line-fail replay to restore + /// `shadow` to its state at the first replayed line's beginning, so + /// the shadow mirrors what the consumer does (reset + apply + /// replayed). + pending_line_start_shadows: Vec, /// Corrected ops produced by a cross-line `fail` replay, to be returned - /// as `ParseLineOutput::replayed` at the end of `parse_line`. + /// as `ParseLineOutput::replayed` at the end of `parse_line`. When + /// populated, entry `i` corresponds to `pending_lines[flushed_ops_start + i]`. flushed_ops: Vec>, + /// Pending-lines index that `flushed_ops[0]` maps to when `flushed_ops` + /// is non-empty. Reset to `None` between `parse_line` calls. + flushed_ops_start: Option, + /// Identity of the branch point whose cross-line replay produced + /// `flushed_ops`. `None` when `flushed_ops_start` is `None`. + /// Used by `prefer_inner_replay_corrections` to discriminate + /// between an inner BP whose correction is structurally a refinement + /// of the outer BP's resolved alternative (prefer) vs. one that + /// would over-apply (keep outer). + flushed_ops_bp: Option, /// Warnings accumulated during parsing, drained into `ParseLineOutput`. warnings: Vec, /// Active escape patterns from embed operations. The escape regex takes /// strict precedence over normal patterns — it is checked first and can /// truncate the search region. escape_stack: Vec, + /// Mirror of the consumer's scope stack. Updated at `parse_line` + /// boundaries (not mid-line) from the returned `ops` and + /// `replayed`, mirroring the consumer's behaviour (reset to the + /// first-replayed line's start, then apply replayed, then apply + /// current ops). `exec_escape` uses it to detect orphan atoms left + /// on the consumer's stack by a prior cross-line replay whose + /// later same-line fails truncated the owning context out of + /// `self.stack` (the Push for the atom is committed in + /// `flushed_ops`, so it can't be taken back by `ops.truncate`) — + /// and emits a balancing Pop before the normal escape pops. + shadow: ScopeStack, + /// Active while `handle_fail` recurses into `parse_line_inner*` to + /// replay a buffered past line under a new alternative. Overrides + /// the "current line" / "pending_lines slot" bookkeeping that + /// branches created during the re-parse record, so they anchor to + /// the replay line `L+i` rather than the outer `parse_line`'s + /// current line. Without it, a later fail on the outer line + /// misclassifies the replay-born branch as same-line and applies + /// its replay-line-relative `match_start` to a shorter outer line + /// (the byte-20-out-of-13 panic on `syntax_test_java.java:10263` + /// inside `@MultiLineAnnotation(...)`). + replay_ctx: Option, + /// Ops the outer cross-line replay has already composed for the + /// first replayed line — outer prefix_ops + new-alt meta/pat/capture + /// emission. A branch_point created during the inner re-parse + /// records `replay_prefix_ops + ops` as its own `prefix_ops`, so + /// when it later fails and reconstructs its line, the outer + /// captures (e.g. `[foo]:` LRD opener) survive instead of being + /// rebuilt from an empty Vec — the cause of `meta.link.reference` + /// scope loss in `syntax_test_markdown.md`'s `[foo]: /url` cases + /// where `link-def-title-continuation`'s fail spawns a nested + /// `link-def-attr-continuation` whose own fail then replayed line 3 + /// without the original captures. + replay_prefix_ops: Option>, + /// Branch_points whose alternatives have all been exhausted at a + /// specific cursor position on the current line. Subsequent + /// `find_best_match` calls at that position skip the matching pattern + /// so the parent context's NEXT rule gets a chance — Sublime Text's + /// behaviour. Without this, syntect's prior approach (advance one + /// character past the lookahead) lets stale keyword rules match in + /// the middle of identifiers (e.g. `package` inside `$package` after + /// `declarations` exhausted on the leading `$`). Cleared whenever + /// the cursor moves. + skipped_branches: Vec<(usize, String)>, +} + +/// Identity of a branch point whose cross-line replay wrote ops to +/// `flushed_ops`. Captured so `prefer_inner_replay_corrections` can +/// compare the inner BP (whose corrections are candidate replacements) +/// against the outer BP (whose locally-computed replay is the default). +#[derive(Debug, Clone, Eq, PartialEq)] +struct BpInfo { + name: String, + /// Stack depth at branch creation (mirrors `BranchPoint::stack_depth`). + stack_depth: usize, + /// Line number at branch creation (mirrors `BranchPoint::line_number`). + line_number: usize, +} + +/// Bookkeeping override used while `handle_fail` is re-parsing a +/// buffered past line. See the `replay_ctx` field on `ParseState`. +#[derive(Debug, Clone, Eq, PartialEq)] +struct ReplayCtx { + /// Virtual "current line" of the inner re-parse (`bp.line_number + i`). + line_number: usize, + /// Slot in `self.pending_lines` that a branch created during this + /// replay iteration should record as its snapshot length, so a + /// future cross-line fail replays from `L+i` onward. + pending_lines_snapshot_offset: usize, } /// A resolved escape pattern from an `embed` operation, stored on the escape stack. @@ -157,6 +243,14 @@ struct BranchPoint { /// ESCAPE …`, every non-whitespace before the `LIKE` fires /// `else-pop` in the escape-alternative, derailing the stack). prefix_ops: Vec<(usize, ScopeStackOp)>, + /// Capture Push/Pop ops emitted alongside the branch_point match's + /// `pat_scope`. Re-emitted on fail-retry between the pat_scope + /// Push and Pop so captures like `keyword.declaration.data.haskell` + /// on the first capture group of `(data)(?:\s+(family|instance))?` + /// survive a branch swap — without this, a `data CtxCls ctx => …` + /// (where `alt[0]` `data-signature` fails into `alt[1]` `data-context`) + /// drops the keyword scope from the `data` token. + capture_ops: Vec<(usize, ScopeStackOp)>, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -180,6 +274,46 @@ struct RegexMatch<'a> { /// Maps the pattern to the start index, which is -1 if not found. type SearchCache = HashMap<*const MatchPattern, Option, BuildHasherDefault>; +/// Build the ordered Push/Pop ops for a match's `captures:` mapping over +/// its regex `regions`. Captures can appear in arbitrary source order +/// (e.g. `((bob)|(hi))*` matching `hibob` — the outer group must Push +/// before any inner group). Empty captures are skipped because they'd +/// otherwise sort a Pop before its Push. The returned ops are already +/// position-ordered and safe to append to a parser ops vec. +/// +/// Each capture's `(cap_start, cap_end)` is clipped to the outer match +/// range `regions.pos(0)` so a group matching inside a `(?=...)` / +/// `(?<=...)` whose own span extends past the consumed range still +/// colours the overlap (the boundary char) and nothing beyond it. This +/// mirrors Sublime Text; without the clip, a lookahead-internal +/// `captures:` entry leaked scope over unmatched trailing chars or was +/// silently dropped upstream in `parse_captures`. +fn build_capture_ops(capture_map: &CaptureMapping, regions: &Region) -> Vec<(usize, ScopeStackOp)> { + let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new(); + let (match_start, match_end) = match regions.pos(0) { + Some(bounds) => bounds, + None => return Vec::new(), + }; + for &(cap_index, ref scopes) in capture_map.iter() { + if let Some((cap_start, cap_end)) = regions.pos(cap_index) { + let clipped_start = cap_start.max(match_start); + let clipped_end = cap_end.min(match_end); + if clipped_start >= clipped_end { + continue; + } + for scope in scopes.iter() { + map.push(( + (clipped_start, -((clipped_end - clipped_start) as i32)), + ScopeStackOp::Push(*scope), + )); + } + map.push(((clipped_end, i32::MIN), ScopeStackOp::Pop(scopes.len()))); + } + } + map.sort_by(|a, b| a.0.cmp(&b.0)); + map.into_iter().map(|((i, _), op)| (i, op)).collect() +} + // To understand the implementation of this, here's an introduction to how // Sublime Text syntax definitions work. // @@ -288,9 +422,16 @@ impl ParseState { branch_points: Vec::new(), line_number: 0, pending_lines: Vec::new(), + pending_line_start_shadows: Vec::new(), flushed_ops: Vec::new(), + flushed_ops_start: None, + flushed_ops_bp: None, warnings: Vec::new(), escape_stack: Vec::new(), + shadow: ScopeStack::new(), + replay_ctx: None, + replay_prefix_ops: None, + skipped_branches: Vec::new(), } } @@ -322,6 +463,10 @@ impl ParseState { return Err(ParsingError::MissingMainContext); } + // Skipped-branch entries are tied to byte offsets within a single + // line — they don't survive across line boundaries. + self.skipped_branches.clear(); + // Prune branch points older than 128 lines let cur_line = self.line_number; let warnings = &mut self.warnings; @@ -337,18 +482,71 @@ impl ParseState { }); self.line_number += 1; + let pending_lines_before = self.pending_lines.len(); + let ops = self.parse_line_inner(line, syntax_set)?; // Collect any corrected ops produced by a cross-line `fail` during the // parse above. These are stored by `handle_fail` in `self.flushed_ops`. let replayed = std::mem::take(&mut self.flushed_ops); + self.flushed_ops_start = None; + self.flushed_ops_bp = None; + + // Update shadow to reflect consumer's view at end of this line. + // The consumer (see `syntest`) resets its scope stack to + // `parsed_line_buffer[start_idx].stack_before` when `replayed` is + // non-empty, then applies replayed then applies current-line ops. + // Mirror that here so `shadow` matches the consumer downstream. + // + // While re-applying replays, also overwrite each buffered line's + // pending_line_start_shadows entry with the corrected baseline. + // Without this, a later replay covering this same line would reset + // shadow to a stale snapshot captured before the prior replay's + // correction landed, causing scope leaks (e.g. + // meta.link.reference.def.markdown persisting past back-to-back + // Markdown link reference definitions, since each LRD's correction + // arrives in the *next* line's parse_line and the snapshot for + // that next line was captured from the buggy uncorrected stack). + if !replayed.is_empty() { + let start_idx = pending_lines_before + .checked_sub(replayed.len()) + .unwrap_or(0); + if let Some(snap) = self.pending_line_start_shadows.get(start_idx) { + self.shadow = snap.clone(); + } + for (i, line_ops) in replayed.iter().enumerate() { + for (_, op) in line_ops { + let _ = self.shadow.apply(op); + } + // After applying replayed[i], shadow == start of buffered + // line (start_idx + i + 1). Overwrite that snapshot so the + // next replay covering it starts from the corrected + // baseline rather than the stale one captured pre-replay. + let next_idx = start_idx + i + 1; + if next_idx < self.pending_line_start_shadows.len() { + self.pending_line_start_shadows[next_idx] = self.shadow.clone(); + } + } + } + + // Snapshot the shadow now (post-replays, pre-current-ops) — this + // becomes the baseline for the next line if the current line ends + // with live branch_points and gets buffered for future replay. + let shadow_at_start_corrected = self.shadow.clone(); + + for (_, op) in &ops { + let _ = self.shadow.apply(op); + } // Keep the line string for potential future cross-line replay. if !self.branch_points.is_empty() { self.pending_lines.push(line.to_string()); + self.pending_line_start_shadows + .push(shadow_at_start_corrected); } else { // No active branch points: any buffered strings are stale. self.pending_lines.clear(); + self.pending_line_start_shadows.clear(); } let warnings = std::mem::take(&mut self.warnings); @@ -540,6 +738,11 @@ impl ParseState { *start = match_end; + // Prune stale skipped_branches entries: a BP marked as skipped + // at an older cursor position no longer matters once the + // cursor has advanced past that position. + self.skipped_branches.retain(|(c, _)| *c >= *start); + // ignore `with_prototype`s below this if a context is pushed if reg_match.from_with_prototype { // use current height, since we're before the actual push @@ -562,6 +765,18 @@ impl ParseState { )?; Ok(true) + } else if self.skipped_branches.iter().any(|(c, _)| *c == *start) { + // No pattern matched, but we suppressed at least one Branch + // at this cursor (its alts all exhausted). Advance one char as + // a last resort to break the loop. + self.skipped_branches.retain(|(c, _)| *c != *start); + if let Some((i, _)) = line[*start..].char_indices().nth(1) { + *start += i; + search_cache.clear(); + Ok(true) + } else { + Ok(false) + } } else { Ok(false) } @@ -646,6 +861,19 @@ impl ParseState { for (pat_context, pat_index) in context_iter(syntax_set, syntax_set.get_context(ctx)?) { let match_pat = pat_context.match_at(pat_index)?; + // Skip Branch patterns whose name was just exhausted at this + // cursor. See ParseState::skipped_branches and the same-line + // exhaustion handler in handle_fail. + if let MatchOperation::Branch { name, .. } = &match_pat.operation { + if self + .skipped_branches + .iter() + .any(|(c, n)| *c == start && n == name) + { + continue; + } + } + if let Some(match_region) = self.search_with_end( line, start, @@ -854,6 +1082,33 @@ impl ParseState { // keyword's own scopes so a same-line fail rewind can // re-emit them (they were truncated off `ops` along // with the alt[0]'s subsequent work). + // When `handle_fail` is mid-replay, `self.line_number` / + // `self.pending_lines` still reflect the *outer* current + // line — read through `replay_ctx` so a branch born + // during replay anchors to the virtual replay line `L+i`. + let (bp_line_number, bp_pending_lines_snapshot_len) = match &self.replay_ctx { + Some(ctx) => (ctx.line_number, ctx.pending_lines_snapshot_offset), + None => (self.line_number.saturating_sub(1), self.pending_lines.len()), + }; + // When this branch is born inside an outer cross-line + // replay's `parse_line_inner_from`, the local `ops` Vec + // is the inner re-parse's `res` — it does *not* include + // the outer prefix the outer replay is about to splice + // in front. Without prepending that outer prefix, a + // later fail of *this* branch reconstructs its line + // from an empty prefix, dropping the outer captures + // entirely (the `[foo]:` LRD opener vanished from + // `syntax_test_markdown.md`'s `[foo]: /url` cases when + // a `link-def-attr-continuation` born inside the + // `link-def-title-continuation` replay later failed). + let prefix_ops = match &self.replay_prefix_ops { + Some(outer) => { + let mut combined = outer.clone(); + combined.extend(ops.iter().cloned()); + combined + } + None => ops.clone(), + }; let bp = BranchPoint { name: name.clone(), next_alternative: 1, // 0 is about to be pushed @@ -863,16 +1118,21 @@ impl ParseState { match_start: *start, // position before this match's advance trigger_match_start: match_start, pat_scope: pat.scope.clone(), - line_number: self.line_number.saturating_sub(1), // current line (already incremented) + line_number: bp_line_number, ops_snapshot_len: ops.len(), stack_depth: self.stack.len(), non_consuming_push_at_snapshot: *non_consuming_push_at, first_line_snapshot: self.first_line, with_prototype: pat.with_prototype.clone(), - pending_lines_snapshot_len: self.pending_lines.len(), + pending_lines_snapshot_len: bp_pending_lines_snapshot_len, escape_stack_snapshot: self.escape_stack.clone(), pop_count, - prefix_ops: ops.clone(), + prefix_ops, + capture_ops: pat + .captures + .as_ref() + .map(|m| build_capture_ops(m, ®_match.regions)) + .unwrap_or_default(), }; self.branch_points.push(bp); // When pop_count > 0 (pop + branch), use Set semantics to @@ -902,31 +1162,12 @@ impl ParseState { for s in &pat.scope { ops.push((match_start, ScopeStackOp::Push(*s))); } - if let Some(ref capture_map) = pat.captures { - // captures could appear in an arbitrary order, have to produce ops in right order - // ex: ((bob)|(hi))* could match hibob in wrong order, and outer has to push first - // we don't have to handle a capture matching multiple times, Sublime doesn't - let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new(); - for &(cap_index, ref scopes) in capture_map.iter() { - if let Some((cap_start, cap_end)) = reg_match.regions.pos(cap_index) { - // marking up empty captures causes pops to be sorted wrong - if cap_start == cap_end { - continue; - } - for scope in scopes.iter() { - map.push(( - (cap_start, -((cap_end - cap_start) as i32)), - ScopeStackOp::Push(*scope), - )); - } - map.push(((cap_end, i32::MIN), ScopeStackOp::Pop(scopes.len()))); - } - } - map.sort_by(|a, b| a.0.cmp(&b.0)); - for ((index, _), op) in map.into_iter() { - ops.push((index, op)); - } - } + let capture_ops = pat + .captures + .as_ref() + .map(|m| build_capture_ops(m, ®_match.regions)) + .unwrap_or_default(); + ops.extend(capture_ops.iter().cloned()); if !pat.scope.is_empty() { ops.push((match_end, ScopeStackOp::Pop(pat.scope.len()))); } @@ -948,13 +1189,101 @@ impl ParseState { } } + /// Merge a cross-line replay's per-line corrected ops into `flushed_ops`. + /// + /// Multiple cross-line fails can fire on a single `parse_line` call (e.g. + /// Java line 624 with two live branches from line 615, both snapshotted at + /// `pending_lines_snapshot_len = 0`). Each fail's replay covers + /// `pending_lines[snap..pending_lines.len())`. A naive `extend` leaves + /// `flushed_ops` with duplicates — consumers of `ParseLineOutput::replayed` + /// index `replayed[i] ↔ pending_lines[i]` by `buf_len - replayed.len()`, + /// so duplicates misalign every byte offset. + /// + /// Composition rule, given current start `a` and new fail's `snap`: + /// - `snap <= a`: new fail supersedes everything; replace. + /// - `snap > a`: keep `[a..snap)` from prior fails, replace `[snap..N)`. + /// + /// `bp_info` records the BP whose replay produced `new_ops`; stored + /// alongside `flushed_ops_start` for later use by + /// `prefer_inner_replay_corrections`. When two cross-line fails on + /// the same `parse_line` both contribute, we keep the BP info of the + /// one that ultimately owns the prefix (matches the start-replacement + /// rules above). + fn merge_flushed( + &mut self, + snap: usize, + new_ops: Vec>, + bp_info: BpInfo, + ) { + match self.flushed_ops_start { + None => { + self.flushed_ops = new_ops; + self.flushed_ops_start = Some(snap); + self.flushed_ops_bp = Some(bp_info); + } + Some(start) if snap <= start => { + self.flushed_ops = new_ops; + self.flushed_ops_start = Some(snap); + self.flushed_ops_bp = Some(bp_info); + } + Some(start) => { + let keep = snap - start; + self.flushed_ops.truncate(keep); + self.flushed_ops.extend(new_ops); + } + } + } + + /// Replace `replayed_ops[i]` with `inner_ops` for overlapping + /// indices. Only fires when the inner BP was created at a stack + /// depth less-than-or-equal to the outer BP's depth — i.e. the + /// inner BP is a sibling-or-shallower correction at the same + /// structural level, not a nested BP firing inside outer's resolved + /// alternative. The "deeper inner" case is the multigen16 + /// regression seat: outer = `class-members` at depth 4, inner = + /// `object-type` at depth 9 (nested inside outer's resolved field + /// alt). Preferring its corrections doubled `meta.field.type.java` + /// on `Java/syntax_test_java.java:3462+` because outer's full-line + /// ops correctly emit one `meta.field.type` and inner's reparse + /// adds another. + /// + /// The depth-equal case is the original `@A.B\n(par=1)\nenum E {}` + /// regression seat from PR #663: outer and inner are both + /// `declarations` at depth 3 — sibling resolutions of the same + /// branch family, and inner's `name`-alt CORRECTLY supersedes + /// outer's locally-computed `path`-alt freeze. + fn prefer_inner_replay_corrections( + outer_snap: usize, + replayed_ops: &mut [Vec<(usize, ScopeStackOp)>], + inner_ops: &[Vec<(usize, ScopeStackOp)>], + inner_start: usize, + outer_bp: &BpInfo, + inner_bp: Option<&BpInfo>, + ) { + if let Some(inner) = inner_bp { + if inner.stack_depth > outer_bp.stack_depth { + return; + } + } + for (i, outer_local) in replayed_ops.iter_mut().enumerate() { + let global_i = outer_snap + i; + if global_i < inner_start { + continue; + } + let inner_idx = global_i - inner_start; + if let Some(corrected) = inner_ops.get(inner_idx) { + *outer_local = corrected.clone(); + } + } + } + /// Handle a `fail` operation by rewinding to the named branch point. /// Returns Ok(true) if backtracking happened (caller should continue from rewound position). /// Returns Ok(false) if the fail had no effect. fn handle_fail( &mut self, name: &str, - line: &str, + _line: &str, start: &mut usize, non_consuming_push_at: &mut (usize, usize), ops: &mut Vec<(usize, ScopeStackOp)>, @@ -980,7 +1309,15 @@ impl ParseState { None => return Ok(false), // No such branch point, fail is no-op }; - let cur_line = self.line_number.saturating_sub(1); + // During a replay recursion, `cur_line` is the virtual replay + // line — without this override a same-line fail fired inside + // the re-parse would be misclassified as cross-line, and a + // fail on the outer line for a branch created during replay + // would be misclassified as same-line. + let cur_line = match &self.replay_ctx { + Some(ctx) => ctx.line_number, + None => self.line_number.saturating_sub(1), + }; let bp = &self.branch_points[bp_index]; // Check validity: not >128 lines old @@ -993,8 +1330,15 @@ impl ParseState { return Ok(false); } - // Check validity: stack depth still >= branch's stack_depth - if self.stack.len() < bp.stack_depth { + // Check validity: the alt frame is still on the stack. Mirrors the + // bp lookup predicate above (`stack_len > bp.stack_depth.saturating_sub(bp.pop_count)`) + // so a `pop: N + branch_point` whose snapshot captures the + // pre-pop depth doesn't false-positive here. Without subtracting + // `pop_count`, Java's `pop: 2 + branch_point: annotation-qualified-parameters` + // failed lookup with `self.stack.len() < bp.stack_depth` and the + // fail became a no-op, leaking `meta.annotation.identifier.java` + // into every nested-annotation extends path. + if self.stack.len() <= bp.stack_depth.saturating_sub(bp.pop_count) { self.branch_points.remove(bp_index); return Ok(false); } @@ -1026,6 +1370,7 @@ impl ParseState { // stack with `meta.type.js, meta.group.js` — 274 cascading // assertion failures in `syntax_test_typescript.ts`. let is_cross_line = bp.line_number < cur_line; + let bp_line_number = bp.line_number; let stack_snapshot = bp.stack_snapshot.clone(); let proto_starts_snapshot = bp.proto_starts_snapshot.clone(); let escape_stack_snapshot = bp.escape_stack_snapshot.clone(); @@ -1035,6 +1380,11 @@ impl ParseState { let match_start_pos = bp.match_start; let pending_lines_snapshot_len = bp.pending_lines_snapshot_len; let prefix_ops = bp.prefix_ops.clone(); + let outer_bp_info = BpInfo { + name: bp.name.clone(), + stack_depth: bp.stack_depth, + line_number: bp.line_number, + }; self.branch_points.remove(bp_index); self.stack = stack_snapshot; @@ -1060,11 +1410,31 @@ impl ParseState { // buffered lines. let truncated_lines: Vec = self.pending_lines[pending_lines_snapshot_len..].to_vec(); + // Save prior flushed_ops state and clear it so any nested + // cross-line fails firing during the replay loop below + // write into a clean slot we can detect afterward. + let saved_flushed = std::mem::take(&mut self.flushed_ops); + let saved_flushed_start = self.flushed_ops_start.take(); + let saved_flushed_bp = self.flushed_ops_bp.take(); let mut replayed_ops: Vec> = Vec::with_capacity(truncated_lines.len()); for (i, replay_line) in truncated_lines.iter().enumerate() { - let line_ops = if i == 0 { - let mut first_line_ops = prefix_ops.clone(); + // Tag branches created during this iteration with + // the replay line's identity, not the outer line's. + let prev_replay_ctx = self.replay_ctx.replace(ReplayCtx { + line_number: bp_line_number + i, + pending_lines_snapshot_offset: pending_lines_snapshot_len + i, + }); + // No new-alt construction here (all alternatives + // exhausted), so the first-line prefix is just + // `prefix_ops`. Surface it to inner branch creations + // so their `prefix_ops` keeps the outer captures. + let prev_replay_prefix = if i == 0 { + self.replay_prefix_ops.replace(prefix_ops.clone()) + } else { + self.replay_prefix_ops.take() + }; + let inner_result = if i == 0 { let resume_at = if let Some((j, _)) = replay_line[match_start_pos..].char_indices().nth(1) { @@ -1072,16 +1442,47 @@ impl ParseState { } else { replay_line.len() }; - let tail_ops = - self.parse_line_inner_from(replay_line, syntax_set, resume_at)?; + self.parse_line_inner_from(replay_line, syntax_set, resume_at) + } else { + self.parse_line_inner(replay_line, syntax_set) + }; + self.replay_ctx = prev_replay_ctx; + self.replay_prefix_ops = prev_replay_prefix; + let tail_ops = inner_result?; + let line_ops = if i == 0 { + let mut first_line_ops = prefix_ops.clone(); first_line_ops.extend(tail_ops); first_line_ops } else { - self.parse_line_inner(replay_line, syntax_set)? + tail_ops }; replayed_ops.push(line_ops); } - self.flushed_ops.extend(replayed_ops); + // Capture inner corrections (if any), restore prior state, + // then prefer the inner corrections for overlapping indices. + let inner_corrections = std::mem::take(&mut self.flushed_ops); + let inner_corrections_start = self.flushed_ops_start.take(); + let inner_corrections_bp = self.flushed_ops_bp.take(); + self.flushed_ops = saved_flushed; + self.flushed_ops_start = saved_flushed_start; + self.flushed_ops_bp = saved_flushed_bp; + if let Some(start) = inner_corrections_start { + if !inner_corrections.is_empty() { + Self::prefer_inner_replay_corrections( + pending_lines_snapshot_len, + &mut replayed_ops, + &inner_corrections, + start, + &outer_bp_info, + inner_corrections_bp.as_ref(), + ); + } + } + self.merge_flushed( + pending_lines_snapshot_len, + replayed_ops, + outer_bp_info.clone(), + ); // Restart the current line from the beginning under the // restored state. @@ -1092,14 +1493,21 @@ impl ParseState { return Ok(true); } - // Same-line exhaustion: advance one char past the branch_point match - // to avoid immediately re-matching the same `(?=...)` lookahead. - if let Some((i, _)) = line[match_start_pos..].char_indices().nth(1) { - *start = match_start_pos + i; - } else { - // End of line — no character to advance past. - *start = line.len(); - } + // Same-line exhaustion: rewind the cursor to the BP's + // original position and record the branch_point's name so + // subsequent `find_best_match` calls at that position skip + // the same-name Branch pattern. This lets the parent + // context's NEXT rule fire instead of advancing past the + // lookahead match — mirrors ST's branch-point exhaustion + // semantics. The previous behaviour (advance one char) let + // stale keyword rules match in the middle of identifiers, + // e.g. `package` inside `$package` after `declarations` + // exhausted on the leading `$`. If `find_best_match` + // returns nothing at this cursor, `parse_next_token`'s + // no-match fallback advances one char as a last resort. + self.skipped_branches + .push((match_start_pos, name.to_string())); + *start = match_start_pos; search_cache.clear(); return Ok(true); } @@ -1108,11 +1516,13 @@ impl ParseState { let is_cross_line = bp.line_number < cur_line; // Extract everything we need from bp before mutating self. + let bp_line_number = bp.line_number; let next_alt_index = bp.next_alternative; let next_alt = bp.alternatives[next_alt_index].clone(); let match_start_pos = bp.match_start; let trigger_match_start = bp.trigger_match_start; let trigger_pat_scope = bp.pat_scope.clone(); + let trigger_capture_ops = bp.capture_ops.clone(); let stack_snapshot = bp.stack_snapshot.clone(); let proto_starts_snapshot = bp.proto_starts_snapshot.clone(); let first_line_snapshot = bp.first_line_snapshot; @@ -1121,12 +1531,20 @@ impl ParseState { let pending_lines_snapshot_len = bp.pending_lines_snapshot_len; let escape_stack_snapshot = bp.escape_stack_snapshot.clone(); let prefix_ops = bp.prefix_ops.clone(); + let outer_bp_info = BpInfo { + name: bp.name.clone(), + stack_depth: bp.stack_depth, + line_number: bp.line_number, + }; // bp borrow ends here. let pop_count = self.branch_points[bp_index].pop_count; // Restore parser state to the snapshot. - self.stack = stack_snapshot; + // Keep `stack_snapshot` available — the same-line fix below needs + // it to compute popped-context meta_scope clearance via + // `push_meta_ops`. + self.stack = stack_snapshot.clone(); self.proto_starts = proto_starts_snapshot; self.escape_stack = escape_stack_snapshot; self.first_line = first_line_snapshot; @@ -1146,7 +1564,6 @@ impl ParseState { // Push the next alternative onto the stack. let with_prototype = self.branch_points[bp_index].with_prototype.clone(); let context_id = next_alt.id()?; - let context = syntax_set.get_context(&context_id)?; let captures = None; // no captures available at rewind time let proto_ids = match with_prototype { @@ -1185,50 +1602,150 @@ impl ParseState { let truncated_lines: Vec = self.pending_lines[pending_lines_snapshot_len..].to_vec(); + // Compose the first replayed line's prefix (outer prefix_ops + + // new-alt meta/pat/capture/meta_content emission) up front so a + // branch_point born inside the inner re-parse can inherit it + // via `self.replay_prefix_ops`. Built once per fail; cloned + // and extended with `tail_ops` to form the final line_ops. + // + // Use `push_meta_ops` with a synthetic Set/Push for the + // new alternative — same path the same-line fail above and + // the original branch creation take — so both the new + // alternative's own meta scopes AND the popped contexts' + // meta_scope/mcs clearance Pop (for `pop: N + branch_point`, + // N > 0) get re-emitted. A bespoke re-emit of just + // `context.meta_scope` / `context.meta_content_scope` is + // missing the popped-contexts Pop, leaving Java's + // `pop: 2 + branch_point: annotation-qualified-parameters` + // crossing a line boundary with both the popped context's + // meta_scope (`meta.annotation.identifier.java`) AND the + // outer declaration's meta_scope (`meta.enum.java` / + // `meta.class.java` / `meta.interface.java`) leaked on the + // stack — cascading across 8000+ lines past the multi-line + // annotation-modified declaration at lines 2260-2297 of + // `syntax_test_java.java`. + // + // `push_meta_ops` reads `self.stack` to compute the popped + // contexts' scope atoms, so swap in `stack_snapshot` (pre-pop + // state captured at branch creation) for the duration of the + // calls — `self.stack` currently holds the post-set state + // (alt N already pushed). + let mut first_line_prefix = prefix_ops.clone(); + let synthetic_op_alt_n = if pop_count > 0 { + MatchOperation::Set { + ctx_refs: vec![next_alt.clone()], + pop_count, + } + } else { + MatchOperation::Push(vec![next_alt.clone()]) + }; + let level_ctx_id = stack_snapshot.last().map(|l| l.context); + let post_set_stack = std::mem::replace(&mut self.stack, stack_snapshot.clone()); + if let Some(level_ctx_id) = level_ctx_id { + let level_context = syntax_set.get_context(&level_ctx_id)?; + self.push_meta_ops( + true, + trigger_match_start, + level_context, + &synthetic_op_alt_n, + syntax_set, + &mut first_line_prefix, + )?; + for scope in &trigger_pat_scope { + first_line_prefix.push((trigger_match_start, ScopeStackOp::Push(*scope))); + } + first_line_prefix.extend(trigger_capture_ops.iter().cloned()); + if !trigger_pat_scope.is_empty() { + first_line_prefix + .push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len()))); + } + self.push_meta_ops( + false, + match_start_pos, + level_context, + &synthetic_op_alt_n, + syntax_set, + &mut first_line_prefix, + )?; + } + self.stack = post_set_stack; + + // Save prior flushed_ops state and clear it so any nested + // cross-line fails firing during the replay loop below write + // into a clean slot we can detect afterward. + let saved_flushed = std::mem::take(&mut self.flushed_ops); + let saved_flushed_start = self.flushed_ops_start.take(); + let saved_flushed_bp = self.flushed_ops_bp.take(); + let mut replayed_ops: Vec> = Vec::with_capacity(truncated_lines.len()); for (i, replay_line) in truncated_lines.iter().enumerate() { + // Tag branches created during this iteration with the + // replay line's identity, not the outer line's. + let prev_replay_ctx = self.replay_ctx.replace(ReplayCtx { + line_number: bp_line_number + i, + pending_lines_snapshot_offset: pending_lines_snapshot_len + i, + }); + // Expose the first-line prefix so a branch_point born + // during this line's re-parse anchors its `prefix_ops` + // to the full line state — outer captures included. + // Subsequent replayed lines start fresh. + let prev_replay_prefix = if i == 0 { + self.replay_prefix_ops.replace(first_line_prefix.clone()) + } else { + self.replay_prefix_ops.take() + }; + let inner_result = if i == 0 { + self.parse_line_inner_from(replay_line, syntax_set, match_start_pos) + } else { + self.parse_line_inner(replay_line, syntax_set) + }; + self.replay_ctx = prev_replay_ctx; + self.replay_prefix_ops = prev_replay_prefix; + let tail_ops = inner_result?; let line_ops = if i == 0 { - // First buffered line: compose prefix + branch ops + resume. - let mut first_line_ops = prefix_ops.clone(); - // Re-emit the trigger's pat.scope and the new - // alternative's meta scope ops in the same order - // the non-fail push path uses: clear_scopes and - // meta_scope at `trigger_match_start` (so the - // matched text sees them), then pat.scope at the - // same position, popped at `match_start_pos`. - // meta_content_scope only applies after the - // matched text, so it lands at `match_start_pos`. - if let Some(clear_amount) = context.clear_scopes { - first_line_ops - .push((trigger_match_start, ScopeStackOp::Clear(clear_amount))); - } - for scope in context.meta_scope.iter() { - first_line_ops.push((trigger_match_start, ScopeStackOp::Push(*scope))); - } - for scope in &trigger_pat_scope { - first_line_ops.push((trigger_match_start, ScopeStackOp::Push(*scope))); - } - if !trigger_pat_scope.is_empty() { - first_line_ops - .push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len()))); - } - for scope in context.meta_content_scope.iter() { - first_line_ops.push((match_start_pos, ScopeStackOp::Push(*scope))); - } - // Resume parsing from the branch match's end position. - let tail_ops = - self.parse_line_inner_from(replay_line, syntax_set, match_start_pos)?; + let mut first_line_ops = first_line_prefix.clone(); first_line_ops.extend(tail_ops); first_line_ops } else { - self.parse_line_inner(replay_line, syntax_set)? + tail_ops }; replayed_ops.push(line_ops); } - // Append (rather than overwrite) in case multiple cross-line fails - // fire on the same parse_line call. - self.flushed_ops.extend(replayed_ops); + // Capture inner corrections (if any), restore prior state, then + // prefer the inner corrections for overlapping indices. Without + // this, the outer's locally-computed `replayed_ops[i]` for + // indices an inner cross-line fail later corrected (during a + // later iteration of this same loop) would silently overwrite + // the inner's more accurate correction in `flushed_ops` — + // observed on Java's `@A.B\n(par=1)\nenum E {}` where the + // outer `declarations` cross-line replay's line-1 ops froze the + // dotted annotation as `path` alt before the inner + // `annotation-qualified-identifier` cross-line fail's `name`-alt + // resolution arrived (during line-2 reparse). + let inner_corrections = std::mem::take(&mut self.flushed_ops); + let inner_corrections_start = self.flushed_ops_start.take(); + let inner_corrections_bp = self.flushed_ops_bp.take(); + self.flushed_ops = saved_flushed; + self.flushed_ops_start = saved_flushed_start; + self.flushed_ops_bp = saved_flushed_bp; + if let Some(start) = inner_corrections_start { + if !inner_corrections.is_empty() { + Self::prefer_inner_replay_corrections( + pending_lines_snapshot_len, + &mut replayed_ops, + &inner_corrections, + start, + &outer_bp_info, + inner_corrections_bp.as_ref(), + ); + } + } + self.merge_flushed( + pending_lines_snapshot_len, + replayed_ops, + outer_bp_info.clone(), + ); // Restart the current line from the beginning. ops.clear(); @@ -1272,32 +1789,67 @@ impl ParseState { // because the original Push/Pop pair was truncated off // `ops` together with alt[0]'s subsequent work. // - // The new alternative's `clear_scopes` and `meta_scope` - // are emitted at `trigger_match_start` *before* the - // trigger's `pat.scope`, mirroring the non-fail push path - // in `push_meta_ops` (initial phase): meta_scope sits - // below the match scope on the stack so the matched text - // sees both. Placing them at `match_start_pos` would mean - // the trigger character (e.g. `(` of `for (var i = 0; …)`) - // never sees the alternative's `meta_scope`. The - // `meta_content_scope` legitimately stays at - // `match_start_pos` — mcs only applies after the matched - // text. - if let Some(clear_amount) = context.clear_scopes { - ops.push((trigger_match_start, ScopeStackOp::Clear(clear_amount))); - } - for scope in context.meta_scope.iter() { - ops.push((trigger_match_start, ScopeStackOp::Push(*scope))); - } - for scope in &trigger_pat_scope { - ops.push((trigger_match_start, ScopeStackOp::Push(*scope))); - } - if !trigger_pat_scope.is_empty() { - ops.push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len()))); - } - for scope in context.meta_content_scope.iter() { - ops.push((match_start_pos, ScopeStackOp::Push(*scope))); + // Use `push_meta_ops` with a synthetic Set/Push for the + // new alternative — same path the original branch creation + // takes — so both the new alternative's own meta scopes + // AND the popped contexts' meta_scope/mcs clearance Pop + // (for `pop: N + branch_point`, N > 0) get re-emitted. + // A bespoke re-emit of just `context.meta_scope` / + // `context.meta_content_scope` was missing the + // popped-contexts Pop, leaving Java's + // `pop: 2 + branch_point: annotation-qualified-parameters` + // with `meta.annotation.identifier.java meta.path.java` + // (annotation-qualified-identifier's `meta_scope`) leaked + // on the stack after the branch_point's first alt failed + // and the second alt (`immediately-pop`) ran. + // + // `push_meta_ops` reads `self.stack` to compute the + // popped contexts' scope atoms, so swap in `stack_snapshot` + // (pre-pop state captured at branch creation) for the + // duration of the calls — `self.stack` currently holds the + // post-set state (alt N already pushed). + let synthetic_op_alt_n = if pop_count > 0 { + MatchOperation::Set { + ctx_refs: vec![next_alt.clone()], + pop_count, + } + } else { + MatchOperation::Push(vec![next_alt.clone()]) + }; + let level_ctx_id = stack_snapshot.last().map(|l| l.context); + let post_set_stack = std::mem::replace(&mut self.stack, stack_snapshot.clone()); + if let Some(level_ctx_id) = level_ctx_id { + let level_context = syntax_set.get_context(&level_ctx_id)?; + self.push_meta_ops( + true, + trigger_match_start, + level_context, + &synthetic_op_alt_n, + syntax_set, + ops, + )?; + for scope in &trigger_pat_scope { + ops.push((trigger_match_start, ScopeStackOp::Push(*scope))); + } + // Captures emitted alongside the original pat.scope (e.g. + // `keyword.declaration.data.haskell` on the first capture of + // `(data)(?:\s+(family|instance))?`) were truncated off with + // alt[0]'s ops. Re-emit them inside the pat_scope brackets so + // the keyword scope survives the branch swap. + ops.extend(trigger_capture_ops.iter().cloned()); + if !trigger_pat_scope.is_empty() { + ops.push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len()))); + } + self.push_meta_ops( + false, + match_start_pos, + level_context, + &synthetic_op_alt_n, + syntax_set, + ops, + )?; } + self.stack = post_set_stack; } // Clear search cache since we're rewinding. @@ -1376,19 +1928,29 @@ impl ParseState { if !cur_context.meta_scope.is_empty() { ops.push((index, ScopeStackOp::Pop(cur_context.meta_scope.len()))); } + // Restore cur_context's `clear_scopes` BEFORE + // popping deeper contexts. The Clear hid the + // deeper context's meta_scope/mcs atoms; with the + // Restore deferred to the very end, the + // depth-loop's `Pop(deep.meta_scope.len())` would + // pop visible-stack scopes that don't belong to + // the deeper context — observed on Java's + // `case DayType when -> "incomplete"`, where + // `case-label-expression`'s `clear_scopes: 1` + // cleared `case-label`'s `meta.case.java` and + // `case-label-end`'s `pop: 2` then popped the + // surrounding `meta.block.java` (switch's block) + // off the consumer's stack instead. + if cur_context.clear_scopes.is_some() { + ops.push((index, ScopeStackOp::Restore)) + } // Each deeper context's scopes are popped in // top-to-bottom order: meta_content_scope first // (pushed after its own meta_scope, hence above on // the stack), then meta_scope, then any Restore // paired with that context's own `clear_scopes` // (mirrors the push order Clear → meta_scope → mcs - // in reverse). Without the Restore, a `pop: N` - // from the top that also unwinds a deeper context - // with `clear_scopes` leaves the originally cleared - // atoms orphaned — observed on Python f/t-string - // interpolation close, where `f-string-replacement-end` - // fires `pop: 2` that also pops - // `f-string-replacement-meta`. + // in reverse). for depth in 1..pop_count { let level_idx = stack_len - 1 - depth; let ctx = syntax_set.get_context(&self.stack[level_idx].context)?; @@ -1409,11 +1971,6 @@ impl ParseState { } } } - - // cleared scopes are restored after the scopes from match pattern that invoked the pop are applied - if !initial && cur_context.clear_scopes.is_some() { - ops.push((index, ScopeStackOp::Restore)) - } } // for some reason the ST3 behaviour of set is convoluted and is inconsistent with the docs and other ops // - the meta_content_scope of the current context is applied to the matched thing, unlike pop @@ -1431,8 +1988,27 @@ impl ParseState { }; // a match pattern that "set"s keeps the meta_content_scope and meta_scope from the previous context if initial { - // v2: pop parent's meta_content_scope so matched text does not see it - if is_set && version >= 2 && !cur_context.meta_content_scope.is_empty() { + // v2: pop parent's meta_content_scope so matched text does not see it. + // Skip when cur_context's mcs was never pushed because the + // context immediately below has embed_scope_replaces (the + // embedded syntax's main mcs is suppressed in favor of + // embed_scope on the wrapper). Without this, the Pop takes + // off the topmost wrapper-pushed scope — observed on + // Markdown bash fenced blocks where `source.shell.bash` + // (the third embed_scope token) was disappearing on the + // embedded main's first `set:` rule. + let stack_len = self.stack.len(); + let skip_cur_mcs_pop = version >= 2 + && stack_len >= 2 + && syntax_set + .get_context(&self.stack[stack_len - 2].context) + .map(|c| c.embed_scope_replaces) + .unwrap_or(false); + if is_set + && version >= 2 + && !cur_context.meta_content_scope.is_empty() + && !skip_cur_mcs_pop + { ops.push(( index, ScopeStackOp::Pop(cur_context.meta_content_scope.len()), @@ -1567,6 +2143,21 @@ impl ParseState { } } + // `pop: N + set:` with clear_scopes on the leaving + // context: restore the cleared atoms BEFORE the + // compound Pop so num_to_pop finds the popped frames' + // full meta_content_scope on the scope stack. Without + // this, Pop eats atoms from below the popped range — + // observed on Batch File `cmd-set-quoted-value-inner-end` + // (`clear_scopes: 1`) firing `pop: 2, set: ignored-tail-outer`, + // which otherwise drops `meta.command.set.dosbatch` from + // the trailing content of every `set "var"=...` line. + let restore_before_pop = + is_set && set_pop_count > 1 && cur_context.clear_scopes.is_some(); + if restore_before_pop { + ops.push((index, ScopeStackOp::Restore)); + } + // do all the popping as one operation if num_to_pop > 0 { ops.push((index, ScopeStackOp::Pop(num_to_pop))); @@ -1576,7 +2167,7 @@ impl ParseState { // cur.meta_scope and the initial phase's target.meta_scope // push have been popped off. The restored atoms land below // the target's upcoming meta_scope / meta_content_scope push. - if is_set && cur_context.clear_scopes.is_some() { + if is_set && cur_context.clear_scopes.is_some() && !restore_before_pop { ops.push((index, ScopeStackOp::Restore)); } @@ -1662,6 +2253,50 @@ impl ParseState { } else { MatchOperation::Push(contexts.clone()) }; + if pop_count > 0 { + // ST-observed divergence from plain `pop + set:`: on + // `pop + embed:` the trigger match's text sees **neither** + // `cur_context.meta_scope` nor `cur_context.meta_content_scope`. + // Both are suppressed on match text, then never restored + // — the embed replaces cur entirely. The probe lives at + // the top of `v2_pop_embed_suppresses_cur_meta_scope_on_match`. + // + // Emit those Pops ourselves in the initial phase, then pass + // a scope-stripped cur_context through to the recursive + // Set-semantic logic so its `num_to_pop` in the non-initial + // phase does not double-count these atoms (they are already + // off the stack). clear_scopes, with_prototype, and other + // fields are preserved on the stripped context — only the + // meta-scope vectors differ. + // + // Observed divergence on ``'s `>`: + // syntect was producing + // [..., meta.tag.jsp.declaration.begin.html, + // meta.tag.jsp.declaration.begin.html, + // punctuation.definition.tag.end.html] + // because the rule's explicit + // scope: meta.tag.jsp.declaration.begin.html + // punctuation.definition.tag.end.html + // was re-adding the atom that ST drops through the embed. + if initial { + if !cur_context.meta_content_scope.is_empty() { + ops.push(( + index, + ScopeStackOp::Pop(cur_context.meta_content_scope.len()), + )); + } + if !cur_context.meta_scope.is_empty() { + ops.push((index, ScopeStackOp::Pop(cur_context.meta_scope.len()))); + } + } + let stripped = Context { + meta_scope: Vec::new(), + meta_content_scope: Vec::new(), + ..cur_context.clone() + }; + return self + .push_meta_ops(initial, index, &stripped, &synthetic, syntax_set, ops); + } return self.push_meta_ops( initial, index, @@ -1721,10 +2356,10 @@ impl ParseState { for _ in 0..pop_count { self.stack.pop(); } + let stack_len = self.stack.len(); self.branch_points - .retain(|bp| bp.stack_depth <= self.stack.len()); - self.escape_stack - .retain(|e| e.stack_depth < self.stack.len()); + .retain(|bp| stack_len > bp.stack_depth.saturating_sub(bp.pop_count)); + self.escape_stack.retain(|e| e.stack_depth < stack_len); } (contexts, None, true) } @@ -1743,11 +2378,25 @@ impl ParseState { self.stack.pop(); } // Prune branch_points / escape_stack against the *final* stack - // length (after the common push loop below), so a branch_point - // captured at the pre-set depth survives a pop-1 + push-1 set - // — the depth the bp references is still valid after the push. + // length (after the common push loop below). + // + // The retain predicate must mirror `handle_fail`'s validity + // check (`stack.len() > bp.stack_depth - bp.pop_count`), + // which subtracts the bp's own `pop_count`. Without that + // subtraction, a `pop: N + branch_point` whose synthetic + // Set has `pop_count: N` removes its own freshly-created + // bp here — `bp.stack_depth` snapshots the *pre-pop* + // depth, so `bp.stack_depth > final_len` even though the + // alt-0 frame lives on at `final_len`. Symptom in Java: + // the `branch_point: annotation-qualified-parameters` + // declared on `annotation-qualified-identifier-name`'s + // `pop: 2 + branch_point` was dropped at creation, + // making its later `(?=\S)` `fail` a no-op and leaking + // `meta.annotation.identifier.java meta.path.java` past + // every nested-annotation extends path. let final_len = self.stack.len() + ctx_refs.len(); - self.branch_points.retain(|bp| bp.stack_depth <= final_len); + self.branch_points + .retain(|bp| final_len > bp.stack_depth.saturating_sub(bp.pop_count)); self.escape_stack.retain(|e| e.stack_depth < final_len); (ctx_refs, old_proto_ids, false) } @@ -1755,12 +2404,14 @@ impl ParseState { for _ in 0..n { self.stack.pop(); } - // Invalidate branch points whose stack depth is now above current stack + // Invalidate branch points whose alt frame is no longer on + // the stack. Use the same threshold as `handle_fail`'s + // validity check — see the comment in the Set arm above. + let stack_len = self.stack.len(); self.branch_points - .retain(|bp| bp.stack_depth <= self.stack.len()); + .retain(|bp| stack_len > bp.stack_depth.saturating_sub(bp.pop_count)); // Remove escape entries whose stack_depth >= current stack - self.escape_stack - .retain(|e| e.stack_depth < self.stack.len()); + self.escape_stack.retain(|e| e.stack_depth < stack_len); return Ok(true); } MatchOperation::None => return Ok(false), @@ -1853,6 +2504,47 @@ impl ParseState { let target_depth = entry.stack_depth; let escape_captures = entry.captures.clone(); + // Drain orphan scope atoms left on the consumer's scope stack by + // a prior cross-line replay whose later same-line fails + // truncated the owning context out of `self.stack` — the Push + // was committed to `flushed_ops` and can't be unwound by + // `ops.truncate`, so we emit a balancing Pop here. Without this, + // e.g. LaTeX `\end{lstlisting}` leaves + // `meta.environment.verbatim.lstlisting.latex` on the stack + // because a speculative `meta.path.java` atom pushed inside the + // embedded Java shifts every subsequent Pop by one. + // + // `shadow` mirrors what the consumer will actually hold at this + // point: end-of-prior-line shadow + ops-so-far on the current + // line. `expected_depth` is what the consumer *should* have + // based on `self.stack`'s meta_scope / meta_content_scope + // contributions (with the v2 `embed_scope_replaces` mcs gating + // applied below, matching the pop loop). + let mut current_shadow = self.shadow.clone(); + for (_, op) in ops.iter() { + let _ = current_shadow.apply(op); + } + let consumer_depth = current_shadow.as_slice().len(); + let expected_depth: usize = { + let mut total = 0usize; + let mut prev_embed_scope_replaces = false; + for lvl in &self.stack { + let ctx = syntax_set.get_context(&lvl.context)?; + total += ctx.meta_scope.len(); + if !prev_embed_scope_replaces { + total += ctx.meta_content_scope.len(); + } + prev_embed_scope_replaces = ctx.embed_scope_replaces; + } + total + }; + if consumer_depth > expected_depth { + ops.push(( + match_start, + ScopeStackOp::Pop(consumer_depth - expected_depth), + )); + } + // Pop all stack levels down to target_depth, emitting proper meta scope pops while self.stack.len() > target_depth { let level = &self.stack[self.stack.len() - 1]; @@ -3495,6 +4187,51 @@ contexts: ); } + /// Regression guard for "branch_point fail-retry drops the + /// trigger match's `captures:` scopes". The non-fail path emits + /// capture Push/Pop ops inside the pat_scope brackets; the + /// same-line fail re-emit must do the same — otherwise the + /// inner capture scopes are truncated off `ops` together with + /// alt[0]'s subsequent work and never replayed. Observed on + /// Haskell's `data CtxCls ctx => ModId.QTyCls`, where the + /// `(data)(?:\s+(family|instance))?` branch_point match's first + /// capture `keyword.declaration.data.haskell` was dropped from + /// the `data` token whenever `data-signature` failed into + /// `data-context` — 22 assertion failures in + /// `syntax_test_haskell.hs`. + #[test] + fn branch_point_capture_scopes_survive_fail_retry() { + // The `(word)\s` branch_point match carries both `scope:` + // and `captures:`. Alt[0] fails on the `!` lookahead, + // forcing replay into alt[1]. `inner.capture` on the first + // capture group must remain on the stack over `word`. + expect_scope_stacks( + "word !", + &[", "], + r#" + name: Branch Capture Re-emit Test + scope: source.test + contexts: + main: + - match: (word)\s + scope: outer.match + captures: + 1: inner.capture + branch_point: bp + branch: + - alt-fails + - alt-succeeds + alt-fails: + - match: (?=!) + fail: bp + alt-succeeds: + - match: \S+ + scope: ok.test + pop: 1 + "#, + ); + } + /// Regression guard for "branch_point fail-retry drops the new /// alternative's `meta_scope` from the trigger character". The /// non-fail push path emits the new context's `meta_scope` at @@ -3759,7 +4496,6 @@ contexts: /// correct ops for the failing file (the replay-consistency issue /// is tracked as a follow-up). #[test] - #[ignore = "requires testdata/Packages submodule"] fn cross_line_fail_with_nested_branch_does_not_panic() { use crate::parsing::SyntaxSet; use std::panic::AssertUnwindSafe; @@ -3860,12 +4596,8 @@ contexts: /// harness-level cause: loads the real Packages Makefile syntax /// and parses two lines, asserting that after `bar := $(foo)\n` /// the scope stack no longer carries `meta.string.makefile` when - /// the next source line is parsed. Gated on the test-assets - /// being available; marked `#[ignore]` so it runs with - /// `cargo test -- --ignored` in the repo root (the Packages - /// submodule is required). + /// the next source line is parsed. #[test] - #[ignore = "requires testdata/Packages submodule"] fn makefile_meta_string_does_not_leak_past_eol() { use crate::parsing::SyntaxSet; let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); @@ -5136,6 +5868,75 @@ contexts: ); } + #[test] + fn captures_clipped_to_match_bounds_when_group_extends_past_match_end() { + // Repro of a C# generic-function-call divergence against ST. + // Rule shape: a consumed identifier, then a lookahead containing + // a capturing group whose match extends *past* the outer rule's + // consumed end, then a second consumed group starting at the + // same column where the lookahead began. `captures: 2:` targets + // the lookahead-internal group. ST clips each captures:N span + // to the rule's match bounds and only colours the overlap — + // which here is the single consumed char at the match-end + // boundary. Syntect used to colour the full group-2 range, + // emitting a Pop past match_end and leaving the scope active + // over chars the match never consumed. + let syntax_str = r#" +name: CapturesClip +scope: source.capclip +contexts: + main: + - match: '(foo)(?=(barrr)baz)(bar)' + captures: + 1: captured-foo.capclip + 2: lookahead-group.capclip + 3: consumed-bar.capclip +"#; + let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap(); + let ss = link(syntax); + let mut state = ParseState::new(&ss.syntaxes()[0]); + let raw_ops = ops(&mut state, "foobarrrbaz\n", &ss); + + // The rule consumes "foobar" — match_start=0, match_end=6. + // Group 2's own range (the lookahead match "barrr") extends to + // column 8. Every op emitted by the captures application must + // sit within [match_start, match_end]; anything at col 7+ means + // the lookahead-internal group's span leaked past the match. + // match_start=0, match_end=6 (rule consumes "foobar"). Group 2's + // own range (the lookahead match "barrr") extends to col 8. + // + // After the fix we expect: + // * `lookahead-group.capclip` Pushed at col 3 (cap_start of + // group 2, which overlaps the consumed region). + // * The matching Pop no later than col 6 (clipped to match_end). + // * No op at col 7 or 8 — anything there means the lookahead + // range leaked past the match. + let lookahead_pushes: Vec = raw_ops + .iter() + .filter_map(|(pos, op)| match op { + ScopeStackOp::Push(s) if format!("{:?}", s).contains("lookahead-group") => { + Some(*pos) + } + _ => None, + }) + .collect(); + assert_eq!( + lookahead_pushes, + vec![3], + "`captures: 2:` (lookahead-internal group) must Push the \ + clipped scope at match_start=3; raw_ops={:?}", + raw_ops + ); + let match_end = 6; + let past_match: Vec<_> = raw_ops.iter().filter(|(pos, _)| *pos > match_end).collect(); + assert!( + past_match.is_empty(), + "No capture op should sit past match_end={}; found {:?}", + match_end, + past_match + ); + } + #[test] fn capture_sort_by_span_length() { // Kills: L709 replace - with + in exec_pattern (capture sort key) @@ -5404,6 +6205,85 @@ contexts: ); } + #[test] + fn pop_n_set_with_cur_clear_scopes_restores_before_popping_deeper_frames() { + // `pop: N + set: X` fired from a context that itself declares + // `clear_scopes` at the context level: the deeper popped frame's + // meta_content_scope is partly on the live scope stack and partly + // in `clear_stack` (stripped by cur's Clear on entry). Emitting the + // compound Pop before Restore makes Pop eat atoms from below the + // intended popped range, dropping the outer frame's meta_scope. + // Shape mirrors Batch File's `cmd-set-quoted-value-inner-end` + // (`clear_scopes: 1`) firing `pop: 2, set: ignored-tail-outer` below + // a `cmd-set-quoted-value-inner` that carries a 2-atom + // meta_content_scope. + let syntax_str = r#" +name: PopNSetClear +scope: source.popnsetclear +version: 2 +contexts: + main: + - match: 'a' + scope: p.a + push: [outer, middle] + + outer: + - meta_scope: outer.test + - match: 'z' + pop: 1 + + middle: + - meta_content_scope: mid1.test mid2.test + - match: 'b' + scope: p.b + push: top + + top: + - clear_scopes: 1 + - match: 'c' + scope: p.c + pop: 2 + set: target + + target: + - meta_scope: target.test + - match: 'd' + scope: p.d +"#; + let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap(); + let ss = link(syntax); + let mut state = ParseState::new(&ss.syntaxes()[0]); + let raw_ops = ops(&mut state, "abcd", &ss); + let states = stack_states(raw_ops); + + // The scope stack state recorded on the `d` token (in `target`): + // must contain `outer.test` (outer's meta_scope still on the stack) + // and `target.test` (target's meta_scope, pushed by the pop:2+set:) + // and must NOT contain `mid1.test` or `mid2.test` (middle was popped + // by pop:2 and its meta_content_scope atoms — one live, one in + // clear_stack — should both be gone). + let d_state = states + .iter() + .find(|s| s.contains("p.d")) + .unwrap_or_else(|| panic!("expected a state containing `p.d`, got: {:?}", states)); + assert!( + d_state.contains("outer.test"), + "outer.test must survive pop:2+set: (it sits below the popped range): {}", + d_state + ); + assert!( + d_state.contains("target.test"), + "target.test must be on the stack (target was pushed by set:): {}", + d_state + ); + assert!( + !d_state.contains("mid1.test") && !d_state.contains("mid2.test"), + "middle's meta_content_scope atoms must not linger after pop:2+set:, \ + including the atom that was in clear_stack: {}", + d_state + ); + } + #[test] fn v2_set_clear_scopes_applies_from_every_context() { // v2: when `set:` lists multiple contexts, `clear_scopes` on any of @@ -5966,4 +6846,875 @@ contexts: ); } } + + #[test] + fn v2_pop_embed_suppresses_cur_meta_scope_on_match() { + // `pop: N + embed:` trigger text must NOT carry the popped context's + // `meta_scope` through, unlike `pop: N + set:` which preserves both + // cur's and target's meta_scope on the match. Probe against ST confirms: + // + // hi (pop+embed) + // col 4 '>' -> ['source.host', 'end.scope'] (cur ms gone) + // col 5 'h' (body) -> ['source.host', 'embed.scope', 'guest.meta'] + // + // after (pop+set — contrast) + // col 4 '>' -> ['source.host', 'meta.a', 'after.meta', 'end.scope'] + // + // Without this guard, syntect emitted + // [source.host, meta.a, meta.a, end.scope] + // because the rule's explicit scope atom shadowed cur.meta_scope onto + // itself on the trigger text — observed as 5 duplicated + // `meta.tag.jsp.*.begin.html` atoms on ``/ expression/ + // scriptlet's `>` in `syntax_test_jsp.jsp`. + let host = SyntaxDefinition::load_from_str( + r#" +name: PopEmbedHost +scope: source.popembed +file_extensions: [popembed] +version: 2 +contexts: + main: + - match: '' + scope: meta.a end.scope + pop: 1 + embed: scope:source.popembedguest + embed_scope: embed.scope + escape: '(?=hi\n", &ss).unwrap().ops; + + // Walk (range, op) pairs; after applying each op, snapshot the stack + // keyed by the character position we're at. The `>` match occupies + // col 4, so we expect the post-op snapshot at that position to have + // exactly ONE `meta.a` atom, not two. + use crate::easy::ScopeRangeIterator; + let line = "hi\n"; + let mut stack = ScopeStack::new(); + let mut at_gt: Option> = None; + for (range, op) in ScopeRangeIterator::new(&ops, line) { + stack.apply(op).expect("op stream must apply cleanly"); + // Capture the stack state for the character range covering the `>` + // trigger (col 4..5, the match text of the pop+embed rule). + if range.start <= 4 && 4 < range.end { + at_gt = Some( + stack + .as_slice() + .iter() + .map(|s| format!("{:?}", s)) + .collect(), + ); + } + } + let at_gt = at_gt.expect("range covering `>` must exist in op stream"); + let meta_a_count = at_gt.iter().filter(|s| s.contains("meta.a")).count(); + assert_eq!( + meta_a_count, 1, + "match text of pop+embed must carry exactly one `meta.a` atom \ + (from the rule's explicit scope); cur_context.meta_scope must \ + not stack a second copy on top. Got stack: {:?}", + at_gt + ); + // And `end.scope` must be the top of the stack (the match's second + // explicit atom) — if the ordering shifted we'd see a different trailer. + assert!( + at_gt + .last() + .map(|s| s.contains("end.scope")) + .unwrap_or(false), + "stack top on `>` must be `end.scope`, got: {:?}", + at_gt + ); + } + + #[test] + fn cross_line_multi_fail_deduplicates_flushed_ops() { + // Two nested branch_points created on line 1 that both fail on a + // later line exercise `handle_fail`'s cross-line path twice on a + // single `parse_line` call. Before dedup, each fail `extend`ed + // `flushed_ops` with its own replay, so `ParseLineOutput::replayed` + // ended up ~2× the pending-lines count — and the consumer (see + // `examples/syntest.rs`) paired `replayed[i]` with + // `parsed_line_buffer[buf_len - replayed.len() + i]`, sliding ops + // from one buffered line onto another's text. That panicked in + // `ScopeRegionIterator::next` as "byte index N out of bounds" — + // observed originally at `syntax_test_java.java` line 624. + let syntax_str = r#" +name: DedupCrossLine +scope: source.dup +contexts: + main: + - match: 'A' + branch_point: bp1 + branch: [a1, a2] + a1: + - match: 'B' + branch_point: bp2 + branch: [b1, b2] + - match: '(?=FAIL)' + fail: bp1 + a2: + - match: '.*' + scope: a2.fallback + pop: true + b1: + - match: '\n' + - match: '(?=FAIL)' + fail: bp2 + - match: 'XYZ' + pop: true + b2: + - match: '\n' + - match: '(?=FAIL)' + fail: bp2 + - match: 'XYZ' + pop: true +"#; + let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap(); + let ss = link(syntax); + let mut state = ParseState::new(&ss.syntaxes()[0]); + + let out1 = state.parse_line("AB\n", &ss).expect("line 1"); + assert!(out1.replayed.is_empty()); + + let out2 = state.parse_line("FOO\n", &ss).expect("line 2"); + assert!(out2.replayed.is_empty()); + + // Line 3 fires `fail: bp2` twice (once for alt[1], once to exhaust) + // and then `fail: bp1` — three cross-line fails back-to-back. + let out3 = state.parse_line("FAIL\n", &ss).expect("line 3"); + + // Invariant: one replayed entry per buffered pending line (2), not + // `number_of_fails × pending_lines`. + assert_eq!( + out3.replayed.len(), + 2, + "expected exactly 2 replayed lines (one per buffered pending line), got {}: {:?}", + out3.replayed.len(), + out3.replayed, + ); + + // Panic guard: each `replayed[i]`'s byte offsets must fit within the + // corresponding buffered line's length. The original misalignment + // paired line 617's ops (77 bytes) with line 609's text (59 bytes). + let line_lens = ["AB\n".len(), "FOO\n".len()]; + for (i, line_ops) in out3.replayed.iter().enumerate() { + for (pos, op) in line_ops { + assert!( + *pos <= line_lens[i], + "replayed[{}] op past EOL: pos={} line_len={} op={:?}", + i, + pos, + line_lens[i], + op, + ); + } + } + } + + #[test] + fn replay_born_branch_routes_as_cross_line_on_later_fail() { + // A branch created while `handle_fail` is re-parsing a past buffered + // line must record the *replay line's* number, not the outer + // `parse_line`'s current line. Otherwise `handle_fail`'s later + // `is_cross_line = bp.line_number < cur_line` sees equal values on + // the second fail, routes into the same-line path, and applies + // `bp.match_start` (a byte offset into the long replay line) to a + // shorter outer line. That shipped as the `byte index N out of + // bounds` panic on `syntax_test_java.java:10263` (` foo = BAR,\n`) + // and on `syntax_test_markdown.md` under multi-line math blocks. + let syntax_str = r#" +name: ReplayBornBranch +scope: source.rbb +contexts: + main: + - match: 'A' + branch_point: bp1 + branch: [a1, a2] + a1: + - match: '(?=FAIL1)' + fail: bp1 + - match: '.' + a2: + - match: 'B' + branch_point: bp2 + branch: [b1, b2] + - match: '.' + b1: + - match: '(?=FAIL2)' + fail: bp2 + - match: '.' + b2: + - match: '.*' + scope: b2.fallback + pop: true +"#; + let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap(); + let ss = link(syntax); + let mut state = ParseState::new(&ss.syntaxes()[0]); + + // Long line 1 so `B` sits past the short outer line's length; bp2's + // replay-relative `match_start` would OOB a same-line rewind there. + let line1 = "A pad pad pad pad pad pad pad pad pad pad B tail\n"; + assert!(line1.find('B').unwrap() > "FAIL2\n".len()); + + let out1 = state.parse_line(line1, &ss).expect("line 1 parses"); + assert!(out1.replayed.is_empty()); + + // First cross-line fail: swap bp1 → a2. During a2's replay of line 1, + // `B` fires bp2 and records (with the fix) `line_number = 0` and + // `pending_lines_snapshot_len = 0` — anchored to line 1, not line 2. + let out2 = state.parse_line("FAIL1\n", &ss).expect("line 2 parses"); + assert_eq!(out2.replayed.len(), 1, "bp1 replay covers line 1"); + + // Second cross-line fail: bp2 must be classified cross-line on this + // outer line. With the fix it is (line 0 < line 2), so the handler + // takes the replay path and re-parses past buffered lines under b2. + // Without the fix bp2 appears same-line (line 2 == line 2) and the + // handler applies `match_start` = offset-of-B-in-line1 to the + // 6-byte outer line, corrupting ops / panicking downstream. + let outer = "FAIL2\n"; + let out3 = state.parse_line(outer, &ss).expect("line 3 parses"); + + // Cross-line classification fired a second replay covering the + // two buffered lines (line 1 + line 2). + assert_eq!( + out3.replayed.len(), + 2, + "expected replay from bp2's cross-line fail to cover both buffered lines, got {}: {:?}", + out3.replayed.len(), + out3.replayed, + ); + + // Panic guard: every op offset in both `ops` and `replayed` must + // fit within its paired line's byte length. + for (pos, op) in &out3.ops { + assert!( + *pos <= outer.len(), + "outer op past EOL: pos={} len={} op={:?}", + pos, + outer.len(), + op, + ); + } + let replay_lines = [line1, "FAIL1\n"]; + for (i, line_ops) in out3.replayed.iter().enumerate() { + for (pos, op) in line_ops { + assert!( + *pos <= replay_lines[i].len(), + "replayed[{}] op past EOL: pos={} len={} op={:?}", + i, + pos, + replay_lines[i].len(), + op, + ); + } + } + } + + #[test] + fn replay_born_branch_inherits_outer_prefix_ops() { + // Branch born inside another branch's cross-line replay must + // record the outer replay's first-line prefix as part of its + // own `prefix_ops`. Otherwise its later cross-line fail + // reconstructs the replayed line from an empty prefix and the + // captures emitted before the *outer* branch trigger vanish. + // Shipped as `[foo]: /url` losing its + // `meta.link.reference.def.markdown` / `entity.name.reference` + // scopes in `syntax_test_markdown.md`: the line creates an + // outer `link-def-title-continuation` branch whose alt-1 + // (`immediately-pop2`) replay spawns a nested + // `link-def-attr-continuation` branch — when *that* branch + // fails on the next line its replay drops the original LRD + // opener captures. + let syntax_str = r#" +name: ReplayPrefix +scope: source.rp +contexts: + main: + - match: '(K)(EY)' + captures: + 1: keyword.k.rp + 2: variable.k.rp + push: outer + outer: + - match: '$' + branch_point: bp1 + branch: [a1, a2] + a1: + - meta_include_prototype: false + - match: '(?=FAIL1)' + fail: bp1 + - match: '.' + a2: + - meta_include_prototype: false + - match: '$' + branch_point: bp2 + branch: [b1, b2] + - match: '.' + b1: + - meta_include_prototype: false + - match: '(?=FAIL2)' + fail: bp2 + - match: '.' + b2: + - meta_include_prototype: false + - match: '\n' + scope: support.fallback.rp + pop: 2 +"#; + let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap(); + let ss = link(syntax); + let mut state = ParseState::new(&ss.syntaxes()[0]); + + // line 1 — captures `K` and `EY`, pushes `outer`, then `$` fires bp1. + let line1 = "KEY\n"; + let _ = state.parse_line(line1, &ss).expect("line 1 parses"); + + // line 2 — `(?=FAIL1)` in a1 trips bp1's cross-line fail. The + // alt-1 replay of line 1 spawns bp2 in a2 at end-of-line. + let line2 = "FAIL1\n"; + let _ = state.parse_line(line2, &ss).expect("line 2 parses"); + + // line 3 — `(?=FAIL2)` in b1 trips bp2's cross-line fail. With + // the fix bp2's `prefix_ops` carries the K / EY captures from + // bp1's replay, so the second cross-line replay re-emits them. + // Without the fix bp2's `prefix_ops` is empty and the replayed + // line 1 ops collapse to just `support.fallback.rp` push/pop. + let line3 = "FAIL2\n"; + let out3 = state.parse_line(line3, &ss).expect("line 3 parses"); + + // bp2's cross-line replay covered both buffered lines (line 1 + // + line 2). Line 1 is the one that must keep its captures. + assert_eq!( + out3.replayed.len(), + 2, + "bp2 cross-line replay should cover line1 + line2, got {:?}", + out3.replayed, + ); + let line1_ops = &out3.replayed[0]; + + let pushes_keyword = line1_ops.iter().any(|(_, op)| { + matches!(op, ScopeStackOp::Push(s) if *s == Scope::new("keyword.k.rp").unwrap()) + }); + let pushes_variable = line1_ops.iter().any(|(_, op)| { + matches!(op, ScopeStackOp::Push(s) if *s == Scope::new("variable.k.rp").unwrap()) + }); + assert!( + pushes_keyword, + "line 1 replayed ops should still push keyword.k.rp; got {:?}", + line1_ops, + ); + assert!( + pushes_variable, + "line 1 replayed ops should still push variable.k.rp; got {:?}", + line1_ops, + ); + } + + /// Two back-to-back link reference definitions followed by a + /// paragraph: each LRD's chain is closed by the *next* line's parse + /// emitting a `Pop` against the LRD's `meta_scope` via + /// `flushed_ops`. Without snapshot-drift correction, the + /// pre-correction `pending_line_start_shadows` (and the consumer's + /// `parsed_line_buffer[i].stack_before`) used by the second + /// replay's stack-reset still reflected the first LRD's leftover + /// `meta.link.reference.def.markdown` push, so the corrected line + /// 4 ops re-applied that scope onto a stale baseline — leaking it + /// into the paragraph and on through the rest of the file. + /// Sized 408 chars / 88 assertions in + /// `syntax_test_markdown.md`. + #[test] + fn back_to_back_lrds_clear_meta_scope_via_corrected_baseline() { + use crate::parsing::SyntaxSet; + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Markdown/Markdown.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + + // Mirror the syntest consumer's stack-tracking with the + // snapshot-drift correction the bug requires. + struct Record { + stack_before: ScopeStack, + } + let mut buffer: Vec = Vec::new(); + let mut stack = ScopeStack::new(); + + for line in ["[foo]: first\n", "[foo]: second\n", "bar\n"] { + let out = state.parse_line(line, &ss).expect("parse"); + if !out.replayed.is_empty() { + let buf_len = buffer.len(); + let start_idx = buf_len - out.replayed.len(); + stack = buffer[start_idx].stack_before.clone(); + let mut corrected: Vec<(usize, ScopeStack)> = Vec::new(); + for (i, replayed_ops) in out.replayed.iter().enumerate() { + for (_, op) in replayed_ops { + let _ = stack.apply(op); + } + let next_idx = start_idx + i + 1; + if next_idx < buf_len { + corrected.push((next_idx, stack.clone())); + } + } + for (idx, c) in corrected { + buffer[idx].stack_before = c; + } + } + let stack_before = stack.clone(); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + buffer.push(Record { stack_before }); + } + + let lrd = Scope::new("meta.link.reference.def.markdown").unwrap(); + let leaked = stack.as_slice().contains(&lrd); + assert!( + !leaked, + "meta.link.reference.def.markdown leaked past back-to-back \ + LRDs into 'bar' paragraph; consumer stack at end: {:?}", + stack, + ); + let shadow_leaked = state.shadow.as_slice().contains(&lrd); + assert!( + !shadow_leaked, + "syntect shadow disagrees with corrected consumer stack; \ + shadow at end: {:?}", + state.shadow, + ); + } + + #[cfg(feature = "default-onig")] + #[test] + fn pop_n_branch_point_keeps_bp_so_alt_fail_unwinds_meta_scope() { + // Real-syntax repro for the Java class-extends annotation leak: + // `class T extends a.@b.c Foo {}`. The branch_point on + // `annotation-qualified-identifier-name`'s `pop: 2 + branch_point: + // annotation-qualified-parameters` was being pruned by perform_op's + // post-Set retain (`bp.stack_depth <= final_len` ignored + // `bp.pop_count`), so the branch's first alt — which has + // `meta_content_scope: meta.annotation.identifier.java` — was + // never failed-out, leaking `meta.annotation.identifier.java` + // past every nested-annotation extends path in the Java suite. + use crate::parsing::SyntaxSet; + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + for line in ["class T extends a.@b.c Foo {}\n"] { + let out = state.parse_line(line, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + } + let ann = Scope::new("meta.annotation.identifier.java").unwrap(); + assert!( + !stack.as_slice().contains(&ann), + "meta.annotation.identifier.java leaked past `@b.c` annotation \ + into the outer extends path; final stack: {:?}", + stack, + ); + assert!( + !state.shadow.as_slice().contains(&ann), + "syntect shadow still carries meta.annotation.identifier.java; \ + shadow: {:?}", + state.shadow, + ); + } + + #[cfg(feature = "default-onig")] + #[test] + fn cross_line_pop_n_branch_point_alt_fail_unwinds_meta_scope() { + // Cross-line variant of the Java annotation leak: + // `@A.B\nclass E {}\n`. At end of line 1, the + // `annotation-qualified-parameters` branch_point is live waiting + // for `(`. Line 2 starts with `class`, so alt 1 fails and alt 2 + // (`immediately-pop`) runs via handle_fail's cross-line path. That + // path used a bespoke re-emit of just `context.meta_scope` / + // `meta_content_scope`, missing the popped contexts' Pop — leaving + // `meta.annotation.identifier.java` and the surrounding + // declaration's meta_scope (`meta.class.java` / + // `meta.enum.java` / `meta.interface.java`) on the stack. Routing + // through `push_meta_ops` with a synthetic Set/Push (mirroring the + // same-line fix) emits the popped contexts' Pop alongside the new + // alternative's meta_scope push. + // + // The consumer must apply `out.replayed` corrected ops the same + // way `examples/syntest.rs` does: rewind to the buffered line's + // pre-parse stack, replay the corrected ops in order, then apply + // the current line's ops. This mirrors the LRD test above. + use crate::parsing::SyntaxSet; + struct Record { + stack_before: ScopeStack, + } + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + let mut buffer: Vec = Vec::new(); + for line in ["@A.B\n", "class E {}\n"] { + let out = state.parse_line(line, &ss).expect("parse"); + if !out.replayed.is_empty() { + let buf_len = buffer.len(); + let start_idx = buf_len - out.replayed.len(); + stack = buffer[start_idx].stack_before.clone(); + let mut corrected: Vec<(usize, ScopeStack)> = Vec::new(); + for (i, replayed_ops) in out.replayed.iter().enumerate() { + for (_, op) in replayed_ops { + let _ = stack.apply(op); + } + let next_idx = start_idx + i + 1; + if next_idx < buf_len { + corrected.push((next_idx, stack.clone())); + } + } + for (idx, c) in corrected { + buffer[idx].stack_before = c; + } + } + let stack_before = stack.clone(); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + buffer.push(Record { stack_before }); + } + let ann = Scope::new("meta.annotation.identifier.java").unwrap(); + let cls = Scope::new("meta.class.java").unwrap(); + assert!( + !stack.as_slice().contains(&ann), + "meta.annotation.identifier.java leaked past `@A.B` annotation \ + into top-level scope after cross-line `class E {{}}`; final \ + stack: {:?}", + stack, + ); + assert!( + !stack.as_slice().contains(&cls), + "meta.class.java leaked past `class E {{}}` body close into \ + top-level scope; final stack: {:?}", + stack, + ); + assert!( + !state.shadow.as_slice().contains(&ann), + "syntect shadow still carries meta.annotation.identifier.java; \ + shadow: {:?}", + state.shadow, + ); + } + + #[cfg(feature = "default-onig")] + #[test] + fn outer_cross_line_replay_prefers_inner_correction() { + // Java's `@A.B\n(par=1)\nenum E {}\n` triggers a NESTED cross-line + // fail: the outer `declarations` BP fails alt-0 (class) on line 3 + // and replays lines 1-2 under alt-1 (enum); during line-2 reparse + // the inner `annotation-qualified-identifier` BP fails alt-0 (path) + // and replays both lines under alt-1 (name). The outer's locally- + // computed `replayed_ops[0]` was the (now stale) path-alt parse; + // the inner's correction in `flushed_ops` was the name-alt parse, + // and the outer's `merge_flushed` previously overwrote it. Without + // preferring the inner correction, `meta.enum.java`, + // `meta.annotation.identifier.java`, and `meta.path.java` leaked + // past the closing `}`. + use crate::parsing::SyntaxSet; + struct Record { + stack_before: ScopeStack, + } + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + let mut buffer: Vec = Vec::new(); + for line in ["@A.B\n", "(par=1)\n", "enum E {}\n"] { + let out = state.parse_line(line, &ss).expect("parse"); + if !out.replayed.is_empty() { + let buf_len = buffer.len(); + let start_idx = buf_len - out.replayed.len(); + stack = buffer[start_idx].stack_before.clone(); + let mut corrected: Vec<(usize, ScopeStack)> = Vec::new(); + for (i, replayed_ops) in out.replayed.iter().enumerate() { + for (_, op) in replayed_ops { + let _ = stack.apply(op); + } + let next_idx = start_idx + i + 1; + if next_idx < buf_len { + corrected.push((next_idx, stack.clone())); + } + } + for (idx, c) in corrected { + buffer[idx].stack_before = c; + } + } + let stack_before = stack.clone(); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + buffer.push(Record { stack_before }); + } + let ann = Scope::new("meta.annotation.identifier.java").unwrap(); + let path = Scope::new("meta.path.java").unwrap(); + let enum_scope = Scope::new("meta.enum.java").unwrap(); + for scope in [ann, path, enum_scope] { + assert!( + !stack.as_slice().contains(&scope), + "{:?} leaked past closing `}}` into top-level scope; \ + final stack: {:?}", + scope, + stack, + ); + assert!( + !state.shadow.as_slice().contains(&scope), + "syntect shadow still carries {:?}; shadow: {:?}", + scope, + state.shadow, + ); + } + } + + #[cfg(feature = "default-onig")] + #[test] + fn exhausted_branch_point_falls_through_to_parent_next_rule() { + // Java's `$x ;` at top level: the `declarations` branch_point's + // zero-width `(?=[\p{L}_$@<])` lookahead matches `$`. All five + // alternatives (class/enum/interface/variable/method) fail + // because `$x` isn't a valid declaration. ST then falls through + // to the `java` context's NEXT rule (`else-expressions`), which + // pushes an `expression` chain that scopes `$x` as + // `meta.variable.identifier.java variable.other.java`. Syntect + // previously advanced one character past the lookahead, letting + // the next iteration's regex set match `package` / `class` etc. + // in the middle of identifiers like `package$` or `$package`. + // + // After this fix, exhausting a branch_point at a position + // rewinds the cursor to that position and marks the + // branch_point's name as skipped — so the parent context's + // remaining rules get a chance to fire at the same cursor. + use crate::parsing::SyntaxSet; + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + let dollar_id = "$x ;\n"; + let out = state.parse_line(dollar_id, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + let var_id = Scope::new("meta.variable.identifier.java").unwrap(); + let var_other = Scope::new("variable.other.java").unwrap(); + // `$x` itself is fully popped at `;`; reconstruct the per-byte + // scope by walking ops up to byte 1 (`x`) and confirm the + // identifier scope was active there. + let mut mid = ScopeStack::new(); + for (pos, op) in &out.ops { + if *pos > 1 { + break; + } + let _ = mid.apply(op); + } + assert!( + mid.as_slice().contains(&var_id), + "expected meta.variable.identifier.java active over `$x`; got: {:?}", + mid, + ); + assert!( + mid.as_slice().contains(&var_other), + "expected variable.other.java active over `$x`; got: {:?}", + mid, + ); + } + + #[cfg(feature = "default-onig")] + #[test] + fn pop_n_restores_clear_before_unwinding_deeper_meta_scopes() { + // Java's `case DayType when -> "incomplete";` lands in + // `case-label-expression` (clear_scopes:1, mcs: case.label). + // The `clear_scopes:1` hides the parent `case-label`'s + // `meta_scope` (meta.case). When `case-label-end` matches + // `->` with `pop: 2`, the rule must unwind both + // `case-label-expression` AND `case-label`. With the deeper + // meta_scope Pop emitted before the cur_context's clear was + // restored, the consumer popped the wrong (still-visible) + // scope — the surrounding `meta.block.java` (switch's block) + // — leaving `meta.case.java` orphaned past the `->`. + // + // Restoring cur_context's clear BEFORE the depth-loop's + // deeper-meta_scope pops makes the previously-cleared atom + // visible again so it can be popped correctly. + use crate::parsing::SyntaxSet; + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + for line in [ + "class C {\n", + " void f(Object o) {\n", + " return switch (o) {\n", + " case DayType when -> \"incomplete\";\n", + ] { + let out = state.parse_line(line, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + } + let case = Scope::new("meta.statement.conditional.case.java").unwrap(); + let label = Scope::new("meta.statement.conditional.case.label.java").unwrap(); + assert!( + !stack.as_slice().contains(&case), + "meta.statement.conditional.case.java leaked past `->`; stack: {:?}", + stack, + ); + assert!( + !stack.as_slice().contains(&label), + "meta.statement.conditional.case.label.java leaked past `->`; stack: {:?}", + stack, + ); + } + + #[cfg(feature = "default-onig")] + #[test] + fn deeper_inner_bp_correction_does_not_double_outer_meta_scope() { + // `class C { @anno /**/ fully\n. @anno qualified\n/**/ . /**/\n@anno /**/ object @anno()` + // triggers a NESTED cross-line replay where the inner BP is + // structurally a child of the outer BP's resolved alternative + // (outer `class-members` at depth 4, inner `object-type` at + // depth 9). PR #663's `prefer_inner_replay_corrections` + // unconditionally replaced outer's locally-computed ops with + // inner's corrections, doubling `meta.field.type.java` on the + // `object @anno()` line — outer's full-line ops correctly + // emit one `meta.field.type` and inner's reparse adds another + // because outer's chosen alt already provides that meta_scope. + // + // Discriminator: only prefer inner when its stack_depth is at + // most outer's. Equal-depth siblings (PR #663's original + // `@A.B\n(par=1)\nenum E {}` case) keep preferring inner; + // strictly-deeper nested BPs stay with outer's ops. + use crate::parsing::SyntaxSet; + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let syntax = ss + .find_syntax_by_path("Packages/Java/Java.sublime-syntax") + .unwrap(); + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + for line in [ + "class C {\n", + " @anno /**/ fully\n", + " . @anno qualified\n", + " /**/ . /**/\n", + " @anno /**/ object @anno()\n", + ] { + let out = state.parse_line(line, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + } + // Reconstruct the running stack at byte position 13 of line 5 + // (`object`), where the regression's doubled push was visible. + let mut at_object = ScopeStack::new(); + let last_line = " @anno /**/ object @anno()\n"; + let mut state2 = ParseState::new(syntax); + let prelude = [ + "class C {\n", + " @anno /**/ fully\n", + " . @anno qualified\n", + " /**/ . /**/\n", + ]; + for line in prelude { + let out = state2.parse_line(line, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = at_object.apply(op); + } + } + let out = state2.parse_line(last_line, &ss).expect("parse"); + for (pos, op) in &out.ops { + if *pos > 13 { + break; + } + let _ = at_object.apply(op); + } + let field_type = Scope::new("meta.field.type.java").unwrap(); + let doubled = at_object + .as_slice() + .iter() + .filter(|s| **s == field_type) + .count(); + assert!( + doubled <= 1, + "meta.field.type.java pushed {} times entering `object` on \ + line 5 (expected at most 1); stack: {:?}", + doubled, + at_object, + ); + } + + /// Regression guard for the `embed_scope`-replaces / inner Set + /// interaction. With a wrapper context that pushes 3 mcs scopes + /// and `embed_scope_replaces=true`, the embedded syntax's first + /// `set:` rule must not drop the topmost wrapper scope — its mcs + /// pop must be skipped because the embedded main's mcs was never + /// pushed. + #[test] + fn embed_scope_replaces_preserves_wrapper_mcs_across_inner_set() { + let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); + let md = ss + .find_syntax_by_scope(Scope::new("text.html.markdown").unwrap()) + .expect("Markdown loaded"); + let mut state = ParseState::new(md); + let mut stack = ScopeStack::new(); + for line in ["```bash\n", "#!/usr/bin/env bash\n"] { + let out = state.parse_line(line, &ss).expect("parse"); + for (_, op) in &out.ops { + let _ = stack.apply(op); + } + } + let bash = Scope::new("source.shell.bash").unwrap(); + assert!( + stack.as_slice().contains(&bash), + "source.shell.bash (wrapper's last embed_scope token) must be \ + on the stack after the embedded syntax's first `set:` fires; \ + stack: {:?}", + stack + ); + } } diff --git a/src/parsing/syntax_definition.rs b/src/parsing/syntax_definition.rs index 1849bebc..9a037004 100644 --- a/src/parsing/syntax_definition.rs +++ b/src/parsing/syntax_definition.rs @@ -274,17 +274,20 @@ impl<'a> Iterator for MatchIter<'a> { _ => return self.next(), }; let ctx_ptr = self.syntax_set.get_context(context_id).unwrap(); - // Also include the external syntax's prototype if the context allows it + // Push target first, then external prototype on top so + // its patterns are iterated first: `MatchIter::next` + // reads the stack top, and ST's `apply_prototype` runs + // the prototype ahead of the included context — mirror + // of `ParseState::find_best_match`'s prototype chaining. + self.ctx_stack.push(ctx_ptr); + self.index_stack.push(0); if ctx_ptr.meta_include_prototype.unwrap_or(true) { if let Some(ref proto_id) = ctx_ptr.prototype { let proto_ctx = self.syntax_set.get_context(proto_id).unwrap(); - // Push prototype first (it will be iterated first) self.ctx_stack.push(proto_ctx); self.index_stack.push(0); } } - self.ctx_stack.push(ctx_ptr); - self.index_stack.push(0); } } } else { diff --git a/src/parsing/syntax_set.rs b/src/parsing/syntax_set.rs index 328527da..5341feb1 100644 --- a/src/parsing/syntax_set.rs +++ b/src/parsing/syntax_set.rs @@ -2576,8 +2576,159 @@ mod tests { builder.add(syntax_using_proto); let ss = builder.build(); - // Just verify it builds without errors and the syntax exists - assert!(ss.find_syntax_by_name("UsingProto").is_some()); + let syntax = ss.find_syntax_by_name("UsingProto").unwrap(); + let mut parse_state = ParseState::new(syntax); + let ops = parse_state.parse_line("#", &ss).expect("#[cfg(test)]").ops; + // The external `prototype`'s `#` rule must be reachable from the + // including context via `apply_prototype: true`. + let expected = (0, ScopeStackOp::Push(Scope::new("comment.proto").unwrap())); + assert_ops_contain(&ops, &expected); + } + + #[test] + fn apply_prototype_prototype_wins_tie_over_target_main() { + // Both the target's `main` and its `prototype` match `|` at the same + // position. ST's `apply_prototype` semantics put the prototype ahead + // of the target, so the prototype's scope wins the tie. + let target = SyntaxDefinition::load_from_str( + r#" + name: Target + scope: source.target + file_extensions: [t] + contexts: + prototype: + - match: '\|' + scope: proto.pipe + main: + - match: '\|' + scope: target.bitor + "#, + true, + None, + ) + .unwrap(); + + let outer = SyntaxDefinition::load_from_str( + r#" + name: Outer + scope: source.outer + file_extensions: [o] + contexts: + main: + - include: scope:source.target + apply_prototype: true + "#, + true, + None, + ) + .unwrap(); + + let mut builder = SyntaxSetBuilder::new(); + builder.add(target); + builder.add(outer); + let ss = builder.build(); + + let syntax = ss.find_syntax_by_name("Outer").unwrap(); + let mut parse_state = ParseState::new(syntax); + let ops = parse_state.parse_line("|", &ss).expect("#[cfg(test)]").ops; + assert_ops_contain( + &ops, + &(0, ScopeStackOp::Push(Scope::new("proto.pipe").unwrap())), + ); + assert!( + !ops.iter() + .any(|(_, op)| matches!(op, ScopeStackOp::Push(s) if s == &Scope::new("target.bitor").unwrap())), + "target main's `|` rule must not pre-empt the external prototype's `|` rule: {:?}", + ops, + ); + } + + #[test] + fn apply_prototype_respects_meta_include_prototype_false() { + // When the include target opts out of prototype inclusion, the + // external prototype must NOT be injected even with + // `apply_prototype: true` on the include. + let target = SyntaxDefinition::load_from_str( + r#" + name: Target + scope: source.target + file_extensions: [t] + contexts: + prototype: + - match: '\|' + scope: proto.pipe + main: + - meta_include_prototype: false + - match: '\|' + scope: target.bitor + "#, + true, + None, + ) + .unwrap(); + + let outer = SyntaxDefinition::load_from_str( + r#" + name: Outer + scope: source.outer + file_extensions: [o] + contexts: + main: + - include: scope:source.target + apply_prototype: true + "#, + true, + None, + ) + .unwrap(); + + let mut builder = SyntaxSetBuilder::new(); + builder.add(target); + builder.add(outer); + let ss = builder.build(); + + let syntax = ss.find_syntax_by_name("Outer").unwrap(); + let mut parse_state = ParseState::new(syntax); + let ops = parse_state.parse_line("|", &ss).expect("#[cfg(test)]").ops; + assert_ops_contain( + &ops, + &(0, ScopeStackOp::Push(Scope::new("target.bitor").unwrap())), + ); + assert!( + !ops.iter() + .any(|(_, op)| matches!(op, ScopeStackOp::Push(s) if s == &Scope::new("proto.pipe").unwrap())), + "prototype must stay out when target's `meta_include_prototype: false`: {:?}", + ops, + ); + } + + #[test] + fn haml_pipe_continuation_wins_over_ruby_bitor() { + // Real-package regression guard for the fix: inside HAML attribute + // braces, a trailing `|` must scope as HAML's pipe-continuation + // (injected via Ruby-for-HAML's prototype under `apply_prototype`), + // not as Ruby's bitwise-or operator. + let ss = &*testdata::PACKAGES_SYN_SET; + let syntax = ss.find_syntax_by_name("HAML").unwrap(); + let mut parse_state = ParseState::new(syntax); + let ops = parse_state + .parse_line("%p{:a => 1, |", ss) + .expect("#[cfg(test)]") + .ops; + let pipe_cont = Scope::new("punctuation.separator.continuation.haml").unwrap(); + let ruby_bitor = Scope::new("keyword.operator.bitwise.ruby").unwrap(); + assert!( + ops.iter() + .any(|(_, op)| matches!(op, ScopeStackOp::Push(s) if s == &pipe_cont)), + "expected punctuation.separator.continuation.haml push: {:?}", + ops, + ); + assert!( + !ops.iter() + .any(|(_, op)| matches!(op, ScopeStackOp::Push(s) if s == &ruby_bitor)), + "Ruby bitwise-or must not win over HAML pipe-continuation: {:?}", + ops, + ); } // ===================================================== diff --git a/src/parsing/yaml_load.rs b/src/parsing/yaml_load.rs index d1194c48..c07cf6d2 100644 --- a/src/parsing/yaml_load.rs +++ b/src/parsing/yaml_load.rs @@ -392,6 +392,20 @@ impl SyntaxDefinition { ctx_refs: SyntaxDefinition::parse_pushargs(s, state, contexts, namer)?, pop_count: y as usize, } + } else if let Ok(p) = get_key(map, "push", Some) { + // `pop: N + push: X` pops N contexts then pushes X — + // equivalent stack effect to `pop: N + set: X`. Without + // this case the `push` was silently dropped and the rule + // degraded to a plain `Pop(N)`, leaving the parser on the + // outer context instead of the intended target. + // Observed on Java's `pop: 2 + push: annotation-parameters-body` + // (annotation arg lists) and `pop: 1 + push: case-label-expression`, + // and on Python's `pop: 2 + push: function-parameter-list-body` + // / `type-parameter-list-body`. + MatchOperation::Set { + ctx_refs: SyntaxDefinition::parse_pushargs(p, state, contexts, namer)?, + pop_count: y as usize, + } } else { MatchOperation::Pop(y as usize) } @@ -611,16 +625,22 @@ impl SyntaxDefinition { fn parse_captures( map: &Hash, - regex_str: &str, + _regex_str: &str, state: &mut ParserState<'_>, ) -> Result { - let valid_indexes = get_consuming_capture_indexes(regex_str); + // Accept every numeric capture entry. Groups inside lookarounds are + // kept — `build_capture_ops` clips each capture's span to the rule's + // consumed match range at parse time, mirroring Sublime Text. An + // earlier version filtered lookaround-internal indices here on the + // assumption "those scopes are not applied," which produced silent + // drops for valid rules like C#'s generic function-call pattern. let mut captures = Vec::new(); for (key, value) in map.iter() { if let (Some(key_int), Some(val_str)) = (key.as_i64(), value.as_str()) { - if valid_indexes.contains(&(key_int as usize)) { - captures.push((key_int as usize, str_to_scopes(val_str, state.scope_repo)?)); + if key_int < 0 { + continue; } + captures.push((key_int as usize, str_to_scopes(val_str, state.scope_repo)?)); } } Ok(captures) @@ -945,101 +965,6 @@ impl RegexRewriterForNoNewlines<'_> { } } -fn get_consuming_capture_indexes(regex: &str) -> Vec { - let parser = ConsumingCaptureIndexParser { - parser: Parser::new(regex.as_bytes()), - }; - parser.get_consuming_capture_indexes() -} - -struct ConsumingCaptureIndexParser<'a> { - parser: Parser<'a>, -} - -impl ConsumingCaptureIndexParser<'_> { - /// Find capture groups which are not inside lookarounds. - /// - /// If, in a YAML syntax definition, a scope stack is applied to a capture group inside a - /// lookaround, (i.e. "captures:\n x: scope.stack goes.here", where "x" is the number of a - /// capture group in a lookahead/behind), those those scopes are not applied, so no need to - /// even parse them. - fn get_consuming_capture_indexes(mut self) -> Vec { - let mut result = Vec::new(); - let mut stack = Vec::new(); - let mut cap_num = 0; - let mut in_lookaround = false; - stack.push(in_lookaround); - result.push(cap_num); - - while let Some(c) = self.parser.peek() { - match c { - b'\\' => { - self.parser.next(); - self.parser.next(); - } - b'[' => { - self.parser.parse_character_class(); - } - b'(' => { - self.parser.next(); - // add the current lookaround state to the stack so we can just pop at a closing paren - stack.push(in_lookaround); - if let Some(c2) = self.parser.peek() { - if c2 != b'?' { - // simple numbered capture group - cap_num += 1; - // if we are not currently in a lookaround, - // add this capture group number to the valid ones - if !in_lookaround { - result.push(cap_num); - } - } else { - self.parser.next(); - if let Some(c3) = self.parser.peek() { - self.parser.next(); - if c3 == b'=' || c3 == b'!' { - // lookahead - in_lookaround = true; - } else if c3 == b'<' { - if let Some(c4) = self.parser.peek() { - if c4 == b'=' || c4 == b'!' { - self.parser.next(); - // lookbehind - in_lookaround = true; - } - } - } else if c3 == b'P' { - if let Some(c4) = self.parser.peek() { - if c4 == b'<' { - // named capture group - cap_num += 1; - // if we are not currently in a lookaround, - // add this capture group number to the valid ones - if !in_lookaround { - result.push(cap_num); - } - } - } - } - } - } - } - } - b')' => { - if let Some(value) = stack.pop() { - in_lookaround = value; - } - self.parser.next(); - } - _ => { - self.parser.next(); - } - } - } - result - } -} - struct Parser<'a> { bytes: &'a [u8], index: usize, @@ -1386,6 +1311,55 @@ mod tests { } } + #[test] + fn pop_plus_push_becomes_set_with_pop_count() { + // Regression: Java's `pop: 2 + push: annotation-parameters-body` + // (and Python's `pop: 2 + push: function-parameter-list-body` etc.) + // were silently dropping the `push` half and degrading to a plain + // `Pop(N)`, so the target context never appeared on the stack. + let def = SyntaxDefinition::load_from_str( + r#" + name: Test + scope: text.test + file_extensions: [test] + contexts: + main: + - match: \( + pop: 2 + push: target + target: + - meta_scope: meta.target + - include: main + "#, + false, + None, + ) + .unwrap(); + + let ctx = &def.contexts["main"]; + if let Pattern::Match(ref match_pattern) = ctx.patterns[0] { + match match_pattern.operation { + MatchOperation::Set { + ref ctx_refs, + pop_count, + } => { + assert_eq!(pop_count, 2); + assert_eq!(ctx_refs.len(), 1); + assert!(matches!( + ctx_refs[0], + ContextReference::Named(ref n) if n == "target" + )); + } + _ => panic!( + "Expected Set operation with pop_count=2, got {:?}", + match_pattern.operation + ), + } + } else { + panic!("Expected Match pattern"); + } + } + #[test] fn errors_on_regex_compile_error() { let def = SyntaxDefinition::load_from_str( @@ -1580,33 +1554,6 @@ mod tests { assert_eq!(&rewrite(r"(?<=\n)ab"), r"(?<=$)ab"); } - #[test] - fn can_get_valid_captures_from_regex() { - let regex = "hello(test)(?=(world))(foo(?Pbar))"; - println!("{:?}", regex); - let valid_indexes = get_consuming_capture_indexes(regex); - println!("{:?}", valid_indexes); - assert_eq!(valid_indexes, [0, 1, 3, 4]); - } - - #[test] - fn can_get_valid_captures_from_regex2() { - let regex = "hello(test)[(?=tricked](foo(bar))"; - println!("{:?}", regex); - let valid_indexes = get_consuming_capture_indexes(regex); - println!("{:?}", valid_indexes); - assert_eq!(valid_indexes, [0, 1, 2, 3]); - } - - #[test] - fn can_get_valid_captures_from_nested_regex() { - let regex = "hello(test)(?=(world(?!(te(?<=(st))))))(foo(bar))"; - println!("{:?}", regex); - let valid_indexes = get_consuming_capture_indexes(regex); - println!("{:?}", valid_indexes); - assert_eq!(valid_indexes, [0, 1, 5, 6]); - } - #[test] fn error_loading_syntax_with_unescaped_backslash() { let load_err = SyntaxDefinition::load_from_str( diff --git a/testdata/known_syntest_failures.txt b/testdata/known_syntest_failures.txt index 54e1a579..49301eaf 100644 --- a/testdata/known_syntest_failures.txt +++ b/testdata/known_syntest_failures.txt @@ -1,24 +1,13 @@ loading syntax definitions from testdata/Packages -FAILED testdata/Packages/ASP/syntax_test_asp.asp: 53 -FAILED testdata/Packages/Batch File/tests/syntax_test_batch_file.bat: 74 FAILED testdata/Packages/C#/tests/syntax_test_C#11.cs: 35 -FAILED testdata/Packages/C#/tests/syntax_test_GeneralStructure.cs: 3 -FAILED testdata/Packages/C#/tests/syntax_test_Generics.cs: 3 -FAILED testdata/Packages/Clojure/tests/syntax_test_clojure.clj: 1 FAILED testdata/Packages/Clojure/tests/syntax_test_shebang.clj: 1 FAILED testdata/Packages/D/tests/syntax_test_shebang.d: 1 -FAILED testdata/Packages/Git Formats/tests/syntax_test_git_config: 17 -FAILED testdata/Packages/Haskell/tests/syntax_test_haskell.hs: 72 -FAILED testdata/Packages/Java/tests/syntax_test_java.java: 1 -FAILED testdata/Packages/Java/tests/syntax_test_jsp.jsp: 44 -FAILED testdata/Packages/JavaScript/tests/syntax_test_typescript.ts: 1 -FAILED testdata/Packages/LaTeX/tests/syntax_test_latex.tex: 76 -FAILED testdata/Packages/Markdown/tests/syntax_test_markdown.md: 1 +FAILED testdata/Packages/Haskell/tests/syntax_test_haskell.hs: 49 +FAILED testdata/Packages/Java/tests/syntax_test_java.java: 245 +FAILED testdata/Packages/Markdown/tests/syntax_test_markdown.md: 31 FAILED testdata/Packages/PHP/tests/syntax_test_php.php: 1 -FAILED testdata/Packages/Python/tests/syntax_test_python.py: 1 +FAILED testdata/Packages/Python/tests/syntax_test_python.py: 32 FAILED testdata/Packages/Python/tests/syntax_test_python_strings.py: 1 -FAILED testdata/Packages/Rails/tests/syntax_test_rails.haml: 65 -FAILED testdata/Packages/Rails/tests/syntax_test_rails.html.erb: 23 -FAILED testdata/Packages/ShellScript/Bash/tests/syntax_test_scope.bash: 1 -FAILED testdata/Packages/ShellScript/Zsh/tests/syntax_test_scope.zsh: 1 +FAILED testdata/Packages/ShellScript/Bash/tests/syntax_test_scope.bash: 249 +FAILED testdata/Packages/ShellScript/Zsh/tests/syntax_test_scope.zsh: 410 exiting with code 1 diff --git a/testdata/known_syntest_failures_fancy.txt b/testdata/known_syntest_failures_fancy.txt index 54e1a579..49301eaf 100644 --- a/testdata/known_syntest_failures_fancy.txt +++ b/testdata/known_syntest_failures_fancy.txt @@ -1,24 +1,13 @@ loading syntax definitions from testdata/Packages -FAILED testdata/Packages/ASP/syntax_test_asp.asp: 53 -FAILED testdata/Packages/Batch File/tests/syntax_test_batch_file.bat: 74 FAILED testdata/Packages/C#/tests/syntax_test_C#11.cs: 35 -FAILED testdata/Packages/C#/tests/syntax_test_GeneralStructure.cs: 3 -FAILED testdata/Packages/C#/tests/syntax_test_Generics.cs: 3 -FAILED testdata/Packages/Clojure/tests/syntax_test_clojure.clj: 1 FAILED testdata/Packages/Clojure/tests/syntax_test_shebang.clj: 1 FAILED testdata/Packages/D/tests/syntax_test_shebang.d: 1 -FAILED testdata/Packages/Git Formats/tests/syntax_test_git_config: 17 -FAILED testdata/Packages/Haskell/tests/syntax_test_haskell.hs: 72 -FAILED testdata/Packages/Java/tests/syntax_test_java.java: 1 -FAILED testdata/Packages/Java/tests/syntax_test_jsp.jsp: 44 -FAILED testdata/Packages/JavaScript/tests/syntax_test_typescript.ts: 1 -FAILED testdata/Packages/LaTeX/tests/syntax_test_latex.tex: 76 -FAILED testdata/Packages/Markdown/tests/syntax_test_markdown.md: 1 +FAILED testdata/Packages/Haskell/tests/syntax_test_haskell.hs: 49 +FAILED testdata/Packages/Java/tests/syntax_test_java.java: 245 +FAILED testdata/Packages/Markdown/tests/syntax_test_markdown.md: 31 FAILED testdata/Packages/PHP/tests/syntax_test_php.php: 1 -FAILED testdata/Packages/Python/tests/syntax_test_python.py: 1 +FAILED testdata/Packages/Python/tests/syntax_test_python.py: 32 FAILED testdata/Packages/Python/tests/syntax_test_python_strings.py: 1 -FAILED testdata/Packages/Rails/tests/syntax_test_rails.haml: 65 -FAILED testdata/Packages/Rails/tests/syntax_test_rails.html.erb: 23 -FAILED testdata/Packages/ShellScript/Bash/tests/syntax_test_scope.bash: 1 -FAILED testdata/Packages/ShellScript/Zsh/tests/syntax_test_scope.zsh: 1 +FAILED testdata/Packages/ShellScript/Bash/tests/syntax_test_scope.bash: 249 +FAILED testdata/Packages/ShellScript/Zsh/tests/syntax_test_scope.zsh: 410 exiting with code 1