diff --git a/.gitignore b/.gitignore index c37173ea..a77bff8c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,14 @@ target/ # VS Code .vscode/ + +# Generated by examples/synhtml-css-classes +synhtml-css-classes.html +synhtml-css-classes.css +theme-dark.css +theme-light.css + +# cargo-mutants +mutants.out/ +mutants.out.old/ +master.diff diff --git a/.gitmodules b/.gitmodules index e42197e5..93ed7fae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "testdata/Packages"] path = testdata/Packages url = https://github.com/sublimehq/Packages + branch = v4202 [submodule "testdata/InspiredGitHub.tmtheme"] path = testdata/InspiredGitHub.tmtheme url = https://github.com/sethlopezme/InspiredGitHub.tmtheme.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 25eb350b..63180f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Breaking changes +- `HighlightLines` and `HighlightFile` deprecated since 6.0.0 — use `syntect::io::HighlightedWriter` instead [#627] +- `line_tokens_to_classed_spans` (`syntect::html`) deprecated since 6.0.0 — cannot correctly handle cross-line branch-point failures (the rendered output for earlier lines is already in the caller's hands when the parser retroactively replays). Use `ClassedHTMLGenerator` or `HighlightedWriter::from_markup` instead, both of which buffer rendered output during speculative parsing and replay corrected ops [#627] +- `SCOPE_REPO` removed from public API (was already deprecated since 5.3.0) [#627] - `ParseState::parse_line` now returns `Result` instead of a bare `Vec` [#614] - `MatchOperation` enum: new `Branch`, `Fail` [#614], and `Embed` [#615] variants - `Pattern` enum: new `IncludeWithPrototype` variant [#610] @@ -15,12 +18,33 @@ ### Improvements +- Add `syntect::io` module containing `HighlightedWriter<'a, R, W>`, a streaming highlighter that implements `std::io::Write` and handles branch-point backtracking, generic over the output sink (`Vec` by default, or any `io::Write` for streaming). Constructed via the new `HighlightedWriterBuilder`: pick a renderer category with `HighlightedWriter::from_themed` / `from_markup` / `from_renderer`, then chain `.with_output(...)` / `.with_state(...)` and finish with `.build()`. End-of-input cleanup runs implicitly via `Drop` on a best-effort basis; call `into_inner` explicitly when you need the inner sink back or want errors propagated [#627] +- Add `syntect::rendering` module with a layered renderer trait design [#627]: + - `ScopeMarkup` — slim trait for stateless renderers that map scope structure 1:1 to output structure (e.g. CSS-classed HTML); receives only pre-resolved atom strings + - `StyledOutput` — small trait for theme-aware renderers (e.g. ANSI, inline-styled HTML, LaTeX `\textcolor`); only describes how to begin/end a styled span and write text + - `ThemedRenderer` — adapter that turns any `StyledOutput` into a `ScopeRenderer` by managing the `Highlighter`, the style stack, and adjacent-token style merging + - `ScopeRenderer` — low-level engine trait kept as an escape hatch for advanced cases that need raw `Scope` / `&[Scope]` access + - `AnsiStyledOutput` — built-in 24-bit ANSI colour emitter; pair with `HighlightedWriter::from_themed` for terminal output +- Add `ClassedHTMLScopeRenderer` (`syntect::html`) implementing `ScopeMarkup` for `` output with CSS classes derived from scope atoms [#627] +- Add `HtmlStyledOutput` (`syntect::html`) implementing `StyledOutput` for theme-aware inline-styled `` HTML output with whitespace-merge optimization [#627] +- Add `Scope::with_atom_strs()` closure-based API for reading atom strings without direct repository access [#627] +- `ClassedHTMLGenerator` preserved as backward-compatible wrapper around `HighlightedWriter` (now driven by `MarkupAdapter`) [#627] +- `highlighted_html_for_string`/`highlighted_html_for_file` reworked to use `HighlightedWriter::from_themed` and `HtmlStyledOutput` [#627] - Implement sublime-syntax Build 4075 features: `extends` (single and multiple inheritance), `meta_prepend`/`meta_append`, `apply_prototype`, `version: 2`, and `first_line_match` variable resolution [#610] [#612] - Implement `branch_point`/`branch`/`fail` backtracking with cross-line replay support [#614] - Implement native `embed`/`escape` support, fixing cases where embedded context patterns could consume escape characters first [#615] - Gracefully skip syntax files that fail to load and hide syntaxes with unresolved `extends` parents [#610] - Add `warnings()` API on `SyntaxSetBuilder` and `SyntaxSet` to replace `eprintln!` warnings with programmatic access [#619] +[#512]: https://github.com/trishume/syntect/pull/512 +[#576]: https://github.com/trishume/syntect/pull/576 +[#610]: https://github.com/trishume/syntect/pull/610 +[#612]: https://github.com/trishume/syntect/pull/612 +[#614]: https://github.com/trishume/syntect/pull/614 +[#615]: https://github.com/trishume/syntect/pull/615 +[#619]: https://github.com/trishume/syntect/pull/619 +[#627]: https://github.com/trishume/syntect/pull/627 + ## [Version 5.3.0](https://github.com/trishume/syntect/compare/v5.2.0...v5.3.0) ### Improvements diff --git a/assets/default_newlines.packdump b/assets/default_newlines.packdump index 54977dc0..d92c1280 100644 Binary files a/assets/default_newlines.packdump and b/assets/default_newlines.packdump differ diff --git a/assets/default_nonewlines.packdump b/assets/default_nonewlines.packdump index 21e4c431..3b5a0dcf 100644 Binary files a/assets/default_nonewlines.packdump and b/assets/default_nonewlines.packdump differ diff --git a/benches/highlight_utils/mod.rs b/benches/highlight_utils/mod.rs index 2170a166..f30f97e7 100644 --- a/benches/highlight_utils/mod.rs +++ b/benches/highlight_utils/mod.rs @@ -1,6 +1,8 @@ -use syntect::easy::HighlightLines; +use std::io::Write; use syntect::highlighting::Theme; +use syntect::io::HighlightedWriter; use syntect::parsing::{SyntaxReference, SyntaxSet}; +use syntect::rendering::AnsiStyledOutput; /// Common helper for benchmarking highlighting. pub fn do_highlight( @@ -9,11 +11,11 @@ pub fn do_highlight( syntax: &SyntaxReference, theme: &Theme, ) -> usize { - let mut h = HighlightLines::new(syntax, theme); - let mut count = 0; + let mut highlight = + HighlightedWriter::from_themed(syntax, syntax_set, theme, AnsiStyledOutput::Transparent) + .build(); for line in s.lines() { - let regions = h.highlight_line(line, syntax_set).unwrap(); - count += regions.len(); + writeln!(highlight, "{}", line).unwrap(); } - count + highlight.into_inner().unwrap().len() } diff --git a/examples/latex-demo.rs b/examples/latex-demo.rs index d40dd2ff..33a8f8b5 100644 --- a/examples/latex-demo.rs +++ b/examples/latex-demo.rs @@ -1,7 +1,44 @@ -use syntect::easy::HighlightLines; +use std::fmt::Write as FmtWrite; +use std::io::Write as IoWrite; use syntect::highlighting::{Style, ThemeSet}; +use syntect::io::HighlightedWriter; use syntect::parsing::SyntaxSet; -use syntect::util::{as_latex_escaped, LinesWithEndings}; +use syntect::rendering::StyledOutput; + +/// A [`StyledOutput`] that produces LaTeX `\textcolor[RGB]{r,g,b}{...}` +/// output. Wrap with [`ThemedRenderer`] (or pass to +/// [`HighlightedWriter::with_themed`]) to plug it into a highlighter. +struct LatexStyledOutput; + +impl StyledOutput for LatexStyledOutput { + fn begin_style(&mut self, style: Style, output: &mut String) { + write!( + output, + "\\textcolor[RGB]{{{},{},{}}}{{", + style.foreground.r, style.foreground.g, style.foreground.b + ) + .unwrap(); + } + + fn end_style(&mut self, output: &mut String) { + output.push('}'); + } + + fn write_text(&mut self, text: &str, output: &mut String) { + // The `ThemedRenderer` adapter guarantees `text` never contains + // '\n' (line breaks are split out and emitted between styled + // spans), so we only need to escape LaTeX's three special + // characters. + for ch in text.chars() { + match ch { + '\\' => output.push_str("\\\\"), + '{' => output.push_str("\\{"), + '}' => output.push_str("\\}"), + _ => output.push(ch), + } + } + } +} fn main() { // Load these once at the start of your program @@ -9,14 +46,19 @@ fn main() { let ts = ThemeSet::load_defaults(); let syntax = ps.find_syntax_by_extension("rs").unwrap(); - let s = "pub struct Wow { hi: u64 }\nfn blah() -> u64 {}\n"; - - let mut h = HighlightLines::new(syntax, &ts.themes["InspiredGitHub"]); - for line in LinesWithEndings::from(s) { - // LinesWithEndings enables use of newlines mode - let ranges: Vec<(Style, &str)> = h.highlight_line(line, &ps).unwrap(); - let escaped = as_latex_escaped(&ranges[..]); - println!("\n{:?}", line); - println!("\n{}", escaped); - } + // No explicit `into_inner` needed at the end: we don't need the + // `StdoutLock` back, and `Drop` runs the close-scopes / partial-line + // cleanup on a best-effort basis when `highlight` falls out of scope. + // Examples that *do* need the inner sink back (e.g. `parsyncat` collecting + // a `Vec` per file) call `into_inner` explicitly to propagate errors. + let mut highlight = HighlightedWriter::from_themed( + syntax, + &ps, + &ts.themes["InspiredGitHub"], + LatexStyledOutput, + ) + .with_output(std::io::stdout().lock()) + .build(); + writeln!(highlight, "pub struct Wow {{ hi: u64 }}").unwrap(); + writeln!(highlight, "fn blah() -> u64 {{}}").unwrap(); } diff --git a/examples/parsyncat.rs b/examples/parsyncat.rs index 4f9aa39e..0944b165 100644 --- a/examples/parsyncat.rs +++ b/examples/parsyncat.rs @@ -2,12 +2,13 @@ //! Prints the highlighted output to stdout. use rayon::prelude::*; -use syntect::easy::HighlightFile; -use syntect::highlighting::{Style, ThemeSet}; +use syntect::highlighting::ThemeSet; +use syntect::io::HighlightedWriter; use syntect::parsing::SyntaxSet; +use syntect::rendering::AnsiStyledOutput; use std::fs::File; -use std::io::{BufRead, BufReader}; +use std::io::{self, Write}; fn main() { let files: Vec = std::env::args().skip(1).collect(); @@ -20,51 +21,33 @@ fn main() { let syntax_set = SyntaxSet::load_defaults_newlines(); let theme_set = ThemeSet::load_defaults(); - // We first collect the contents of the files... - let contents: Vec> = files + // Highlight each file in parallel, buffering each result into its own + // `Vec` so we can stream them out in order at the end. + let outputs: Vec> = files .par_iter() .map(|filename| { - let mut lines = Vec::new(); - // We use `String::new()` and `read_line()` instead of `BufRead::lines()` - // in order to preserve the newlines and get better highlighting. - let mut line = String::new(); - let mut reader = BufReader::new(File::open(filename).unwrap()); - while reader.read_line(&mut line).unwrap() > 0 { - lines.push(line); - line = String::new(); - } - lines - }) - .collect(); - - // ...so that the highlighted regions have valid lifetimes... - let regions: Vec> = files - .par_iter() - .zip(&contents) - .map(|(filename, contents)| { - let mut regions = Vec::new(); let theme = &theme_set.themes["base16-ocean.dark"]; - let mut highlighter = HighlightFile::new(filename, &syntax_set, theme).unwrap(); - - for line in contents { - for region in highlighter - .highlight_lines - .highlight_line(line, &syntax_set) - .unwrap() - { - regions.push(region); - } - } - - regions + let syntax = syntax_set + .find_syntax_for_file(filename) + .unwrap() + .unwrap_or_else(|| syntax_set.find_syntax_plain_text()); + let mut f = File::open(filename).unwrap(); + let mut w = HighlightedWriter::from_themed( + syntax, + &syntax_set, + theme, + AnsiStyledOutput::Transparent, + ) + .build(); + io::copy(&mut f, &mut w).unwrap(); + w.into_inner().unwrap() }) .collect(); // ...and then print them all out. - for file_regions in regions { - print!( - "{}", - syntect::util::as_24_bit_terminal_escaped(&file_regions[..], true) - ); + let stdout = io::stdout(); + let mut out = stdout.lock(); + for output in outputs { + out.write_all(&output).unwrap(); } } diff --git a/examples/scopedump.rs b/examples/scopedump.rs new file mode 100644 index 00000000..6aad12a8 --- /dev/null +++ b/examples/scopedump.rs @@ -0,0 +1,261 @@ +//! Differential harness for parser engine changes. +//! +//! Walks a directory tree (default `testdata/Packages`), parses every file +//! whose syntax can be resolved, and prints one fingerprint line per input +//! line: an FNV-1a 64 hash over that line's folded scope-stack regions. +//! `--full` prints the regions themselves instead, for human diffing. +//! +//! Cross-line backtracking (`ParseLineOutput::replayed`) is folded in before +//! anything is printed, so the dump reflects the final corrected scopes, not +//! the provisional op stream. Two engines that emit differently-shaped op +//! streams but reach the same scopes per region produce identical dumps — +//! this is deliberate: the comparison unit is observable highlighting +//! behavior, not op-stream stability. +//! +//! Typical use — capture a baseline, change the engine, diff: +//! +//! ```sh +//! cargo run --release --example scopedump > before.dump +//! # ... engine change ... +//! cargo run --release --example scopedump > after.dump +//! diff before.dump after.dump +//! ``` +//! +//! Run once per regex backend (the fancy backend swaps in via the same +//! flags the syntest targets use): +//! +//! ```sh +//! cargo run --features default-fancy --no-default-features --release --example scopedump +//! ``` +//! +//! To inspect a divergence flagged by the hash diff: +//! +//! ```sh +//! cargo run --release --example scopedump -- --full testdata/Packages/Java > java.dump +//! ``` + +use std::fmt::Write as _; +use std::path::Path; + +use getopts::Options; +use walkdir::{DirEntry, WalkDir}; + +use syntect::easy::ScopeRegionIterator; +use syntect::parsing::{ParseLineOutput, ParseState, ScopeStack, SyntaxSet, SyntaxSetBuilder}; +use syntect::util::LinesWithEndings; + +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &b in bytes { + hash ^= u64::from(b); + hash = hash.wrapping_mul(0x100_0000_01b3); + } + hash +} + +struct LineRecord { + text: String, + /// Scope stack at the start of this line. Kept so a later replay window + /// beginning at this line can rebuild from the corrected baseline. + stack_before: ScopeStack, + /// Canonical serialization of the line's scope regions: one + /// `charcol:charlen scope scope ...` entry per non-empty region. + folded: String, +} + +/// Applies `ops` to `stack`, returning the canonical folded-region +/// serialization for the line. +fn fold_line( + ops: &[(usize, syntect::parsing::ScopeStackOp)], + text: &str, + stack: &mut ScopeStack, +) -> Result> { + let mut folded = String::new(); + let mut col = 0usize; + for (region, op) in ScopeRegionIterator::new(ops, text) { + stack.apply(op)?; + if region.is_empty() { + continue; + } + let len = region.chars().count(); + write!(folded, "{}:{}", col, len)?; + for scope in stack.as_slice() { + write!(folded, " {}", scope)?; + } + folded.push('\n'); + col += len; + } + Ok(folded) +} + +/// Hidden files and directories are environment, not corpus: a submodule's +/// `.git` is a pointer file whose content is checkout-specific and would +/// make dumps from different clones incomparable. +fn is_hidden(entry: &DirEntry) -> bool { + entry + .file_name() + .to_str() + .map(|s| s.starts_with('.')) + .unwrap_or(false) +} + +enum FileOutcome { + Dumped { lines: usize }, + SkippedNoSyntax, + SkippedNotUtf8, +} + +fn dump_file(ss: &SyntaxSet, path: &Path, rel: &Path, full: bool) -> FileOutcome { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(_) => return FileOutcome::SkippedNotUtf8, + }; + let syntax = match ss.find_syntax_for_file(path) { + Ok(Some(syntax)) => syntax, + _ => return FileOutcome::SkippedNoSyntax, + }; + + println!("== {} [{}]", rel.display(), syntax.name); + + let mut state = ParseState::new(syntax); + let mut stack = ScopeStack::new(); + let mut records: Vec = Vec::new(); + // (1-based line number, message) — printed after the per-line dump so a + // replay can't interleave with fingerprint lines. + let mut notes: Vec<(usize, String)> = Vec::new(); + + 'lines: for line in LinesWithEndings::from(&text) { + let current_line_number = records.len() + 1; + let ParseLineOutput { + ops, + replayed, + warnings, + } = match state.parse_line(line, ss) { + Ok(output) => output, + Err(e) => { + notes.push((current_line_number, format!("parse error: {}", e))); + break 'lines; + } + }; + for warning in warnings { + notes.push((current_line_number, format!("warning: {}", warning))); + } + + // Fold corrected ops for previously-parsed lines back into their + // records, rebuilding the running stack from the window base. + if !replayed.is_empty() { + let start = records.len() - replayed.len(); + stack = records[start].stack_before.clone(); + for (i, replayed_ops) in replayed.iter().enumerate() { + let record = &mut records[start + i]; + record.stack_before = stack.clone(); + match fold_line(replayed_ops, &record.text, &mut stack) { + Ok(folded) => record.folded = folded, + Err(e) => { + notes.push((start + i + 1, format!("replay fold error: {}", e))); + break 'lines; + } + } + } + } + + // Snapshot post-replay, pre-current-ops: this is the corrected + // baseline a future replay covering this line must reset to. + let stack_before = stack.clone(); + match fold_line(&ops, line, &mut stack) { + Ok(folded) => records.push(LineRecord { + text: line.to_string(), + stack_before, + folded, + }), + Err(e) => { + notes.push((current_line_number, format!("fold error: {}", e))); + break 'lines; + } + } + } + + for (i, record) in records.iter().enumerate() { + if full { + for region in record.folded.lines() { + println!("{:04} {}", i + 1, region); + } + } else { + println!("{:04} {:016x}", i + 1, fnv1a64(record.folded.as_bytes())); + } + } + for (line_number, note) in ¬es { + println!("!! line {}: {}", line_number, note); + } + + FileOutcome::Dumped { + lines: records.len(), + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let mut opts = Options::new(); + opts.optflag("", "full", "print scope regions instead of per-line hashes"); + opts.optflag("h", "help", "print this help menu"); + let matches = opts.parse(&args[1..]).unwrap_or_else(|e| { + eprintln!("{}", e); + std::process::exit(2); + }); + if matches.opt_present("h") { + println!( + "{}", + opts.usage("Usage: scopedump [--full] [TARGET] [SYNTAXES_DIR]") + ); + return; + } + let full = matches.opt_present("full"); + let target = matches + .free + .first() + .map(String::as_str) + .unwrap_or("testdata/Packages"); + let syntaxes_dir = matches + .free + .get(1) + .map(String::as_str) + .unwrap_or("testdata/Packages"); + + let mut builder = SyntaxSetBuilder::new(); + builder + .add_from_folder(syntaxes_dir, true) // with newlines, matching syntest + .unwrap(); + let ss = builder.build(); + + let mut dumped = 0usize; + let mut total_lines = 0usize; + let mut skipped_no_syntax = 0usize; + let mut skipped_not_utf8 = 0usize; + + let target_path = Path::new(target); + for entry in WalkDir::new(target_path) + .sort_by_file_name() + .into_iter() + .filter_entry(|e| e.depth() == 0 || !is_hidden(e)) + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + { + let rel = entry + .path() + .strip_prefix(target_path) + .unwrap_or(entry.path()); + match dump_file(&ss, entry.path(), rel, full) { + FileOutcome::Dumped { lines } => { + dumped += 1; + total_lines += lines; + } + FileOutcome::SkippedNoSyntax => skipped_no_syntax += 1, + FileOutcome::SkippedNotUtf8 => skipped_not_utf8 += 1, + } + } + + eprintln!( + "{} files dumped ({} lines); skipped: {} without a resolvable syntax, {} not UTF-8", + dumped, total_lines, skipped_no_syntax, skipped_not_utf8 + ); +} diff --git a/examples/syncat.rs b/examples/syncat.rs index 05a97401..e268d517 100644 --- a/examples/syncat.rs +++ b/examples/syncat.rs @@ -1,12 +1,12 @@ use getopts::Options; use std::borrow::Cow; -use std::io::BufRead; +use std::io::{self, Write}; use std::path::Path; use syntect::dumps::{dump_to_file, from_dump_file}; -use syntect::easy::HighlightFile; -use syntect::highlighting::{Style, Theme, ThemeSet}; +use syntect::highlighting::{Theme, ThemeSet}; +use syntect::io::HighlightedWriter; use syntect::parsing::SyntaxSet; -use syntect::util::as_24_bit_terminal_escaped; +use syntect::rendering::AnsiStyledOutput; fn load_theme(tm_file: &str, enable_caching: bool) -> Theme { let tm_path = Path::new(tm_file); @@ -111,34 +111,25 @@ fn main() { println!("==> {} <==", src); } - let mut highlighter = HighlightFile::new(src, &ss, &theme).unwrap(); - - // We use read_line instead of `for line in highlighter.reader.lines()` because that - // doesn't return strings with a `\n`, and including the `\n` gets us more robust highlighting. - // See the documentation for `SyntaxSetBuilder::add_from_folder`. - // It also allows re-using the line buffer, which should be a tiny bit faster. - let mut line = String::new(); - while highlighter.reader.read_line(&mut line).unwrap() > 0 { - if no_newlines && line.ends_with('\n') { - let _ = line.pop(); - } - - { - let regions: Vec<(Style, &str)> = highlighter - .highlight_lines - .highlight_line(&line, &ss) - .unwrap(); - print!("{}", as_24_bit_terminal_escaped(®ions[..], true)); - } - line.clear(); - - if no_newlines { - println!(); - } - } + let path = std::path::Path::new(src); + let mut f = std::fs::File::open(path).unwrap(); + let syntax = ss + .find_syntax_for_file(path) + .unwrap() + .unwrap_or_else(|| ss.find_syntax_plain_text()); + let mut highlighter = + HighlightedWriter::from_themed(syntax, &ss, &theme, AnsiStyledOutput::Opaque) + .with_output(io::stdout().lock()) + .build(); + + // HighlightedWriter implements `io::Write`, so we can stream the + // file straight through it without managing line buffers. + io::copy(&mut f, &mut highlighter).unwrap(); + + let mut out = highlighter.into_inner().unwrap(); // Clear the formatting - println!("\x1b[0m"); + out.write_all(b"\x1b[0m\n").unwrap(); } } } diff --git a/examples/syntest.rs b/examples/syntest.rs index f65ff25b..3d6920b3 100644 --- a/examples/syntest.rs +++ b/examples/syntest.rs @@ -45,7 +45,7 @@ pub enum SyntaxTestFileResult { } // The header-line shape Sublime Text accepts is: -// SYNTAX TEST [] "" [ []] +// SYNTAX TEST [] "" [] // // * `testtoken_start` is the line's comment marker (e.g. `//`, `#`, // `/*`, ``, `%>`, `}}`, …). Restricted -// to punctuation-only so alphabetic tails like `dotnet` in -// `#! SYNTAX TEST "…" dotnet run` don't get mis-captured. -// * The trailing `(?:\s.*)?$` absorbs whatever follows the (optional) -// testtoken_end — shebang-style instructions like `dotnet run` -// go here and are ignored. +// * `testtoken_end` is whatever follows the closing quote, with +// surrounding whitespace stripped. For block-comment syntaxes it +// is the matching closing marker (`*/`, `-->`, `%>`, `}}`, …); +// for line-comment shebang headers it is the entire trailing tail +// (`dmd`, `clojure`, `dotnet run`, `tclsh8.6`, …) — alphabetic +// and multi-word tails included. ST searches each assertion line +// for this token and clips the scope-selector text at the first +// substring match, so capturing the full tail is required to keep +// selectors like `<- keyword.operator.logical.d dmd` from leaking +// the trailing `dmd` into the selector. pub static SYNTAX_TEST_HEADER_PATTERN: LazyLock = LazyLock::new(|| { Regex::new( r#"(?xm) @@ -73,8 +76,8 @@ pub static SYNTAX_TEST_HEADER_PATTERN: LazyLock = LazyLock::new(|| { \s*SYNTAX\sTEST\s+ (?:(?P[^\s"]+)\s+)? "(?P[^"]+)" - (?:\s+(?P[^\s\w]+))? - (?:\s.*)?$ + (?:\s+(?P\S(?:.*\S)?))? + \s*$ "#, ) .unwrap() @@ -236,15 +239,20 @@ fn get_line_assertion_details<'a>( // The trailing `(.*)$` group comes after the four named alternatives, // so use the last group to stay robust against future marker additions. let mut sst = captures.get(captures.len() - 1).unwrap().as_str(); - let mut only_whitespace_after_token_end = true; if let Some(token) = testtoken_end { - // if there is an end token defined in the test file header + // ST clips the scope-selector text at the first occurrence of + // testtoken_end (substring match) — necessary so block-comment + // closers (`*/`, `-->`) and shebang tails (`dmd`, `clojure`, + // `dotnet run`) don't leak into the selector. Note this fires + // even when the token is a substring of a scope name (the + // `clojure` inside `comment.line.shebang.clojure` for the + // Clojure shebang test); ST relies on the test author not + // picking selectors whose first dotted atom shares a prefix + // with the closing token. if let Some(end_token_pos) = sst.find(token) { - // and there is an end token in the line - let (ss, after_token_end) = sst.split_at(end_token_pos); // the scope selector text ends at the end token + let (ss, _) = sst.split_at(end_token_pos); sst = ss; - only_whitespace_after_token_end = after_token_end.trim_end().is_empty(); } } // A column-range marker (`^+` or `@+`) spans the columns it covers; @@ -262,12 +270,30 @@ fn get_line_assertion_details<'a>( }; let is_reference = captures.name("reference_label").is_some() || captures.name("reference_assertion").is_some(); + // ST's syntax-test format requires assertions on dedicated + // comment-only lines: only whitespace before testtoken_start. + // When source code precedes the testtoken (e.g. bash `: ${#^pat}`, + // where `#` is the parameter-length operator and `^pat` is + // substitution), the markers that follow are coincidental code, + // not test-author intent. Such lines must not be treated as + // assertion-bearing — otherwise spurious `process_assertions` + // failures pile up against the wrong target line, and the + // misclassification also blocks `test_against_line_number` from + // advancing onto the source line, so the *next* genuine + // assertion tests against stale scopes. We don't gate on what + // follows testtoken_end: ST itself processes such lines as + // assertions (the trailing content is just ignored), and our + // tests don't include lines that mix an assertion with real + // source code after the closing token. + let is_pure_assertion_line = before_token_start.trim_start().is_empty(); + if !is_pure_assertion_line { + return None; + } return Some(AssertionRange { begin_char, end_char, scope_selector_text: sst, - is_pure_assertion_line: before_token_start.trim_start().is_empty() - && only_whitespace_after_token_end, // if only whitespace surrounds the test tokens on the line, then it is a pure assertion line + is_pure_assertion_line, is_reference, }); } @@ -278,6 +304,7 @@ fn get_line_assertion_details<'a>( fn process_assertions( assertion: &AssertionRange<'_>, test_against_line_scopes: &[ScopedText], + next_line_scopes: Option<&[ScopedText]>, ) -> Vec { // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -" // and they are sometimes in the syntax test as ^^^-comment, for example @@ -301,16 +328,59 @@ fn process_assertions( }; results.push(result); } - // don't ignore assertions after the newline, they should be treated as though they are asserting against the newline + // Past-EOL columns: ST's `view.text_point(row, col)` overflows into the + // next row when `col` exceeds the current line's char count, so its + // syntax-test framework evaluates past-EOL assertions against the + // corresponding column on the line below. Mirror that here when we have + // the next line's scopes; otherwise fall back to the last char's scope. let last = test_against_line_scopes.last().unwrap(); - if last.char_start + last.text_len < assertion.end_char { - let match_value = selector.does_match(last.scope.as_slice()); - let result = RangeTestResult { - column_begin: max(last.char_start + last.text_len, assertion.begin_char), - column_end: assertion.end_char, - success: match_value.is_some(), - }; - results.push(result); + let last_end = last.char_start + last.text_len; + if last_end < assertion.end_char { + let past_eol_begin = max(last_end, assertion.begin_char); + let past_eol_end = assertion.end_char; + if let Some(next_scopes) = next_line_scopes.filter(|s| !s.is_empty()) { + // Wrap formula: position `col` past EOL of a line of total length + // `last_end` (chars including trailing `\n`) lands on column + // `col - last_end` of the next line. + let wrap_begin = past_eol_begin - last_end; + let wrap_end = past_eol_end - last_end; + let mut covered = wrap_begin; + for scoped_text in next_scopes + .iter() + .skip_while(|s| s.char_start + s.text_len <= wrap_begin) + .take_while(|s| s.char_start < wrap_end) + { + let next_begin = max(scoped_text.char_start, wrap_begin); + let next_end = min(scoped_text.char_start + scoped_text.text_len, wrap_end); + let match_value = selector.does_match(scoped_text.scope.as_slice()); + results.push(RangeTestResult { + column_begin: next_begin + last_end, + column_end: next_end + last_end, + success: match_value.is_some(), + }); + covered = next_end; + } + // Wrap target extends past the next line's content too — recursive + // wrap is not yet implemented; fall back to the next line's last + // scope so the assertion still gets a defined verdict instead of + // silently passing. + if covered < wrap_end { + let next_last = next_scopes.last().unwrap(); + let match_value = selector.does_match(next_last.scope.as_slice()); + results.push(RangeTestResult { + column_begin: covered + last_end, + column_end: past_eol_end, + success: match_value.is_some(), + }); + } + } else { + let match_value = selector.does_match(last.scope.as_slice()); + results.push(RangeTestResult { + column_begin: past_eol_begin, + column_end: past_eol_end, + success: match_value.is_some(), + }); + } } results } @@ -368,7 +438,14 @@ fn test_file( let mut current_line_number = 1; let mut test_against_line_number = 1; - let mut scopes_on_line_being_tested = Vec::new(); + let mut scopes_on_line_being_tested: Vec = Vec::new(); + // Scopes of the first line that follows the current target line. ST's + // syntax-test framework evaluates past-EOL assertion columns against the + // corresponding column on the next line (because `text_point(row, col)` + // overflows into the next row when `col` exceeds the row's length); we + // mirror that by remembering the next line's scopes once and feeding + // them into `process_assertions`. Reset whenever the target line changes. + let mut next_target_line_scopes: Option> = None; let mut previous_non_assertion_line = line.to_string(); let mut assertion_failures: usize = 0; @@ -383,61 +460,21 @@ fn test_file( loop { // over lines of file, starting with the header line - let mut line_only_has_assertion = false; - let mut line_has_assertion = false; - if let Some(assertion) = get_line_assertion_details(testtoken_start, testtoken_end, &line) { - // `@+` and `>` lines are annotation-only (reference labels / reference - // assertions). They must be recognised so they do not drive - // `test_against_line_number`, but we do not yet implement - // cross-line label lookups, so no scope checks run here. - let mut current_assertion_failures: usize = 0; - if !assertion.is_reference { - let result = process_assertions(&assertion, &scopes_on_line_being_tested); - total_assertions += assertion.end_char - assertion.begin_char; - for failure in result.iter().filter(|r| !r.success) { - let length = failure.column_end - failure.column_begin; - let text: String = previous_non_assertion_line - .chars() - .skip(failure.column_begin) - .take(length) - .collect(); - pending_messages.push(BufferedFailureMessage { - selector_text: assertion.scope_selector_text.trim().to_string(), - assertion_line_number: current_line_number, - test_against_line_number, - column_begin: failure.column_begin, - column_end: failure.column_end, - text, - scope: scopes_on_line_being_tested - .iter() - .find(|s| s.char_start + s.text_len > failure.column_begin) - .unwrap_or_else(|| scopes_on_line_being_tested.last().unwrap()) - .scope - .clone(), - }); - assertion_failures += failure.column_end - failure.column_begin; - current_assertion_failures += failure.column_end - failure.column_begin; - } - // Buffer this assertion for re-evaluation if backtracking replays the target line - if let Some(idx) = current_test_line_buffer_idx { - if let Some(ref mut data) = parsed_line_buffer[idx].non_assertion_data { - data.assertions.push(BufferedAssertion { - begin_char: assertion.begin_char, - end_char: assertion.end_char, - scope_selector_text: assertion.scope_selector_text.to_string(), - assertion_line_number: current_line_number, - }); - data.assertion_failures += current_assertion_failures; - } - } - } // end `if !assertion.is_reference` - line_only_has_assertion = assertion.is_pure_assertion_line; - line_has_assertion = true; - } + let assertion_opt = get_line_assertion_details(testtoken_start, testtoken_end, &line); + let line_has_assertion = assertion_opt.is_some(); + let line_only_has_assertion = assertion_opt + .as_ref() + .map(|a| a.is_pure_assertion_line) + .unwrap_or(false); + + // Parse first so the just-parsed scopes are available when the + // assertion runs immediately after — needed for past-EOL wrap on the + // first assertion line after a target. if !line_only_has_assertion || parse_test_lines { if !line_has_assertion { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against scopes_on_line_being_tested.clear(); + next_target_line_scopes = None; test_against_line_number = current_line_number; previous_non_assertion_line = line.to_string(); } @@ -447,7 +484,6 @@ fn test_file( current_line_number, stack, state ); } - let stack_before = stack.clone(); let output = match state.parse_line(&line, ss) { Ok(output) => output, Err(e) => { @@ -496,6 +532,19 @@ fn test_file( // Reset stack to the state before the first replayed line stack = parsed_line_buffer[start_idx].stack_before.clone(); + // Collect corrected baselines to apply post-loop, since the + // mutable record borrow inside the loop forbids touching + // sibling buffer entries. After applying replayed[i], + // `stack` is the corrected end-of-line for the buffered + // record at start_idx + i, i.e. the corrected + // start-of-line baseline for record at start_idx + i + 1. + // Overwriting that baseline prevents a future replay from + // resurrecting any meta_scope the prior replay had unwound + // (observed as meta.link.reference.def.markdown leak past + // back-to-back Markdown link reference definitions). + let buf_len = parsed_line_buffer.len(); + let mut corrected_baselines: Vec<(usize, ScopeStack)> = Vec::new(); + for (i, replayed_ops) in replayed.iter().enumerate() { let record = &mut parsed_line_buffer[start_idx + i]; let has_non_assertion = record.non_assertion_data.is_some(); @@ -516,6 +565,10 @@ fn test_file( col += len; } } + let next_idx = start_idx + i + 1; + if next_idx < buf_len { + corrected_baselines.push((next_idx, stack.clone())); + } if let Some(ref mut data) = record.non_assertion_data { if !data.assertions.is_empty() { @@ -532,7 +585,12 @@ fn test_file( // above, so replays never hit reference lines. is_reference: false, }; - let result = process_assertions(&temp_assertion, &new_scoped); + // Replay path: fall back to the previous + // past-EOL semantics by passing no next-line + // scopes. Replays are rare and per-target; + // recomputing the next line's scopes here + // would require also replaying its ops. + let result = process_assertions(&temp_assertion, &new_scoped, None); for failure in result.iter().filter(|r| !r.success) { let length = failure.column_end - failure.column_begin; let text: String = record @@ -574,6 +632,9 @@ fn test_file( } } } + for (idx, corrected) in corrected_baselines { + parsed_line_buffer[idx].stack_before = corrected; + } } if out_opts.debug && !line_only_has_assertion { @@ -583,6 +644,18 @@ fn test_file( debug_print_ops(&line, &ops); } } + // Snapshot the stack now (post-replays, pre-current-ops) — this + // becomes the buffered `stack_before` for the current line, so a + // future replay covering it resets to the corrected baseline + // rather than the stale pre-parse value. + let stack_before = stack.clone(); + // Build the just-parsed line's scopes. For non-assertion (target) + // lines they go into `scopes_on_line_being_tested`; for the FIRST + // assertion line that follows a target they go into a fresh vec + // that becomes `next_target_line_scopes` (used for past-EOL wrap). + // Subsequent assertion lines don't need their scopes captured. + let capture_for_next_target = line_has_assertion && next_target_line_scopes.is_none(); + let mut next_target_buffer: Vec = Vec::new(); let mut col: usize = 0; for (s, op) in ScopeRegionIterator::new(&ops, &line) { if let Err(_) = stack.apply(op) { @@ -599,17 +672,23 @@ fn test_file( // in this case we don't care about blank tokens continue; } + let len = s.chars().count(); + let scoped = ScopedText { + char_start: col, + text_len: len, + scope: stack.as_slice().to_vec(), + }; if !line_has_assertion { // if the line has no assertions on it, remember the scopes on the line so we can test against them later - let len = s.chars().count(); - scopes_on_line_being_tested.push(ScopedText { - char_start: col, - text_len: len, - scope: stack.as_slice().to_vec(), - }); - // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired - col += len; + scopes_on_line_being_tested.push(scoped); + } else if capture_for_next_target { + next_target_buffer.push(scoped); } + // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired + col += len; + } + if capture_for_next_target { + next_target_line_scopes = Some(next_target_buffer); } // Buffer this parsed line for potential future replay @@ -630,13 +709,67 @@ fn test_file( if !line_has_assertion { current_test_line_buffer_idx = Some(parsed_line_buffer.len() - 1); } + } - // Flush buffered failure messages once the parser commits - if !state.is_speculative() { - flush_pending_messages(&mut pending_messages, out_opts.summary); + // Process the assertion (after parsing, so past-EOL wrap can see the + // current line's scopes via `next_target_line_scopes`). + if let Some(assertion) = assertion_opt { + // `@+` and `>` lines are annotation-only (reference labels / + // reference assertions). They must be recognised so they do not + // drive `test_against_line_number`, but we do not yet implement + // cross-line label lookups, so no scope checks run here. + let mut current_assertion_failures: usize = 0; + if !assertion.is_reference { + let result = process_assertions( + &assertion, + &scopes_on_line_being_tested, + next_target_line_scopes.as_deref(), + ); + total_assertions += assertion.end_char - assertion.begin_char; + for failure in result.iter().filter(|r| !r.success) { + let length = failure.column_end - failure.column_begin; + let text: String = previous_non_assertion_line + .chars() + .skip(failure.column_begin) + .take(length) + .collect(); + pending_messages.push(BufferedFailureMessage { + selector_text: assertion.scope_selector_text.trim().to_string(), + assertion_line_number: current_line_number, + test_against_line_number, + column_begin: failure.column_begin, + column_end: failure.column_end, + text, + scope: scopes_on_line_being_tested + .iter() + .find(|s| s.char_start + s.text_len > failure.column_begin) + .unwrap_or_else(|| scopes_on_line_being_tested.last().unwrap()) + .scope + .clone(), + }); + assertion_failures += failure.column_end - failure.column_begin; + current_assertion_failures += failure.column_end - failure.column_begin; + } + // Buffer this assertion for re-evaluation if backtracking replays the target line + if let Some(idx) = current_test_line_buffer_idx { + if let Some(ref mut data) = parsed_line_buffer[idx].non_assertion_data { + data.assertions.push(BufferedAssertion { + begin_char: assertion.begin_char, + end_char: assertion.end_char, + scope_selector_text: assertion.scope_selector_text.to_string(), + assertion_line_number: current_line_number, + }); + data.assertion_failures += current_assertion_failures; + } + } } } + // Flush buffered failure messages once the parser commits + if !state.is_speculative() { + flush_pending_messages(&mut pending_messages, out_opts.summary); + } + line.clear(); current_line_number += 1; if reader.read_line(&mut line).unwrap() == 0 { @@ -760,12 +893,6 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 { files.sort(); for path in &files { - if let Some(reason) = should_skip(path) { - if !out_opts.summary { - println!("Skipping file {}: {}", path.display(), reason); - } - continue; - } if !out_opts.summary { println!("Testing file {}", path.display()); } @@ -805,25 +932,6 @@ fn is_a_syntax_test_file(entry: &DirEntry) -> bool { .unwrap_or(false) } -/// Return `Some(reason)` for syntax test files that are known to hang the -/// parser. These entries exist as a temporary CI unblocker until the -/// underlying parser loop protection is extended to cover the triggering -/// patterns. See `slow-perl.md` (at the repository root, intentionally -/// untracked) for the investigation notes. -fn should_skip(path: &Path) -> Option<&'static str> { - const SKIP: &[(&str, &str)] = &[( - "Perl/syntax_test_perl.pl", - "POD-embedded language sections hang the parser", - )]; - let s = path.to_string_lossy(); - for (suffix, reason) in SKIP { - if s.ends_with(suffix) { - return Some(*reason); - } - } - None -} - #[cfg(test)] mod tests { //! Unit tests for `get_line_assertion_details`. @@ -923,4 +1031,119 @@ mod tests { let a = details(CC, None, "// @@@").unwrap(); assert!(a.is_reference); } + #[test] + fn shell_parameter_length_is_not_mistaken_for_column_range() { + // bash `: ${#^pattern}` — `#` is the parameter-length operator, not + // a comment. Source code precedes the testtoken, so the line is not + // a pure-assertion line and must not be processed as an assertion. + assert!(details(HASH, None, ": ${#^pattern}").is_none()); + assert!(details(HASH, None, ": ${#^^pattern}").is_none()); + } + #[test] + fn trailing_comment_with_left_arrow_is_not_mistaken_for_assertion() { + // bash `[ < ScopedText { + ScopedText { + char_start, + text_len, + scope: scopes + .split_whitespace() + .map(|s| Scope::new(s).unwrap()) + .collect(), + } + } + fn assertion_range(begin_char: usize, end_char: usize, sel: &str) -> AssertionRange<'_> { + AssertionRange { + begin_char, + end_char, + scope_selector_text: sel, + is_pure_assertion_line: true, + is_reference: false, + } + } + + #[test] + fn past_eol_negative_assertion_uses_next_line_wrap() { + // git_config-shaped: the consumed `\n` of the target line still + // carries the parent meta_scope (`meta.section`), but the next line + // is a comment whose scope does not include `meta.section`. + // Negative past-EOL assertion `- meta.section` must pass. + let target = vec![ + st(0, 1, "text meta.section punctuation.section.brackets.begin"), + st( + 28, + 1, + "text meta.section meta.brackets invalid.illegal.unexpected.eol", + ), + ]; + let next = vec![ + st(0, 1, "text comment.line punctuation.definition.comment"), + st(1, 1, "text comment.line"), + ]; + let a = assertion_range(29, 30, "- meta.section"); + let r = process_assertions(&a, &target, Some(&next)); + assert_eq!(r.len(), 1); + assert!(r[0].success, "expected pass via wrap to next line: {r:?}"); + } + + #[test] + fn past_eol_positive_assertion_matches_next_line_scope() { + // Past-EOL assertion expecting `comment.line` finds it on the next + // line — both git_config (lines 554-555) and Clojure (line 33) work + // this way in ST. + let target = vec![ + st( + 0, + 1, + "text meta.mapping.value string.quoted.double punctuation.definition.string.end", + ), + st(81, 1, "text"), + ]; + let next = vec![ + st(0, 1, "text comment.line punctuation.definition.comment"), + st(1, 14, "text comment.line"), + ]; + let a = assertion_range(82, 97, "comment.line"); + let r = process_assertions(&a, &target, Some(&next)); + assert!( + r.iter().all(|res| res.success), + "expected all wrap segments to pass: {r:?}", + ); + } + + #[test] + fn past_eol_falls_back_when_next_line_unavailable() { + // Without next-line scopes (end-of-file or replay path), keep the + // previous behaviour of testing against the last char's scope. + let target = vec![st(0, 5, "text constant.numeric"), st(5, 1, "text")]; + let a_pass = assertion_range(6, 7, "- constant.numeric"); + let r_pass = process_assertions(&a_pass, &target, None); + assert_eq!(r_pass.len(), 1); + assert!(r_pass[0].success); + let a_fail = assertion_range(6, 7, "constant.numeric"); + let r_fail = process_assertions(&a_fail, &target, None); + assert_eq!(r_fail.len(), 1); + assert!(!r_fail[0].success); + } + + #[test] + fn past_eol_wrap_overshooting_next_line_uses_next_line_last_scope() { + // When the wrap target extends past the next line's content too, + // fall back to the next line's last scope (recursive wrap is a v2 + // concern). The verdict must still be defined, not silently skipped. + let target = vec![st(0, 1, "text"), st(1, 1, "text")]; + let next = vec![st(0, 3, "text source.x")]; + // Wrap range: cols [2, 8) on target → cols [0, 6) on next. + // Cols [3, 6) on next overshoot the 3-char `next` and should fall + // back to the next-line last scope (`text source.x`). + let a = assertion_range(2, 8, "source.x"); + let r = process_assertions(&a, &target, Some(&next)); + assert!(r.iter().all(|res| res.success), "got: {r:?}"); + } } diff --git a/src/easy.rs b/src/easy.rs index 3cd3dd21..526222e5 100644 --- a/src/easy.rs +++ b/src/easy.rs @@ -40,12 +40,17 @@ use std::path::Path; /// print!("{}", escaped); /// } /// ``` +#[deprecated( + since = "6.0.0", + note = "Use `syntect::io::HighlightedWriter` instead, which implements `std::io::Write`, supports pluggable renderers, and handles branch-point backtracking" +)] pub struct HighlightLines<'a> { highlighter: Highlighter<'a>, parse_state: ParseState, highlight_state: HighlightState, } +#[allow(deprecated)] impl<'a> HighlightLines<'a> { pub fn new(syntax: &SyntaxReference, theme: &'a Theme) -> HighlightLines<'a> { let highlighter = Highlighter::new(theme); @@ -111,11 +116,17 @@ impl<'a> HighlightLines<'a> { /// the [`new`] method docs for more information. /// /// [`new`]: #method.new +#[deprecated( + since = "6.0.0", + note = "Use `syntect::io::HighlightedWriter` instead, which implements `std::io::Write`, supports pluggable renderers, and handles branch-point backtracking" +)] pub struct HighlightFile<'a> { pub reader: BufReader, + #[allow(deprecated)] pub highlight_lines: HighlightLines<'a>, } +#[allow(deprecated)] impl<'a> HighlightFile<'a> { /// Constructs a file reader and a line highlighter to get you reading files as fast as possible. /// @@ -300,6 +311,7 @@ mod tests { #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))] #[test] + #[allow(deprecated)] fn can_highlight_lines() { let ss = SyntaxSet::load_defaults_nonewlines(); let ts = ThemeSet::load_defaults(); @@ -313,6 +325,7 @@ mod tests { #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))] #[test] + #[allow(deprecated)] fn can_highlight_file() { let ss = SyntaxSet::load_defaults_nonewlines(); let ts = ThemeSet::load_defaults(); @@ -379,6 +392,7 @@ mod tests { #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))] #[test] + #[allow(deprecated)] fn can_start_again_from_previous_state() { let ss = SyntaxSet::load_defaults_nonewlines(); let ts = ThemeSet::load_defaults(); diff --git a/src/html.rs b/src/html.rs index 69a32d61..17bbd981 100644 --- a/src/html.rs +++ b/src/html.rs @@ -1,32 +1,68 @@ //! Rendering highlighted code as HTML+CSS -use crate::easy::{HighlightFile, HighlightLines}; use crate::escape::Escape; use crate::highlighting::{Color, FontStyle, Style, Theme}; +use crate::io::HighlightedWriter; use crate::parsing::{ - lock_global_scope_repo, BasicScopeStackOp, ParseState, Scope, ScopeStack, ScopeStackOp, - SyntaxReference, SyntaxSet, + lock_global_scope_repo, Scope, ScopeRepository, ScopeStack, ScopeStackOp, SyntaxReference, + SyntaxSet, }; -use crate::util::LinesWithEndings; +use crate::rendering::{render_line, MarkupAdapter, ScopeMarkup, StyledOutput}; use crate::Error; use std::fmt::Write; -use std::io::BufRead; +use std::io::Write as IoWrite; use std::path::Path; -/// Output HTML for a line of code with `` elements using class names -/// -/// Because this has to keep track of open and closed `` tags, it is a `struct` with -/// additional state. +/// An HTML renderer that produces `` elements with +/// CSS class names derived from scope atoms. +pub struct ClassedHTMLScopeRenderer { + style: ClassStyle, +} + +impl ClassedHTMLScopeRenderer { + pub fn new(style: ClassStyle) -> Self { + Self { style } + } +} + +impl ScopeMarkup for ClassedHTMLScopeRenderer { + fn begin_scope(&mut self, atom_strs: &[&str], output: &mut String) { + output.push_str(""); + } + + fn end_scope(&mut self, output: &mut String) { + output.push_str(""); + } + + fn write_text(&mut self, text: &str, output: &mut String) { + write!(output, "{}", Escape(text)).expect("writing to a String never fails"); + } +} + +/// HTML-specific convenience wrapper around [`HighlightedWriter`]. /// -/// There is a [`finalize()`] method that must be called in the end in order -/// to close all open `` tags. +/// Uses [`ClassedHTMLScopeRenderer`] to produce `` output with +/// CSS class names derived from scope atoms. /// /// Note that because CSS classes have slightly different matching semantics /// than Textmate themes, this may produce somewhat less accurate /// highlighting than the other highlighting functions which directly use /// inline colors as opposed to classes and a stylesheet. /// -/// [`finalize()`]: #method.finalize +/// There is a [`finalize()`] method that must be called in the end in order +/// to close all open `` tags. +/// +/// [`finalize()`]: ClassedHTMLGenerator::finalize /// /// # Example /// @@ -50,12 +86,7 @@ use std::path::Path; /// let output_html = html_generator.finalize(); /// ``` pub struct ClassedHTMLGenerator<'a> { - syntax_set: &'a SyntaxSet, - open_spans: isize, - parse_state: ParseState, - scope_stack: ScopeStack, - html: String, - style: ClassStyle, + inner: HighlightedWriter<'a, MarkupAdapter>, } impl<'a> ClassedHTMLGenerator<'a> { @@ -72,17 +103,13 @@ impl<'a> ClassedHTMLGenerator<'a> { syntax_set: &'a SyntaxSet, style: ClassStyle, ) -> ClassedHTMLGenerator<'a> { - let parse_state = ParseState::new(syntax_reference); - let open_spans = 0; - let html = String::new(); - let scope_stack = ScopeStack::new(); ClassedHTMLGenerator { - syntax_set, - open_spans, - parse_state, - scope_stack, - html, - style, + inner: HighlightedWriter::from_markup( + syntax_reference, + syntax_set, + ClassedHTMLScopeRenderer::new(style), + ) + .build(), } } @@ -91,16 +118,7 @@ impl<'a> ClassedHTMLGenerator<'a> { /// *Note:* This function requires `line` to include a newline at the end and /// also use of the `load_defaults_newlines` version of the syntaxes. pub fn parse_html_for_line_which_includes_newline(&mut self, line: &str) -> Result<(), Error> { - let parsed_line = self.parse_state.parse_line(line, self.syntax_set)?.ops; - let (formatted_line, delta) = line_tokens_to_classed_spans( - line, - parsed_line.as_slice(), - self.style, - &mut self.scope_stack, - )?; - self.open_spans += delta; - self.html.push_str(formatted_line.as_str()); - + self.inner.write_all(line.as_bytes())?; Ok(()) } @@ -118,18 +136,23 @@ impl<'a> ClassedHTMLGenerator<'a> { note = "Please use `parse_html_for_line_which_includes_newline` instead" )] pub fn parse_html_for_line(&mut self, line: &str) { - self.parse_html_for_line_which_includes_newline(line) + // The deprecated quirk: append a newline so the renderer treats the + // input as a complete line and the output ends with `\n`. + let mut owned = String::with_capacity(line.len() + 1); + owned.push_str(line); + owned.push('\n'); + self.inner + .write_all(owned.as_bytes()) .expect("Please use `parse_html_for_line_which_includes_newline` instead"); - // retain newline - self.html.push('\n'); } - /// Close all open `` tags and return the finished HTML string - pub fn finalize(mut self) -> String { - for _ in 0..self.open_spans { - self.html.push_str(""); - } - self.html + /// Close any remaining open `` tags and return the finished HTML. + pub fn finalize(self) -> String { + let bytes = self + .inner + .into_inner() + .expect("renderer produces valid UTF-8"); + String::from_utf8(bytes).expect("renderer produces valid UTF-8") } } @@ -177,11 +200,12 @@ pub fn css_for_theme_with_class_style(theme: &Theme, style: ClassStyle) -> Resul } css.push_str("}\n\n"); + let repo = lock_global_scope_repo(); for i in &theme.scopes { for scope_selector in &i.scope.selectors { let scopes = scope_selector.extract_scopes(); for k in &scopes { - scope_to_selector(&mut css, *k, style); + scope_to_selector(&mut css, *k, style, &repo); css.push(' '); // join multiple scopes } css.pop(); // remove trailing space @@ -241,26 +265,7 @@ pub enum ClassStyle { SpacedPrefixed { prefix: &'static str }, } -fn scope_to_classes(s: &mut String, scope: Scope, style: ClassStyle) { - let repo = lock_global_scope_repo(); - for i in 0..(scope.len()) { - let atom = scope.atom_at(i as usize); - let atom_s = repo.atom_str(atom); - if i != 0 { - s.push(' ') - } - match style { - ClassStyle::Spaced => {} - ClassStyle::SpacedPrefixed { prefix } => { - s.push_str(prefix); - } - } - s.push_str(atom_s); - } -} - -fn scope_to_selector(s: &mut String, scope: Scope, style: ClassStyle) { - let repo = lock_global_scope_repo(); +fn scope_to_selector(s: &mut String, scope: Scope, style: ClassStyle, repo: &ScopeRepository) { for i in 0..(scope.len()) { let atom = scope.atom_at(i as usize); let atom_s = repo.atom_str(atom); @@ -294,9 +299,117 @@ fn escape_css_identifier(identifier: &str) -> String { ) } -/// Convenience method that combines `start_highlighted_html_snippet`, `styled_line_to_highlighted_html` -/// and `HighlightLines` from `syntect::easy` to create a full highlighted HTML snippet for -/// a string (which can contain many lines). +/// Output HTML for a line of code with `` elements +/// specifying classes for each token. The span elements are nested +/// like the scope stack and the scopes are mapped to classes based +/// on the `ClassStyle` (see it's docs). +/// +/// **Deprecated.** This function cannot correctly handle cross-line +/// branch-point failures: when the parser retroactively replays a span of +/// previously-emitted ops, the rendered output for those lines has already +/// been written to the caller and cannot be retracted. Use +/// [`ClassedHTMLGenerator`] (or [`crate::io::HighlightedWriter::from_markup`] +/// directly) instead — both buffer rendered output during speculative +/// parsing and replay corrected ops once speculation resolves. +/// +/// Returns the HTML string and the number of `` tags opened +/// (negative for closed). So that you can emit the correct number of closing +/// tags at the end. +#[deprecated( + since = "6.0.0", + note = "Cannot correctly handle cross-line branch-point failures. Use `ClassedHTMLGenerator` or `HighlightedWriter::from_markup` instead, which buffer rendered output during speculative parsing and replay corrected ops." +)] +pub fn line_tokens_to_classed_spans( + line: &str, + ops: &[(usize, ScopeStackOp)], + style: ClassStyle, + stack: &mut ScopeStack, +) -> Result<(String, isize), Error> { + let mut renderer = MarkupAdapter::new(ClassedHTMLScopeRenderer::new(style)); + render_line(line, ops, stack, &mut renderer, 0) +} + +// --------------------------------------------------------------------------- +// HtmlStyledOutput — inline-styled HTML via StyledOutput +// --------------------------------------------------------------------------- + +/// A [`StyledOutput`] that produces `` elements with +/// inline CSS resolved from a theme. +/// +/// Wrap with [`crate::rendering::ThemedRenderer`] to use it as a +/// [`crate::rendering::ScopeRenderer`], or pass directly to +/// [`HighlightedWriter::from_themed`]. +/// +/// `default_bg` is the background colour of the containing element; the +/// `background-color` CSS property is only emitted when the token's +/// background differs from this default. +/// +/// Adjacent text tokens with the same resolved [`Style`] are automatically +/// merged into a single ``. Whitespace-only tokens additionally fold +/// into the previous span when their backgrounds match, even when the +/// foreground differs — whitespace doesn't reveal a foreground difference. +pub struct HtmlStyledOutput { + default_bg: Color, +} + +impl HtmlStyledOutput { + /// Create a new inline HTML emitter. + /// + /// `default_bg` is the background colour of the containing element. + pub fn new(default_bg: Color) -> Self { + Self { default_bg } + } + + /// Returns the configured default background colour. + pub fn default_bg(&self) -> Color { + self.default_bg + } +} + +impl StyledOutput for HtmlStyledOutput { + fn begin_style(&mut self, style: Style, output: &mut String) { + output.push_str(""); + } + + fn end_style(&mut self, output: &mut String) { + output.push_str(""); + } + + fn write_text(&mut self, text: &str, output: &mut String) { + write!(output, "{}", Escape(text)).expect("writing to a String never fails"); + } + + fn should_merge(&self, prev: Style, next: Style, text: &str) -> bool { + // Merge into the previous span when styles match exactly, or when + // the text is only whitespace and the background colours agree. + // Whitespace reveals no foreground difference, so collapsing the + // boundary is safe and produces smaller output. + prev == next || (prev.background == next.background && text.trim().is_empty()) + } +} + +/// Convenience method that creates a full highlighted HTML snippet for +/// a string (which can contain many lines), using inline `style` attributes. +/// +/// Uses [`HighlightedWriter`] with [`HtmlStyledOutput`] internally, +/// which correctly handles branch-point backtracking. /// /// Note that the `syntax` passed in must be from a `SyntaxSet` compiled for newline characters. /// This is easy to get with `SyntaxSet::load_defaults_newlines()`. (Note: this was different before v3.0) @@ -306,24 +419,20 @@ pub fn highlighted_html_for_string( syntax: &SyntaxReference, theme: &Theme, ) -> Result { - let mut highlighter = HighlightLines::new(syntax, theme); let (mut output, bg) = start_highlighted_html_snippet(theme); - - for line in LinesWithEndings::from(s) { - let regions = highlighter.highlight_line(line, ss)?; - append_highlighted_html_for_styled_line( - ®ions[..], - IncludeBackground::IfDifferent(bg), - &mut output, - )?; - } + let mut w = + HighlightedWriter::from_themed(syntax, ss, theme, HtmlStyledOutput::new(bg)).build(); + w.write_all(s.as_bytes())?; + output.push_str(&String::from_utf8(w.into_inner()?).expect("renderer produces valid UTF-8")); output.push_str("\n"); Ok(output) } -/// Convenience method that combines `start_highlighted_html_snippet`, `styled_line_to_highlighted_html` -/// and `HighlightFile` from `syntect::easy` to create a full highlighted HTML snippet for -/// a file. +/// Convenience method that creates a full highlighted HTML snippet for +/// a file, using inline `style` attributes. +/// +/// Uses [`HighlightedWriter`] with [`HtmlStyledOutput`] internally, +/// which correctly handles branch-point backtracking. /// /// Note that the `syntax` passed in must be from a `SyntaxSet` compiled for newline characters. /// This is easy to get with `SyntaxSet::load_defaults_newlines()`. (Note: this was different before v3.0) @@ -332,114 +441,48 @@ pub fn highlighted_html_for_file>( ss: &SyntaxSet, theme: &Theme, ) -> Result { - let mut highlighter = HighlightFile::new(path, ss, theme)?; - let (mut output, bg) = start_highlighted_html_snippet(theme); + let path_ref: &Path = path.as_ref(); + let mut f = std::fs::File::open(path_ref)?; + let syntax = ss + .find_syntax_for_file(path_ref)? + .unwrap_or_else(|| ss.find_syntax_plain_text()); - let mut line = String::new(); - while highlighter.reader.read_line(&mut line)? > 0 { - { - let regions = highlighter.highlight_lines.highlight_line(&line, ss)?; - append_highlighted_html_for_styled_line( - ®ions[..], - IncludeBackground::IfDifferent(bg), - &mut output, - )?; - } - line.clear(); - } + let (mut output, bg) = start_highlighted_html_snippet(theme); + let mut w = + HighlightedWriter::from_themed(syntax, ss, theme, HtmlStyledOutput::new(bg)).build(); + std::io::copy(&mut f, &mut w)?; + output.push_str(&String::from_utf8(w.into_inner()?).expect("renderer produces valid UTF-8")); output.push_str("\n"); Ok(output) } -/// Output HTML for a line of code with `` elements -/// specifying classes for each token. The span elements are nested -/// like the scope stack and the scopes are mapped to classes based -/// on the `ClassStyle` (see it's docs). -/// -/// See `ClassedHTMLGenerator` for a more convenient wrapper, this is the advanced -/// version of the function that gives more control over the parsing flow. -/// -/// For this to work correctly you must concatenate all the lines in a `
`
-/// tag since some span tags opened on a line may not be closed on that line
-/// and later lines may close tags from previous lines.
-///
-/// Returns the HTML string and the number of `` tags opened
-/// (negative for closed). So that you can emit the correct number of closing
-/// tags at the end.
-pub fn line_tokens_to_classed_spans(
-    line: &str,
-    ops: &[(usize, ScopeStackOp)],
-    style: ClassStyle,
-    stack: &mut ScopeStack,
-) -> Result<(String, isize), Error> {
-    let mut s = String::with_capacity(line.len() + ops.len() * 8); // a guess
-    let mut cur_index = 0;
-    let mut span_delta = 0;
-
-    // check and skip emty inner  tags
-    let mut span_empty = false;
-    let mut span_start = 0;
-
-    for &(i, ref op) in ops {
-        if i > cur_index {
-            span_empty = false;
-            write!(s, "{}", Escape(&line[cur_index..i]))?;
-            cur_index = i
-        }
-        stack.apply_with_hook(op, |basic_op, _| match basic_op {
-            BasicScopeStackOp::Push(scope) => {
-                span_start = s.len();
-                span_empty = true;
-                s.push_str("");
-                span_delta += 1;
-            }
-            BasicScopeStackOp::Pop => {
-                if !span_empty {
-                    s.push_str("");
-                } else {
-                    s.truncate(span_start);
-                }
-                span_delta -= 1;
-                span_empty = false;
-            }
-        })?;
-    }
-    write!(s, "{}", Escape(&line[cur_index..line.len()]))?;
-    Ok((s, span_delta))
-}
-
-/// Preserved for compatibility, always use `line_tokens_to_classed_spans`
-/// and keep a `ScopeStack` between lines for correct highlighting that won't
-/// sometimes crash.
+/// Preserved for compatibility. Use [`ClassedHTMLGenerator`] instead.
 #[deprecated(
     since = "4.6.0",
-    note = "Use `line_tokens_to_classed_spans` instead, this can panic and highlight incorrectly"
+    note = "Use `ClassedHTMLGenerator` instead, this can panic and highlight incorrectly and cannot handle cross-line branch-point failures"
 )]
+#[allow(deprecated)]
 pub fn tokens_to_classed_spans(
     line: &str,
     ops: &[(usize, ScopeStackOp)],
     style: ClassStyle,
 ) -> (String, isize) {
-    line_tokens_to_classed_spans(line, ops, style, &mut ScopeStack::new()).expect(
-        "Use `line_tokens_to_classed_spans` instead, this can panic and highlight incorrectly",
-    )
+    line_tokens_to_classed_spans(line, ops, style, &mut ScopeStack::new())
+        .expect("Use `ClassedHTMLGenerator` instead")
 }
 
 #[deprecated(
     since = "3.1.0",
-    note = "Use `line_tokens_to_classed_spans` instead to avoid incorrect highlighting and panics"
+    note = "Use `ClassedHTMLGenerator` instead to avoid incorrect highlighting, panics, and cross-line branch-point failures"
 )]
+#[allow(deprecated)]
 pub fn tokens_to_classed_html(
     line: &str,
     ops: &[(usize, ScopeStackOp)],
     style: ClassStyle,
 ) -> String {
     line_tokens_to_classed_spans(line, ops, style, &mut ScopeStack::new())
-        .expect(
-            "Use `line_tokens_to_classed_spans` instead to avoid incorrect highlighting and panics",
-        )
+        .expect("Use `ClassedHTMLGenerator` instead")
         .0
 }
 
@@ -471,18 +514,20 @@ fn write_css_color(s: &mut String, c: Color) {
 /// # Examples
 ///
 /// ```
-/// use syntect::easy::HighlightLines;
-/// use syntect::parsing::SyntaxSet;
-/// use syntect::highlighting::{ThemeSet, Style};
+/// use syntect::highlighting::{HighlightIterator, HighlightState, Highlighter, Style, ThemeSet};
+/// use syntect::parsing::{ParseState, ScopeStack, SyntaxSet};
 /// use syntect::html::{styled_line_to_highlighted_html, IncludeBackground};
 ///
-/// // Load these once at the start of your program
 /// let ps = SyntaxSet::load_defaults_newlines();
 /// let ts = ThemeSet::load_defaults();
 ///
 /// let syntax = ps.find_syntax_by_name("Ruby").unwrap();
-/// let mut h = HighlightLines::new(syntax, &ts.themes["base16-ocean.dark"]);
-/// let regions = h.highlight_line("5", &ps).unwrap();
+/// let mut parse_state = ParseState::new(syntax);
+/// let highlighter = Highlighter::new(&ts.themes["base16-ocean.dark"]);
+/// let mut highlight_state = HighlightState::new(&highlighter, ScopeStack::new());
+/// let ops = parse_state.parse_line("5", &ps).unwrap().ops;
+/// let iter = HighlightIterator::new(&mut highlight_state, &ops, "5", &highlighter);
+/// let regions: Vec<(Style, &str)> = iter.collect();
 /// let html = styled_line_to_highlighted_html(®ions[..], IncludeBackground::No).unwrap();
 /// assert_eq!(html, "5");
 /// ```
@@ -579,6 +624,7 @@ mod tests {
     use crate::parsing::{ParseState, ScopeStack, SyntaxDefinition, SyntaxSet, SyntaxSetBuilder};
     use crate::util::LinesWithEndings;
     #[test]
+    #[allow(deprecated)]
     fn tokens() {
         let ss = SyntaxSet::load_defaults_newlines();
         let syntax = ss.find_syntax_by_name("Markdown").unwrap();
@@ -751,4 +797,340 @@ fn main() {
         assert!(!css.contains(".c++"));
         assert!(css.contains(".c\\2b \\2b "));
     }
+
+    #[test]
+    fn test_custom_renderer_receives_atom_strs() {
+        use crate::rendering::ScopeMarkup;
+        use std::cell::RefCell;
+
+        struct CapturingMarkup {
+            captured: RefCell>>,
+        }
+        impl ScopeMarkup for CapturingMarkup {
+            fn begin_scope(&mut self, atom_strs: &[&str], output: &mut String) {
+                self.captured
+                    .borrow_mut()
+                    .push(atom_strs.iter().map(|s| s.to_string()).collect());
+                output.push_str("");
+            }
+            fn end_scope(&mut self, output: &mut String) {
+                output.push_str("");
+            }
+        }
+
+        let code = "x + y\n";
+        let syntax_set = SyntaxSet::load_defaults_newlines();
+        let syntax = syntax_set.find_syntax_by_name("R").unwrap();
+        let markup = CapturingMarkup {
+            captured: RefCell::new(Vec::new()),
+        };
+        let mut gen =
+            crate::io::HighlightedWriter::from_markup(syntax, &syntax_set, markup).build();
+        gen.write_all(code.as_bytes()).expect("#[cfg(test)]");
+
+        // The R syntax should produce "source r" as the first scope
+        let captured = gen.renderer().inner().captured.borrow().clone();
+        assert!(!captured.is_empty());
+        assert_eq!(captured[0], vec!["source", "r"]);
+
+        gen.into_inner().expect("#[cfg(test)]");
+    }
+
+    #[test]
+    fn test_scope_with_atom_strs() {
+        let scope = Scope::new("keyword.operator.arithmetic").unwrap();
+        let atoms: Vec =
+            scope.with_atom_strs(|atoms| atoms.iter().map(|s| s.to_string()).collect());
+        assert_eq!(atoms, vec!["keyword", "operator", "arithmetic"]);
+    }
+
+    /// Synthetic syntax + theme used by `HtmlStyledOutput` byte-exact tests.
+    /// Defining them inline (rather than relying on bundled defaults) keeps
+    /// the assertions stable across syntax/theme updates.
+    #[allow(clippy::type_complexity)]
+    fn make_pinned_ss_and_theme() -> (SyntaxSet, Theme, crate::highlighting::Color) {
+        use crate::highlighting::{
+            Color, FontStyle, ScopeSelectors, StyleModifier, Theme, ThemeItem, ThemeSettings,
+        };
+        use std::str::FromStr;
+
+        let syntax_str = r#"
+name: PingThing
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+    - match: 'thing'
+      scope: lit.thing
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+
+        let bg = Color {
+            r: 0,
+            g: 0,
+            b: 0,
+            a: 0xff,
+        };
+        let theme = Theme {
+            name: Some("html-pin".into()),
+            author: None,
+            settings: ThemeSettings {
+                foreground: Some(Color {
+                    r: 200,
+                    g: 200,
+                    b: 200,
+                    a: 0xff,
+                }),
+                background: Some(bg),
+                ..Default::default()
+            },
+            scopes: vec![
+                ThemeItem {
+                    scope: ScopeSelectors::from_str("kw").unwrap(),
+                    style: StyleModifier {
+                        foreground: Some(Color {
+                            r: 11,
+                            g: 22,
+                            b: 33,
+                            a: 0xff,
+                        }),
+                        background: None,
+                        font_style: Some(FontStyle::BOLD),
+                    },
+                },
+                ThemeItem {
+                    scope: ScopeSelectors::from_str("lit").unwrap(),
+                    style: StyleModifier {
+                        foreground: Some(Color {
+                            r: 44,
+                            g: 55,
+                            b: 66,
+                            a: 0xff,
+                        }),
+                        background: None,
+                        font_style: Some(FontStyle::ITALIC),
+                    },
+                },
+            ],
+        };
+        (ss, theme, bg)
+    }
+
+    #[test]
+    fn html_styled_output_emits_byte_exact_inline_spans() {
+        // Pin the exact `` markup for a known input. Catches
+        // mutants in `HtmlStyledOutput::begin_style` (e.g. dropping the
+        // `color:` attribute, swapping `background-color` for `color`, omitting
+        // the closing `;\">`), in `should_merge` (the boolean flips would
+        // either over-merge into one big span or under-merge into many tiny
+        // ones), and in `end_style` (which must emit ``).
+        let (ss, theme, _bg) = make_pinned_ss_and_theme();
+        let syntax = &ss.syntaxes()[0];
+
+        let html = highlighted_html_for_string("ping thing\n", &ss, syntax, &theme).unwrap();
+
+        // Pin the entire snippet (including the `
` wrapper) so any
+        // change in the styled spans, the merge boundary between the kw
+        // span and the trailing space, or the line-break handling will
+        // surface immediately.
+        //
+        // Layout: bold "ping " (the trailing space merges into the kw
+        // span via the whitespace-folding `should_merge` predicate),
+        // then italic "thing", then a literal `\n` outside any styled
+        // span (because `ThemedRenderer::write_text` always splits on
+        // newline).
+        assert_eq!(
+            html,
+            "
\n\
+             ping \
+             thing\n\
+             
\n"
+        );
+    }
+
+    #[test]
+    fn html_styled_output_should_merge_whitespace_into_previous_span() {
+        // Verify that the `should_merge` whitespace-fold path is actually
+        // exercised: the trailing space after "ping" must end up *inside*
+        // the kw span, NOT in its own span. This is the observable
+        // difference between `should_merge` returning true (correct,
+        // whitespace folds) and the mutant `should_merge -> false`
+        // (whitespace would get its own span with the default fg colour).
+        let (ss, theme, _bg) = make_pinned_ss_and_theme();
+        let syntax = &ss.syntaxes()[0];
+
+        let html = highlighted_html_for_string("ping thing\n", &ss, syntax, &theme).unwrap();
+
+        // The kw span (bold) must contain "ping " (with trailing space).
+        // If the merge predicate is broken, the space would land in a
+        // separate `color:#c8c8c8;` span instead.
+        assert!(
+            html.contains("ping "),
+            "expected the trailing space to merge into the kw span, got: {html}"
+        );
+        assert!(
+            !html.contains(""),
+            "the default-foreground span for whitespace must not appear (the space should fold into the kw span), got: {html}"
+        );
+    }
+
+    #[test]
+    #[allow(deprecated)]
+    fn deprecated_parse_html_for_line_emits_nontrivial_html() {
+        // The deprecated line-parsing entry point must still drive the
+        // underlying parser and emit highlighted markup. A bug that turns
+        // it into a silent no-op would only surface as missing token
+        // markup in the finalized output.
+        let syntax_set = SyntaxSet::load_defaults_newlines();
+        let syntax = syntax_set.find_syntax_by_name("R").unwrap();
+        let mut g =
+            ClassedHTMLGenerator::new_with_class_style(syntax, &syntax_set, ClassStyle::Spaced);
+        // Note: this deprecated function takes a line WITHOUT a trailing
+        // newline (the function appends one internally — see its doc).
+        g.parse_html_for_line("x + y");
+        let html = g.finalize();
+        // The output must contain the literal token text and a classed span;
+        // a `with ()` mutant would skip the parse and the output would only
+        // hold the trailing newline appended by the (broken) function.
+        assert!(
+            html.contains("x + y\n"
+        );
+        // delta is +1 because exactly one scope (source.r) is left open at
+        // end of line.
+        assert_eq!(delta, 1);
+    }
+
+    #[test]
+    #[allow(deprecated)]
+    fn deprecated_tokens_to_classed_html_returns_html_string_only() {
+        // The HTML-only convenience wrapper must produce the exact same
+        // markup as the spans variant for the same input. Catches a
+        // degenerate body that returns an empty or sentinel string.
+        let syntax_set = SyntaxSet::load_defaults_newlines();
+        let syntax = syntax_set.find_syntax_by_name("R").unwrap();
+        let mut state = ParseState::new(syntax);
+        let line = "x + y\n";
+        let ops = state.parse_line(line, &syntax_set).unwrap().ops;
+
+        let html = tokens_to_classed_html(line, &ops, ClassStyle::Spaced);
+        assert_eq!(
+            html,
+            "x + y\n"
+        );
+    }
+
+    #[test]
+    fn html_styled_output_emits_background_when_distinct_from_default() {
+        // When a token style has a background different from the containing
+        // element's default background, `HtmlStyledOutput::begin_style` must
+        // emit a `background-color:` declaration before the `color:` one.
+        // Catches mutants in the `style.background != self.default_bg`
+        // branch (replacing != with == would invert the emission).
+        use crate::highlighting::{
+            Color, FontStyle, ScopeSelectors, StyleModifier, Theme, ThemeItem, ThemeSettings,
+        };
+        use std::str::FromStr;
+
+        let syntax_str = r#"
+name: PingThing
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+
+        let bg = Color {
+            r: 0,
+            g: 0,
+            b: 0,
+            a: 0xff,
+        };
+        let theme = Theme {
+            name: Some("html-pin-bg".into()),
+            author: None,
+            settings: ThemeSettings {
+                foreground: Some(Color {
+                    r: 200,
+                    g: 200,
+                    b: 200,
+                    a: 0xff,
+                }),
+                background: Some(bg),
+                ..Default::default()
+            },
+            scopes: vec![ThemeItem {
+                scope: ScopeSelectors::from_str("kw").unwrap(),
+                style: StyleModifier {
+                    foreground: Some(Color {
+                        r: 11,
+                        g: 22,
+                        b: 33,
+                        a: 0xff,
+                    }),
+                    background: Some(Color {
+                        r: 99,
+                        g: 88,
+                        b: 77,
+                        a: 0xff,
+                    }),
+                    font_style: Some(FontStyle::empty()),
+                },
+            }],
+        };
+
+        let syntax_ref = &ss.syntaxes()[0];
+        let html = highlighted_html_for_string("ping\n", &ss, syntax_ref, &theme).unwrap();
+        // Search the inner part — outside the `
` wrapper.
+        assert!(
+            html.contains("background-color:#63584d;"),
+            "expected per-token background-color declaration, got: {html}"
+        );
+        // The order must be: background-color first, then color.
+        assert!(
+            html.contains("background-color:#63584d;color:#0b1621;"),
+            "expected `background-color:` before `color:` in begin_style emission, got: {html}"
+        );
+    }
 }
diff --git a/src/io.rs b/src/io.rs
new file mode 100644
index 00000000..9221b2b2
--- /dev/null
+++ b/src/io.rs
@@ -0,0 +1,1638 @@
+//! Streaming syntax highlighting built on top of [`std::io::Write`].
+//!
+//! The central type is [`HighlightedWriter`], an implementation of
+//! [`std::io::Write`] that drives syntax parsing and delegates rendering to a
+//! pluggable [`ScopeRenderer`]. Bytes written into the writer are buffered
+//! into complete lines and forwarded to the highlighter; rendered output is
+//! streamed to the inner sink.
+//!
+//! Renderer plumbing — the [`ScopeRenderer`] / [`ScopeMarkup`] /
+//! [`StyledOutput`] traits, the [`ThemedRenderer`] adapter, and the built-in
+//! [`AnsiStyledOutput`] — lives in [`crate::rendering`]. Construct a
+//! `HighlightedWriter` by calling one of [`HighlightedWriter::from_themed`],
+//! [`HighlightedWriter::from_markup`], or [`HighlightedWriter::from_renderer`] to
+//! get a [`HighlightedWriterBuilder`], then chain `.with_output(...)` /
+//! `.with_state(...)` as needed and finish with `.build()`.
+//!
+//! For HTML output, see [`crate::html::ClassedHTMLGenerator`],
+//! [`crate::html::ClassedHTMLScopeRenderer`], and
+//! [`crate::html::HtmlStyledOutput`].
+
+use crate::highlighting::Theme;
+use crate::parsing::{
+    lock_global_scope_repo, ParseState, ScopeStack, ScopeStackOp, SyntaxReference, SyntaxSet,
+};
+use crate::rendering::{
+    render_line, resolve_atom_strs, AnsiStyledOutput, MarkupAdapter, ScopeMarkup, ScopeRenderer,
+    StyledOutput, ThemedRenderer,
+};
+use crate::Error;
+use std::io::{self, Write};
+
+// ---------------------------------------------------------------------------
+// HighlightedWriter — io::Write-based highlighting driver
+// ---------------------------------------------------------------------------
+
+/// A streaming syntax highlighter that implements [`std::io::Write`].
+///
+/// Bytes written into the writer are accumulated until a newline is seen,
+/// at which point each complete line is parsed, rendered through the
+/// configured renderer, and forwarded to the inner [`Write`] sink.
+///
+/// Construct one via the [`HighlightedWriterBuilder`]. There are three
+/// entry points, one per renderer category:
+///
+/// - [`from_themed`] — pair a [`Theme`] with any [`StyledOutput`] (the
+///   standard choice for terminal colours via [`AnsiStyledOutput`],
+///   inline-styled HTML, LaTeX `\textcolor`, etc.).
+/// - [`from_markup`] — pass any [`ScopeMarkup`] (stateless renderer like
+///   CSS-classed HTML).
+/// - [`from_renderer`] — low-level escape hatch that takes a raw
+///   [`ScopeRenderer`].
+///
+/// Each entry point returns a [`HighlightedWriterBuilder`] pre-populated
+/// with a `Vec` sink and no resume state. Configure further with
+/// [`Builder::with_output`] / [`Builder::with_state`] and finish with
+/// [`Builder::build`].
+///
+/// When the parser is in speculative mode (inside a branch point),
+/// `HighlightedWriter` buffers rendered output internally and flushes it
+/// only once the speculation resolves, replaying corrected operations if a
+/// cross-line `fail` occurred.
+///
+/// A trailing partial line (one that did not end with `\n`) is held in the
+/// internal buffer until either another `\n` arrives or end-of-input cleanup
+/// runs. End-of-input cleanup runs in two ways:
+///
+/// - **Implicitly**, in [`Drop`], on a best-effort basis when the writer
+///   falls out of scope. Errors are silently swallowed. This is the natural
+///   choice when you don't need the inner sink back and don't care about
+///   error reporting.
+/// - **Explicitly**, via [`into_inner`], which consumes the writer and
+///   returns the inner sink (`io::Result`) so you can both inspect any
+///   errors and recover the bytes (the default `Vec` sink is the common
+///   case).
+///
+/// [`from_themed`]: HighlightedWriter::from_themed
+/// [`from_markup`]: HighlightedWriter::from_markup
+/// [`from_renderer`]: HighlightedWriter::from_renderer
+/// [`Builder::with_output`]: HighlightedWriterBuilder::with_output
+/// [`Builder::with_state`]: HighlightedWriterBuilder::with_state
+/// [`Builder::build`]: HighlightedWriterBuilder::build
+/// [`into_inner`]: HighlightedWriter::into_inner
+///
+/// # Examples
+///
+/// Theme-based ANSI highlighting streamed straight to a sink, composing with
+/// [`std::io::copy`]:
+///
+/// ```no_run
+/// use std::io::{self, Write};
+/// use syntect::io::HighlightedWriter;
+/// use syntect::highlighting::ThemeSet;
+/// use syntect::parsing::SyntaxSet;
+/// use syntect::rendering::AnsiStyledOutput;
+///
+/// let ss = SyntaxSet::load_defaults_newlines();
+/// let ts = ThemeSet::load_defaults();
+/// let syntax = ss.find_syntax_by_extension("rs").unwrap();
+///
+/// let mut f = std::fs::File::open("examples/parsyncat.rs").unwrap();
+/// let mut w = HighlightedWriter::from_themed(
+///     syntax,
+///     &ss,
+///     &ts.themes["base16-ocean.dark"],
+///     AnsiStyledOutput::Transparent,
+/// )
+/// .with_output(io::stdout().lock())
+/// .build();
+/// io::copy(&mut f, &mut w).unwrap();
+/// w.into_inner().unwrap();
+/// ```
+///
+/// Highlighting an in-memory string with the default `Vec` sink:
+///
+/// ```
+/// use std::io::Write;
+/// use syntect::io::HighlightedWriter;
+/// use syntect::highlighting::ThemeSet;
+/// use syntect::parsing::SyntaxSet;
+/// use syntect::rendering::AnsiStyledOutput;
+///
+/// let ss = SyntaxSet::load_defaults_newlines();
+/// let ts = ThemeSet::load_defaults();
+/// let syntax = ss.find_syntax_by_extension("rs").unwrap();
+///
+/// let mut w = HighlightedWriter::from_themed(
+///     syntax,
+///     &ss,
+///     &ts.themes["base16-ocean.dark"],
+///     AnsiStyledOutput::Transparent,
+/// )
+/// .build();
+/// w.write_all(b"fn main() {}\n").unwrap();
+/// let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+/// assert!(output.contains("\x1b[38;2;"));
+/// ```
+pub struct HighlightedWriter<
+    'a,
+    R: ScopeRenderer = ThemedRenderer<'a, AnsiStyledOutput>,
+    W: io::Write = Vec,
+> {
+    syntax_set: &'a SyntaxSet,
+    open_scopes: isize,
+    parse_state: ParseState,
+    scope_stack: ScopeStack,
+    // Wrapped in `Option` so that `into_inner` can take ownership of the
+    // sink via `&mut self`, and so that `Drop` can detect whether `into_inner`
+    // has already been called and skip the cleanup if so.
+    output: Option,
+    renderer: R,
+    line_index: usize,
+    line_buf: Vec,
+    // Branch-point buffering
+    pending_lines: Vec,
+    pending_ops: Vec>,
+    scope_stack_snapshot: Option,
+    open_scopes_snapshot: Option,
+}
+
+impl<'a> HighlightedWriter<'a> {
+    /// Start a builder for a theme-aware renderer.
+    ///
+    /// Pairs the given [`Theme`] with any [`StyledOutput`] (`AnsiStyledOutput`,
+    /// `HtmlStyledOutput`, or your own format) and returns a
+    /// [`HighlightedWriterBuilder`] that you can further configure with
+    /// [`with_output`] / [`with_state`] before calling [`build`].
+    ///
+    /// For ANSI 24-bit terminal colours, pass
+    /// [`AnsiStyledOutput::Transparent`] or [`AnsiStyledOutput::Opaque`].
+    ///
+    /// [`with_output`]: HighlightedWriterBuilder::with_output
+    /// [`with_state`]: HighlightedWriterBuilder::with_state
+    /// [`build`]: HighlightedWriterBuilder::build
+    pub fn from_themed(
+        syntax_reference: &'a SyntaxReference,
+        syntax_set: &'a SyntaxSet,
+        theme: &'a Theme,
+        output: O,
+    ) -> HighlightedWriterBuilder<'a, ThemedRenderer<'a, O>> {
+        HighlightedWriterBuilder::new(
+            syntax_reference,
+            syntax_set,
+            ThemedRenderer::new(theme, output),
+        )
+    }
+
+    /// Start a builder for a stateless markup renderer.
+    ///
+    /// The given [`ScopeMarkup`] implementation is wrapped in an internal
+    /// adapter that bridges it to the engine's low-level renderer trait.
+    /// Use this for renderers that map scope structure 1:1 to output
+    /// structure (e.g. [`crate::html::ClassedHTMLScopeRenderer`]).
+    pub fn from_markup(
+        syntax_reference: &'a SyntaxReference,
+        syntax_set: &'a SyntaxSet,
+        markup: M,
+    ) -> HighlightedWriterBuilder<'a, MarkupAdapter> {
+        HighlightedWriterBuilder::new(syntax_reference, syntax_set, MarkupAdapter::new(markup))
+    }
+
+    /// Start a builder with a low-level [`ScopeRenderer`].
+    ///
+    /// This is the escape hatch for advanced cases that need raw [`Scope`]
+    /// / `&[Scope]` access or selective `bool` returns from `begin_scope`.
+    /// Most users should reach for [`from_themed`] or [`from_markup`] instead.
+    ///
+    /// [`Scope`]: crate::parsing::Scope
+    /// [`from_themed`]: HighlightedWriter::from_themed
+    /// [`from_markup`]: HighlightedWriter::from_markup
+    pub fn from_renderer(
+        syntax_reference: &'a SyntaxReference,
+        syntax_set: &'a SyntaxSet,
+        renderer: R,
+    ) -> HighlightedWriterBuilder<'a, R> {
+        HighlightedWriterBuilder::new(syntax_reference, syntax_set, renderer)
+    }
+}
+
+/// Fluent builder for a [`HighlightedWriter`].
+///
+/// Construct one via [`HighlightedWriter::from_themed`],
+/// [`HighlightedWriter::from_markup`], or
+/// [`HighlightedWriter::from_renderer`] — each picks a renderer category and
+/// returns a builder pre-populated with a default `Vec` output sink and
+/// no resume state. Configure further with [`with_output`] (replace the sink)
+/// and [`with_state`] (resume from a saved checkpoint), then call [`build`]
+/// to materialise the writer.
+///
+/// [`with_output`]: HighlightedWriterBuilder::with_output
+/// [`with_state`]: HighlightedWriterBuilder::with_state
+/// [`build`]: HighlightedWriterBuilder::build
+#[must_use = "HighlightedWriterBuilder produces nothing until `.build()` is called"]
+pub struct HighlightedWriterBuilder<'a, R: ScopeRenderer, W: io::Write = Vec> {
+    syntax_reference: &'a SyntaxReference,
+    syntax_set: &'a SyntaxSet,
+    renderer: R,
+    output: W,
+    state: Option<(ParseState, ScopeStack)>,
+}
+
+impl<'a, R: ScopeRenderer> HighlightedWriterBuilder<'a, R, Vec> {
+    fn new(syntax_reference: &'a SyntaxReference, syntax_set: &'a SyntaxSet, renderer: R) -> Self {
+        Self {
+            syntax_reference,
+            syntax_set,
+            renderer,
+            output: Vec::new(),
+            state: None,
+        }
+    }
+}
+
+impl<'a, R: ScopeRenderer, W: io::Write> HighlightedWriterBuilder<'a, R, W> {
+    /// Replace the default `Vec` output sink with an arbitrary
+    /// [`io::Write`].
+    pub fn with_output(self, output: W2) -> HighlightedWriterBuilder<'a, R, W2> {
+        HighlightedWriterBuilder {
+            syntax_reference: self.syntax_reference,
+            syntax_set: self.syntax_set,
+            renderer: self.renderer,
+            output,
+            state: self.state,
+        }
+    }
+
+    /// Resume highlighting from a previously saved parse + scope state.
+    ///
+    /// Useful for incremental highlighting where you cache the state at
+    /// checkpoints and want to pick up from where a previous run left off.
+    /// The renderer's internal state (e.g. a `ThemedRenderer`'s style stack)
+    /// is replayed to match the restored scope stack.
+    pub fn with_state(mut self, parse_state: ParseState, scope_stack: ScopeStack) -> Self {
+        self.state = Some((parse_state, scope_stack));
+        self
+    }
+
+    /// Materialise the configured [`HighlightedWriter`].
+    pub fn build(self) -> HighlightedWriter<'a, R, W> {
+        let HighlightedWriterBuilder {
+            syntax_reference,
+            syntax_set,
+            mut renderer,
+            output,
+            state,
+        } = self;
+
+        match state {
+            None => HighlightedWriter {
+                syntax_set,
+                open_scopes: 0,
+                parse_state: ParseState::new(syntax_reference),
+                scope_stack: ScopeStack::new(),
+                output: Some(output),
+                renderer,
+                line_index: 0,
+                line_buf: Vec::new(),
+                pending_lines: Vec::new(),
+                pending_ops: Vec::new(),
+                scope_stack_snapshot: None,
+                open_scopes_snapshot: None,
+            },
+            Some((parse_state, scope_stack)) => {
+                // Replay the existing scope stack to the renderer so that
+                // its internal state (e.g. a ThemedRenderer's style stack)
+                // matches the restored scopes.
+                {
+                    let repo = lock_global_scope_repo();
+                    let scopes = &scope_stack.scopes;
+                    for (i, &scope) in scopes.iter().enumerate() {
+                        let atom_strs = resolve_atom_strs(scope, &repo);
+                        let stack_slice = &scopes[..=i];
+                        let mut dummy = String::new();
+                        renderer.begin_scope(&atom_strs, scope, stack_slice, &mut dummy);
+                    }
+                }
+                let open_scopes = scope_stack.scopes.len() as isize;
+                HighlightedWriter {
+                    syntax_set,
+                    open_scopes,
+                    parse_state,
+                    scope_stack,
+                    output: Some(output),
+                    renderer,
+                    line_index: 0,
+                    line_buf: Vec::new(),
+                    pending_lines: Vec::new(),
+                    pending_ops: Vec::new(),
+                    scope_stack_snapshot: None,
+                    open_scopes_snapshot: None,
+                }
+            }
+        }
+    }
+}
+
+impl<'a, R: ScopeRenderer, W: io::Write> HighlightedWriter<'a, R, W> {
+    /// Returns a reference to the renderer.
+    pub fn renderer(&self) -> &R {
+        &self.renderer
+    }
+
+    /// Returns a mutable reference to the renderer.
+    pub fn renderer_mut(&mut self) -> &mut R {
+        &mut self.renderer
+    }
+
+    /// Returns references to the current parse and scope state.
+    pub fn state(&self) -> (&ParseState, &ScopeStack) {
+        (&self.parse_state, &self.scope_stack)
+    }
+
+    /// Parse one complete line and forward the rendered output to the sink.
+    ///
+    /// This is the inner per-line entry point used by the [`Write`]
+    /// implementation once a complete `\n`-terminated line has been
+    /// assembled in the internal buffer.
+    fn highlight_line(&mut self, line: &str) -> Result<(), Error> {
+        let parse_output = self.parse_state.parse_line(line, self.syntax_set)?;
+
+        // If replayed ops arrived, patch the pending buffer in place.
+        // Per `ParseLineOutput::replayed`'s contract, the corrected ops
+        // align with the *last* `replayed.len()` entries of the pending
+        // buffer (`replayed[i] ↔ pending_lines[buf_len - replayed.len() + i]`).
+        // Zipping from index 0 instead would slide ops onto unrelated
+        // line text, panicking later in `render_line` as "byte index N
+        // out of bounds" when an op offset overshoots the misaligned
+        // line.
+        if !parse_output.replayed.is_empty() {
+            let start = self.pending_ops.len() - parse_output.replayed.len();
+            for (i, ops) in parse_output.replayed.into_iter().enumerate() {
+                self.pending_ops[start + i] = ops;
+            }
+        }
+
+        // Fast path: not speculative, nothing buffered — render + write directly.
+        if !self.parse_state.is_speculative() && self.pending_lines.is_empty() {
+            let (formatted, delta) = render_line(
+                line,
+                &parse_output.ops,
+                &mut self.scope_stack,
+                &mut self.renderer,
+                self.line_index,
+            )?;
+            self.open_scopes += delta;
+            self.output
+                .as_mut()
+                .expect("output already taken")
+                .write_all(formatted.as_bytes())?;
+            self.line_index += 1;
+            return Ok(());
+        }
+
+        // Buffer this line.
+        if self.scope_stack_snapshot.is_none() {
+            self.scope_stack_snapshot = Some(self.scope_stack.clone());
+            self.open_scopes_snapshot = Some(self.open_scopes);
+        }
+        self.pending_lines.push(line.to_owned());
+        self.pending_ops.push(parse_output.ops);
+        self.line_index += 1;
+
+        // If speculation ended, flush.
+        if !self.parse_state.is_speculative() {
+            self.flush_pending()?;
+        }
+
+        Ok(())
+    }
+
+    fn flush_pending(&mut self) -> Result<(), Error> {
+        if self.pending_lines.is_empty() {
+            return Ok(());
+        }
+        let mut scope_stack = self.scope_stack_snapshot.take().unwrap();
+        let mut open_scopes = self.open_scopes_snapshot.take().unwrap();
+        let line_index_offset = self.line_index - self.pending_lines.len();
+
+        for (i, (line, ops)) in self
+            .pending_lines
+            .iter()
+            .zip(self.pending_ops.iter())
+            .enumerate()
+        {
+            let (formatted, delta) = render_line(
+                line,
+                ops,
+                &mut scope_stack,
+                &mut self.renderer,
+                line_index_offset + i,
+            )?;
+            open_scopes += delta;
+            self.output
+                .as_mut()
+                .expect("output already taken")
+                .write_all(formatted.as_bytes())?;
+        }
+
+        self.scope_stack = scope_stack;
+        self.open_scopes = open_scopes;
+        self.pending_lines.clear();
+        self.pending_ops.clear();
+        Ok(())
+    }
+
+    /// Run end-of-input cleanup and return the inner sink.
+    ///
+    /// This is the explicit, error-propagating counterpart to the implicit
+    /// best-effort cleanup that runs in [`Drop`]. Call this when you need:
+    ///
+    /// - the inner sink back (the default `Vec` sink is the common case),
+    /// - error handling for any failures during the trailing-line drain,
+    ///   the speculative-line flush, or the close-scopes emission.
+    ///
+    /// Otherwise, you can let the writer fall out of scope and `Drop` will
+    /// run the same cleanup on a best-effort basis (errors are silently
+    /// swallowed).
+    pub fn into_inner(mut self) -> io::Result {
+        self.finalize_inner()
+    }
+
+    /// Internal cleanup pathway shared by [`into_inner`] and `Drop::drop`.
+    ///
+    /// Drains the trailing partial line, flushes any pending speculatively
+    /// buffered lines, emits close markers for scopes still open at EOF,
+    /// then `take`s the inner sink and returns it. After this returns,
+    /// `self.output` is `None`.
+    ///
+    /// [`into_inner`]: HighlightedWriter::into_inner
+    fn finalize_inner(&mut self) -> io::Result {
+        // 1. Drain a trailing partial line, if any.
+        if !self.line_buf.is_empty() {
+            let line_bytes = std::mem::take(&mut self.line_buf);
+            let line = std::str::from_utf8(&line_bytes)
+                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+            self.highlight_line(line).map_err(io::Error::other)?;
+        }
+
+        // 2. Flush any pending speculatively-buffered lines.
+        self.flush_pending().map_err(io::Error::other)?;
+
+        // 3. Emit close markers for any scopes still open at EOF.
+        let mut buf = String::new();
+        for _ in 0..self.open_scopes {
+            self.renderer.end_scope(&mut buf);
+        }
+        self.open_scopes = 0;
+        self.output
+            .as_mut()
+            .expect("output already taken")
+            .write_all(buf.as_bytes())?;
+
+        // 4. Take the sink out, leaving the field as `None`.
+        Ok(self.output.take().expect("output already taken"))
+    }
+}
+
+impl<'a, R: ScopeRenderer, W: io::Write> Drop for HighlightedWriter<'a, R, W> {
+    fn drop(&mut self) {
+        // Best-effort cleanup. If `into_inner` has already been called,
+        // `self.output` is `None` and there's nothing to do.
+        if self.output.is_some() {
+            let _ = self.finalize_inner();
+        }
+    }
+}
+
+impl<'a, R: ScopeRenderer, W: io::Write> Write for HighlightedWriter<'a, R, W> {
+    fn write(&mut self, buf: &[u8]) -> io::Result {
+        if buf.is_empty() {
+            return Ok(0);
+        }
+        self.line_buf.extend_from_slice(buf);
+
+        // Process every complete line in a single batch. Newlines (0x0A) cannot
+        // appear inside a multi-byte UTF-8 sequence, so locating the last
+        // newline is safe on the raw byte buffer.
+        if let Some(last_nl) = self.line_buf.iter().rposition(|&b| b == b'\n') {
+            let completed: Vec = self.line_buf.drain(..=last_nl).collect();
+            let s = std::str::from_utf8(&completed)
+                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+            for line in s.split_inclusive('\n') {
+                self.highlight_line(line).map_err(io::Error::other)?;
+            }
+        }
+        Ok(buf.len())
+    }
+
+    /// Forwards to the inner sink. Partial trailing lines remain buffered;
+    /// they are not highlightable until terminated by `\n` or by
+    /// [`into_inner`](HighlightedWriter::into_inner) (or implicitly by `Drop`).
+    fn flush(&mut self) -> io::Result<()> {
+        self.output.as_mut().expect("output already taken").flush()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::highlighting::ThemeSet;
+    use crate::parsing::SyntaxSet;
+    use crate::rendering::AnsiStyledOutput;
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn theme_scope_renderer_produces_output() {
+        let ss = SyntaxSet::load_defaults_newlines();
+        let ts = ThemeSet::load_defaults();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let mut w = HighlightedWriter::from_themed(
+            syntax,
+            &ss,
+            &ts.themes["base16-ocean.dark"],
+            AnsiStyledOutput::Transparent,
+        )
+        .build();
+        w.write_all(b"pub struct Wow { hi: u64 }\n").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+        assert!(!output.is_empty());
+        assert!(output.contains("\x1b[38;2;"));
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn style_merging_coalesces_same_style_tokens() {
+        let ss = SyntaxSet::load_defaults_newlines();
+        let ts = ThemeSet::load_defaults();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let mut w = HighlightedWriter::from_themed(
+            syntax,
+            &ss,
+            &ts.themes["base16-ocean.dark"],
+            AnsiStyledOutput::Transparent,
+        )
+        .build();
+        w.write_all(b"fn main() {}\n").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+
+        // Style merging means we should NOT see consecutive identical ANSI
+        // escape codes with no text between them.
+        assert!(!output.contains("m\x1b[38;2;"));
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn buffers_partial_line_until_newline_or_finalize() {
+        let ss = SyntaxSet::load_defaults_newlines();
+        let ts = ThemeSet::load_defaults();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let mut w = HighlightedWriter::from_themed(
+            syntax,
+            &ss,
+            &ts.themes["base16-ocean.dark"],
+            AnsiStyledOutput::Transparent,
+        )
+        .build();
+        // Write a line in three chunks, with the newline arriving last.
+        w.write_all(b"fn main").unwrap();
+        w.write_all(b"() {").unwrap();
+        w.write_all(b"}\n").unwrap();
+        // And a trailing line that finalize must flush.
+        w.write_all(b"struct S;").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+        assert!(output.contains("fn"));
+        assert!(output.contains("main"));
+        assert!(output.contains("struct"));
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn handles_multibyte_chars_split_across_writes() {
+        let ss = SyntaxSet::load_defaults_newlines();
+        let ts = ThemeSet::load_defaults();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let mut w = HighlightedWriter::from_themed(
+            syntax,
+            &ss,
+            &ts.themes["base16-ocean.dark"],
+            AnsiStyledOutput::Transparent,
+        )
+        .build();
+        // 'é' is two bytes (0xC3 0xA9). Split between writes.
+        w.write_all(b"// caf\xC3").unwrap();
+        w.write_all(b"\xA9\n").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+        assert!(output.contains("café"));
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn can_start_again_from_previous_state() {
+        let ss = SyntaxSet::load_defaults_newlines();
+        let ts = ThemeSet::load_defaults();
+        let theme = &ts.themes["base16-ocean.dark"];
+        let mut w = HighlightedWriter::from_themed(
+            ss.find_syntax_by_extension("py").unwrap(),
+            &ss,
+            theme,
+            AnsiStyledOutput::Transparent,
+        )
+        .build();
+
+        let lines = ["\"\"\"\n", "def foo():\n", "\"\"\"\n"];
+        w.write_all(lines[0].as_bytes()).unwrap();
+
+        let (parse_state, scope_stack) = w.state();
+        let (parse_state, scope_stack) = (parse_state.clone(), scope_stack.clone());
+        let first_output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+
+        let mut other = HighlightedWriter::from_themed(
+            ss.find_syntax_by_extension("py").unwrap(),
+            &ss,
+            theme,
+            AnsiStyledOutput::Transparent,
+        )
+        .with_state(parse_state, scope_stack)
+        .build();
+        other.write_all(lines[1].as_bytes()).unwrap();
+        let second_output = String::from_utf8(other.into_inner().unwrap()).unwrap();
+
+        // The second line should be highlighted as a docstring (same style as
+        // the first line's triple-quote) because the parse state carries the
+        // string context forward.
+        assert!(!second_output.is_empty());
+        let extract_fg =
+            |s: &str| -> Option { s.find("\x1b[38;2;").map(|i| s[i..i + 16].to_string()) };
+        assert_eq!(extract_fg(&first_output), extract_fg(&second_output));
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "default-themes"))]
+    #[test]
+    fn drop_runs_cleanup_when_into_inner_is_skipped() {
+        // Verify that letting a `HighlightedWriter` fall out of scope without
+        // calling `into_inner` still drains a trailing partial line and emits
+        // close markers for any open scopes — i.e. that `Drop` runs the same
+        // cleanup as `into_inner`. We compare the bytes a borrowed sink ends
+        // up with against the bytes the same input produces under explicit
+        // `into_inner`.
+        let ss = SyntaxSet::load_defaults_newlines();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        // The input deliberately ends WITHOUT a trailing newline, so the
+        // last token sits in `line_buf` until cleanup. The classed-HTML
+        // markup renderer also leaves the outer ``
+        // open at end-of-input, exercising the close-scopes path.
+        let input = b"fn main()";
+
+        // 1. Reference: explicit `into_inner`.
+        let mut explicit = HighlightedWriter::from_markup(
+            syntax,
+            &ss,
+            crate::html::ClassedHTMLScopeRenderer::new(crate::html::ClassStyle::Spaced),
+        )
+        .build();
+        explicit.write_all(input).unwrap();
+        let explicit_bytes = explicit.into_inner().unwrap();
+
+        // 2. Drop-only: borrow a sink, write input, let the writer fall out
+        // of scope without calling `into_inner`.
+        let mut implicit_buf: Vec = Vec::new();
+        {
+            let mut implicit = HighlightedWriter::from_markup(
+                syntax,
+                &ss,
+                crate::html::ClassedHTMLScopeRenderer::new(crate::html::ClassStyle::Spaced),
+            )
+            .with_output(&mut implicit_buf)
+            .build();
+            implicit.write_all(input).unwrap();
+            // No `into_inner` here. `implicit` drops at the end of this
+            // scope and `Drop` runs cleanup against the borrowed sink.
+        }
+
+        assert_eq!(
+            explicit_bytes, implicit_buf,
+            "Drop should produce identical bytes to an explicit into_inner"
+        );
+        // Sanity-check that cleanup actually happened: the markup output
+        // should contain the closing tag for the outer source scope.
+        let s = String::from_utf8(implicit_buf).unwrap();
+        assert!(
+            s.ends_with(""),
+            "expected outer  to be emitted by cleanup, got: {s}"
+        );
+    }
+
+    // ── Phase B mutation-killing tests ────────────────────────────────────
+
+    /// Capturing `ScopeMarkup` that records the exact sequence of method
+    /// invocations. Used by tests below to assert on `begin_line`/`end_line`
+    /// pairing, monotonic line indices, and atom-string forwarding.
+    ///
+    /// Events are stored behind an `Rc>` so the test can hold its
+    /// own handle to them, drop the writer (triggering the implicit Drop
+    /// cleanup which itself emits more events), and then read the final
+    /// event log without depending on the writer being alive.
+    #[derive(Debug, Clone, PartialEq, Eq)]
+    enum MarkupEvent {
+        BeginLine(usize),
+        EndLine(usize),
+        BeginScope(Vec),
+        EndScope,
+        Text(String),
+    }
+
+    type SharedEvents = std::rc::Rc>>;
+
+    struct CapturingMarkup {
+        events: SharedEvents,
+    }
+
+    impl CapturingMarkup {
+        fn new() -> (Self, SharedEvents) {
+            let events: SharedEvents = std::rc::Rc::new(std::cell::RefCell::new(Vec::new()));
+            (
+                Self {
+                    events: events.clone(),
+                },
+                events,
+            )
+        }
+    }
+
+    impl ScopeMarkup for CapturingMarkup {
+        fn begin_line(&mut self, line_index: usize, _output: &mut String) {
+            self.events
+                .borrow_mut()
+                .push(MarkupEvent::BeginLine(line_index));
+        }
+        fn end_line(&mut self, line_index: usize, _output: &mut String) {
+            self.events
+                .borrow_mut()
+                .push(MarkupEvent::EndLine(line_index));
+        }
+        fn begin_scope(&mut self, atom_strs: &[&str], _output: &mut String) {
+            self.events.borrow_mut().push(MarkupEvent::BeginScope(
+                atom_strs.iter().map(|s| (*s).to_string()).collect(),
+            ));
+        }
+        fn end_scope(&mut self, _output: &mut String) {
+            self.events.borrow_mut().push(MarkupEvent::EndScope);
+        }
+        fn write_text(&mut self, text: &str, _output: &mut String) {
+            self.events
+                .borrow_mut()
+                .push(MarkupEvent::Text(text.to_string()));
+        }
+    }
+
+    #[cfg(feature = "default-syntaxes")]
+    #[test]
+    fn markup_receives_paired_line_hooks_with_monotonic_line_index() {
+        // Three lines of Rust → exactly three begin_line/end_line pairs with
+        // line indices 0, 1, 2 in order. Catches mutants that drop the calls,
+        // emit a wrong line index, mutate `+= 1` on `line_index`, or replace
+        // `line_index_offset + i` arithmetic in `flush_pending`.
+        let ss = SyntaxSet::load_defaults_newlines();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let (capture, events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax, &ss, capture).build();
+        w.write_all(b"fn a() {}\nfn b() {}\nfn c() {}\n").unwrap();
+        // Drop the writer first so any cleanup-time events are recorded
+        // BEFORE we read them.
+        drop(w);
+        let events = events.borrow().clone();
+
+        let line_starts: Vec = events
+            .iter()
+            .filter_map(|e| match e {
+                MarkupEvent::BeginLine(i) => Some(*i),
+                _ => None,
+            })
+            .collect();
+        let line_ends: Vec = events
+            .iter()
+            .filter_map(|e| match e {
+                MarkupEvent::EndLine(i) => Some(*i),
+                _ => None,
+            })
+            .collect();
+        assert_eq!(
+            line_starts,
+            vec![0, 1, 2],
+            "begin_line must fire once per line with monotonically increasing 0-based indices"
+        );
+        assert_eq!(
+            line_ends,
+            vec![0, 1, 2],
+            "end_line must fire once per line with the same indices as begin_line"
+        );
+
+        // begin_line(i) must precede the matching end_line(i) and the
+        // pairing must alternate (no two starts in a row, no two ends in a
+        // row, no end before its start).
+        let mut open: Option = None;
+        for ev in &events {
+            match ev {
+                MarkupEvent::BeginLine(i) => {
+                    assert!(open.is_none(), "nested begin_line for {i}: prev still open");
+                    open = Some(*i);
+                }
+                MarkupEvent::EndLine(i) => {
+                    assert_eq!(open, Some(*i), "end_line({i}) without matching begin_line");
+                    open = None;
+                }
+                _ => {}
+            }
+        }
+        assert!(open.is_none(), "trailing begin_line with no end_line");
+    }
+
+    #[cfg(feature = "default-syntaxes")]
+    #[test]
+    fn markup_forwards_atom_strs_and_pairs_scopes() {
+        // Catches: MarkupAdapter::begin_scope dropping atom_strs derivation,
+        // MarkupAdapter::write_text being replaced with (), and
+        // resolve_atom_strs returning vec![] / vec![""] / vec!["xyzzy"].
+        let ss = SyntaxSet::load_defaults_newlines();
+        let syntax = ss.find_syntax_by_name("R").unwrap();
+
+        let (capture, events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax, &ss, capture).build();
+        w.write_all(b"x + y\n").unwrap();
+        drop(w);
+        let events = events.borrow().clone();
+
+        // The first scope must be ["source", "r"].
+        let first_scope = events
+            .iter()
+            .find_map(|e| match e {
+                MarkupEvent::BeginScope(atoms) => Some(atoms.clone()),
+                _ => None,
+            })
+            .expect("expected at least one begin_scope event");
+        assert_eq!(first_scope, vec!["source".to_string(), "r".to_string()]);
+
+        // begin_scope and end_scope must be paired (count equal).
+        let begins = events
+            .iter()
+            .filter(|e| matches!(e, MarkupEvent::BeginScope(_)))
+            .count();
+        let ends = events
+            .iter()
+            .filter(|e| matches!(e, MarkupEvent::EndScope))
+            .count();
+        assert_eq!(begins, ends, "begin_scope/end_scope must be paired");
+        assert!(begins > 0, "expected at least one scope to be opened");
+
+        // The literal text "x" must be passed verbatim to write_text.
+        // (Catches MarkupAdapter::write_text being replaced with ().)
+        let text_concat: String = events
+            .iter()
+            .filter_map(|e| match e {
+                MarkupEvent::Text(t) => Some(t.as_str()),
+                _ => None,
+            })
+            .collect();
+        assert!(
+            text_concat.contains('x') && text_concat.contains('y'),
+            "expected literal x and y to flow through write_text, got: {text_concat:?}"
+        );
+    }
+
+    #[test]
+    fn ansi_output_is_byte_exact_for_known_input() {
+        // Use a hand-built one-syntax SyntaxSet and a tiny synthetic theme
+        // so the test isn't sensitive to bundled-syntax/theme drift. Pin
+        // the exact ANSI byte sequence emitted for a known token. This
+        // locks down: that the foreground escape is emitted at all (a
+        // silently-empty `begin_style` would produce just the literal
+        // text), that the RGB channels appear in the right order with
+        // the right separators, and that the post-token newline
+        // splitting in the streaming layer advances past the newline
+        // rather than getting stuck on it.
+        use crate::highlighting::{
+            Color, FontStyle, ScopeSelectors, StyleModifier, Theme, ThemeItem, ThemeSettings,
+        };
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+        use std::str::FromStr;
+
+        let syntax_str = r#"
+name: PlainPing
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let theme = Theme {
+            name: Some("ansi-test".into()),
+            author: None,
+            settings: ThemeSettings {
+                foreground: Some(Color {
+                    r: 200,
+                    g: 100,
+                    b: 50,
+                    a: 0xff,
+                }),
+                background: Some(Color {
+                    r: 0,
+                    g: 0,
+                    b: 0,
+                    a: 0xff,
+                }),
+                ..Default::default()
+            },
+            scopes: vec![ThemeItem {
+                scope: ScopeSelectors::from_str("kw").unwrap(),
+                style: StyleModifier {
+                    foreground: Some(Color {
+                        r: 10,
+                        g: 20,
+                        b: 30,
+                        a: 0xff,
+                    }),
+                    background: None,
+                    font_style: Some(FontStyle::empty()),
+                },
+            }],
+        };
+
+        let mut w =
+            HighlightedWriter::from_themed(syntax_ref, &ss, &theme, AnsiStyledOutput::Transparent)
+                .build();
+        w.write_all(b"ping\n").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+
+        // Exactly: open span with kw style, "ping", close span (no-op for
+        // ANSI), then "\n".
+        assert_eq!(output, "\x1b[38;2;10;20;30mping\n");
+    }
+
+    #[test]
+    fn ansi_output_with_background_emits_both_escapes() {
+        // Catches AnsiStyledOutput::begin_style with `AnsiStyledOutput::Opaque` being
+        // mutated (e.g. swapping the order, dropping one escape, swapping
+        // 38/48 codes, channel swaps).
+        use crate::highlighting::{
+            Color, FontStyle, ScopeSelectors, StyleModifier, Theme, ThemeItem, ThemeSettings,
+        };
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+        use std::str::FromStr;
+
+        let syntax_str = r#"
+name: PlainPing
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let theme = Theme {
+            name: Some("ansi-bg-test".into()),
+            author: None,
+            settings: ThemeSettings::default(),
+            scopes: vec![ThemeItem {
+                scope: ScopeSelectors::from_str("kw").unwrap(),
+                style: StyleModifier {
+                    foreground: Some(Color {
+                        r: 11,
+                        g: 22,
+                        b: 33,
+                        a: 0xff,
+                    }),
+                    background: Some(Color {
+                        r: 44,
+                        g: 55,
+                        b: 66,
+                        a: 0xff,
+                    }),
+                    font_style: Some(FontStyle::empty()),
+                },
+            }],
+        };
+
+        let mut w =
+            HighlightedWriter::from_themed(syntax_ref, &ss, &theme, AnsiStyledOutput::Opaque)
+                .build();
+        w.write_all(b"ping\n").unwrap();
+        let output = String::from_utf8(w.into_inner().unwrap()).unwrap();
+
+        // The background escape (48) must come first, followed by foreground
+        // (38), then the literal text, then the trailing newline. The kw
+        // style ought to apply to "ping" precisely.
+        assert!(
+            output.contains("\x1b[48;2;44;55;66m\x1b[38;2;11;22;33mping"),
+            "expected bg+fg pair around 'ping', got {output:?}"
+        );
+        assert!(output.ends_with('\n'));
+    }
+
+    #[cfg(feature = "default-syntaxes")]
+    #[test]
+    fn line_index_is_threaded_correctly_across_branch_point_replay() {
+        // The hardest-to-reach gap: cross-line branch-point speculation.
+        // Build a custom syntax that triggers `fail: bp` on line 2, forcing
+        // the parser to retroactively replay line 1 under the fallback
+        // alternative. The streaming writer must:
+        //   1. Buffer line 1's output until speculation resolves.
+        //   2. Render line 1 once, with the corrected ops, with line_index=0.
+        //   3. Render line 2 with line_index=1.
+        //   4. Emit no `try.word` scope (the wrong-alternative result).
+        //   5. Pass `begin_line(0)` BEFORE `begin_line(1)` to the renderer.
+        //
+        // Catches mutants in highlight_line, flush_pending, line_index_offset
+        // arithmetic, the speculative buffering branch, and the replay patch.
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+
+        let syntax_str = r#"
+name: CrossLineBranchTest
+scope: source.clbt
+contexts:
+  main:
+    - match: 'TRY'
+      branch_point: bp
+      branch: [try-ctx, fallback-ctx]
+    - match: '.*'
+      scope: main.other
+  try-ctx:
+    - match: '\n'
+    - match: 'FAIL'
+      fail: bp
+    - match: '\w+'
+      scope: try.word
+      pop: true
+  fallback-ctx:
+    - match: '.*'
+      scope: fallback.content
+      pop: true
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let (capture, events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax_ref, &ss, capture).build();
+        // Line 1 triggers branch_point. Line 2 triggers fail: bp, forcing
+        // retroactive replay of line 1 under fallback-ctx.
+        w.write_all(b"TRY\nFAIL\n").unwrap();
+        drop(w);
+        let events = events.borrow().clone();
+
+        // 1. begin_line indices must be 0 then 1, in order.
+        let line_starts: Vec = events
+            .iter()
+            .filter_map(|e| match e {
+                MarkupEvent::BeginLine(i) => Some(*i),
+                _ => None,
+            })
+            .collect();
+        assert_eq!(
+            line_starts,
+            vec![0, 1],
+            "begin_line should fire for both lines after speculation flush, in order"
+        );
+
+        // 2. The wrong-alternative scope `try.word` must NOT appear.
+        let saw_try_word = events.iter().any(|e| match e {
+            MarkupEvent::BeginScope(atoms) => atoms.iter().any(|a| a == "try"),
+            _ => false,
+        });
+        assert!(
+            !saw_try_word,
+            "the speculation-failed `try.*` scope must not surface in rendered output"
+        );
+
+        // 3. The corrected `fallback.content` scope MUST appear.
+        let saw_fallback = events.iter().any(|e| match e {
+            MarkupEvent::BeginScope(atoms) => atoms.iter().any(|a| a == "fallback"),
+            _ => false,
+        });
+        assert!(
+            saw_fallback,
+            "expected the post-replay `fallback.content` scope in rendered output"
+        );
+
+        // 4. begin_line(0) must precede begin_line(1).
+        let pos0 = events
+            .iter()
+            .position(|e| matches!(e, MarkupEvent::BeginLine(0)))
+            .expect("begin_line(0) missing");
+        let pos1 = events
+            .iter()
+            .position(|e| matches!(e, MarkupEvent::BeginLine(1)))
+            .expect("begin_line(1) missing");
+        assert!(pos0 < pos1, "line 0 events must precede line 1 events");
+    }
+
+    #[test]
+    fn replay_window_narrower_than_pending_buffer_aligns_to_last_slots() {
+        // The parser's `ParseLineOutput::replayed` aligns to the *last*
+        // `replayed.len()` entries of the consumer's pending buffer
+        // (`replayed[i] ↔ pending_ops[buf_len - replayed.len() + i]`).
+        // Sliding ops onto the wrong slot panicked TypeScript's
+        // `checker.ts` in `render_line` as "byte index N out of bounds".
+        //
+        // Force `pending_ops.len() > replayed.len()` by opening an outer
+        // BP on line 1 and an inner BP on line 3, then failing the inner
+        // on line 4. Replay covers line 3 only; pending = [L1, L2, L3].
+        // L1 is short and L3 is long, so a slot misalignment lands ops
+        // with offsets past L1's end and panics the byte-index slice.
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+
+        let syntax_str = r#"
+name: NestedBpReplay
+scope: source.nbr
+contexts:
+  main:
+    - match: 'OUTER'
+      branch_point: bp1
+      branch: [outer-a, outer-b]
+  outer-a:
+    - match: '\n'
+    - match: 'INNER'
+      branch_point: bp2
+      branch: [inner-a, inner-b]
+    - match: '(?=NEVERMATCH)'
+      fail: bp1
+    - match: '.'
+      scope: outer.a.char
+  outer-b:
+    - match: '.*'
+      scope: outer.b.fallback
+      pop: true
+  inner-a:
+    - match: '\n'
+    - match: 'KILLBP2'
+      fail: bp2
+    - match: '.'
+      scope: inner.a.text
+  inner-b:
+    - match: '\n'
+    - match: 'KILLBP2'
+      fail: bp2
+    - match: '.'
+      scope: inner.b.text
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let (capture, _events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax_ref, &ss, capture).build();
+        // L1: "OUTER\n" (6 bytes) opens bp1.
+        // L2: "AB\n"    (3 bytes) ordinary content under outer-a.
+        // L3: long INNER... (>>L1 len) opens bp2 inside outer-a.
+        // L4: "KILLBP2\n" fails bp2 → replayed.len() == 1 covering L3.
+        // Pre-fix the L3 corrected ops slid onto L1's text and tripped
+        // `&line[cur_index..i]` once `i` exceeded L1's 6 bytes.
+        w.write_all(b"OUTER\nAB\nINNERZZZZZZZZZZZZZZZZZZZZZZZZZ\nKILLBP2\n")
+            .unwrap();
+        drop(w);
+        // Reaching here means `flush_pending` rendered every (line, ops)
+        // pair without overflowing the line slice.
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "html"))]
+    #[test]
+    fn drop_flushes_pending_speculation_buffer() {
+        // Variant of `drop_runs_cleanup_when_into_inner_is_skipped` that also
+        // exercises the speculative-line flush path inside `finalize_inner`.
+        // Construct a writer over a syntax with cross-line branch-point
+        // failures, feed it input that leaves `pending_lines` non-empty (the
+        // branch is still active), drop the writer without `into_inner`, and
+        // assert that the borrowed sink contains both rendered lines.
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+
+        let syntax_str = r#"
+name: PendingDropTest
+scope: source.pdt
+contexts:
+  main:
+    - match: 'OPEN'
+      branch_point: bp
+      branch: [try-ctx, fallback-ctx]
+    - match: '.*'
+      scope: main.other
+  try-ctx:
+    - match: '\n'
+    - match: '\w+'
+      scope: try.word
+      pop: true
+  fallback-ctx:
+    - match: '.*'
+      scope: fallback.content
+      pop: true
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        // Reference output: explicit `into_inner` after writing input that
+        // *does* trigger speculation but never resolves it before EOF.
+        let mut explicit = HighlightedWriter::from_markup(
+            syntax_ref,
+            &ss,
+            crate::html::ClassedHTMLScopeRenderer::new(crate::html::ClassStyle::Spaced),
+        )
+        .build();
+        explicit.write_all(b"OPEN\n").unwrap();
+        let explicit_bytes = explicit.into_inner().unwrap();
+
+        // Drop-only path: borrowed sink, identical input, no into_inner call.
+        let mut implicit_buf: Vec = Vec::new();
+        {
+            let mut implicit = HighlightedWriter::from_markup(
+                syntax_ref,
+                &ss,
+                crate::html::ClassedHTMLScopeRenderer::new(crate::html::ClassStyle::Spaced),
+            )
+            .with_output(&mut implicit_buf)
+            .build();
+            implicit.write_all(b"OPEN\n").unwrap();
+        }
+
+        assert_eq!(
+            explicit_bytes, implicit_buf,
+            "Drop must flush pending speculative lines just like into_inner"
+        );
+        // Sanity: the buffer must not be empty (catches Drop replaced with ()
+        // and finalize_inner replaced with `Ok(Default::default())`).
+        assert!(
+            !implicit_buf.is_empty(),
+            "Drop cleanup produced no output; pending-line flush must have been skipped"
+        );
+        let s = String::from_utf8(implicit_buf).unwrap();
+        assert!(
+            s.contains("OPEN"),
+            "expected the speculative line content to surface after Drop, got: {s}"
+        );
+        // Absolute span balance: every opened classed span must be balanced
+        // by a closing tag in the FLUSHED output. This is the assertion that
+        // catches `open_scopes += delta → -=/*=` in flush_pending — both the
+        // explicit and implicit paths route through `flush_pending`, so a
+        // byte-equality check would treat the mutated count as a no-op (both
+        // sides produce the same wrong output). Tag balance is an absolute
+        // invariant, mutation-blind.
+        let opens = s.matches("").count();
+        assert_eq!(
+            opens, closes,
+            "post-flush span balance must hold (opens={opens}, closes={closes}); the cleanup-tag accumulator inside the speculative-flush path is likely wrong: {s}"
+        );
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "html"))]
+    #[test]
+    fn empty_scope_optimisation_truncates_push_pop_with_no_text() {
+        // The empty-scope optimisation: when the parser pushes a scope and
+        // immediately pops it at the same byte position with no intervening
+        // text, the renderer must NOT emit a corresponding markup pair.
+        // The optimisation hinges on tracking when a scope is "still
+        // empty" between its push and its matching pop, and on driving
+        // the open-scope count consistently in both directions; any of
+        // those bookkeeping invariants going wrong would surface here as
+        // empty span pairs leaking into the output.
+        //
+        // We use the high-level HTML generator rather than a capturing
+        // markup so that we observe the *truncated* output buffer
+        // directly: the optimisation happens at the rendered-string
+        // level, and only the bytes that survive truncation reach the
+        // inner sink.
+        //
+        // Empty-scope cases are triggered when a Push and a matching Pop
+        // occur at the same byte position with no intervening text. The
+        // bundled Rust syntax exercises this naturally for a function
+        // with an empty body — there are several adjacent push/pop pairs
+        // around the brace and parenthesis tokens.
+        let ss = SyntaxSet::load_defaults_newlines();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+        let mut g = crate::html::ClassedHTMLGenerator::new_with_class_style(
+            syntax,
+            &ss,
+            crate::html::ClassStyle::Spaced,
+        );
+        g.parse_html_for_line_which_includes_newline("fn f() {}\n")
+            .unwrap();
+        let html = g.finalize();
+
+        // No empty `` pair should appear in the
+        // output (where … contains no whitespace, indicating an
+        // immediately-closed pushed scope). Catches mutants that flip the
+        // truncation logic in `render_line`'s `scope_empty` tracking and
+        // would cause empty span pairs to leak into the output.
+        //
+        // The regex check is intentionally conservative: any pair of the
+        // form `.` would be a leak, where
+        // `.` matches any single non-newline character. We use a simple
+        // string contains check on the simplest possible empty pattern
+        // because Rust's syntax highlighting doesn't produce empty class
+        // names; if the optimisation is broken, the parser's empty scope
+        // pushes/pops would surface as `` for some
+        // class `X`.
+        let mut found_empty = false;
+        let mut idx = 0;
+        while let Some(open_start) = html[idx..].find("` of the open tag.
+            if let Some(rel_close) = html[after_open_tag..].find("\">") {
+                let content_start = after_open_tag + rel_close + 2;
+                // Empty if the very next characters are "".
+                if html[content_start..].starts_with("") {
+                    found_empty = true;
+                    break;
+                }
+                idx = content_start;
+            } else {
+                break;
+            }
+        }
+        assert!(
+            !found_empty,
+            "empty-scope optimisation must prevent  pairs from leaking, got HTML:\n{html}"
+        );
+
+        // Sanity: the output must still contain *some* nontrivial spans
+        // (otherwise we'd be passing trivially because nothing was
+        // emitted at all).
+        assert!(
+            html.contains("").count(),
+            "every  must be balanced by a ; missing close means end_line was skipped: {html}"
+        );
+        assert!(
+            html.ends_with(""),
+            "expected output to end with the trailing , got: {html:?}"
+        );
+    }
+
+    /// `io::Write` sink that records every call. Used to assert that
+    /// `HighlightedWriter::flush` actually delegates to the inner sink and
+    /// that bytes propagate.
+    struct CountingSink {
+        inner: Vec,
+        flush_count: std::rc::Rc>,
+    }
+
+    impl io::Write for CountingSink {
+        fn write(&mut self, buf: &[u8]) -> io::Result {
+            self.inner.extend_from_slice(buf);
+            Ok(buf.len())
+        }
+        fn flush(&mut self) -> io::Result<()> {
+            self.flush_count.set(self.flush_count.get() + 1);
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn flush_delegates_to_inner_sink() {
+        // The streaming writer's `flush` must call through to the inner
+        // sink's `flush`, not silently succeed. A no-op `flush` would
+        // leave buffered downstream sinks (e.g. a `BufWriter` wrapped
+        // around a file) holding bytes after the caller had asked for
+        // them to be made durable.
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+
+        let syntax_str = r#"
+name: PingNoop
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let flush_count = std::rc::Rc::new(std::cell::Cell::new(0usize));
+        let sink = CountingSink {
+            inner: Vec::new(),
+            flush_count: flush_count.clone(),
+        };
+        let (capture, _events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax_ref, &ss, capture)
+            .with_output(sink)
+            .build();
+        w.write_all(b"ping\n").unwrap();
+        assert_eq!(
+            flush_count.get(),
+            0,
+            "flush should not be called by write_all"
+        );
+
+        w.flush().unwrap();
+        assert_eq!(
+            flush_count.get(),
+            1,
+            "HighlightedWriter::flush must delegate to the inner sink exactly once"
+        );
+
+        w.flush().unwrap();
+        assert_eq!(flush_count.get(), 2);
+    }
+
+    #[cfg(all(feature = "default-syntaxes", feature = "html"))]
+    #[test]
+    fn drop_emits_exact_count_of_close_tags_for_open_scopes() {
+        // End-of-input cleanup must emit exactly as many close-scope
+        // markers as there are scopes still open at EOF — no more, no
+        // fewer. The accumulator that tracks "scopes still open" runs
+        // through both the fast path and the speculative-flush path; a
+        // sign or factor mutation on the accumulator would either leak
+        // unclosed tags or emit phantom closes. The expected closing-tag
+        // count is pinned absolutely (against the markup tag balance
+        // invariant), not relative to another path that shares the same
+        // bug.
+        let ss = SyntaxSet::load_defaults_newlines();
+        let syntax = ss.find_syntax_by_extension("rs").unwrap();
+
+        let mut w = HighlightedWriter::from_markup(
+            syntax,
+            &ss,
+            crate::html::ClassedHTMLScopeRenderer::new(crate::html::ClassStyle::Spaced),
+        )
+        .build();
+        // Trailing newline so the line is fully highlighted; the only scope
+        // still open at EOF is the outer source.rust meta-scope.
+        w.write_all(b"fn main() {}\n").unwrap();
+        let bytes = w.into_inner().unwrap();
+        let html = String::from_utf8(bytes).unwrap();
+
+        let opens = html.matches("").count();
+        assert!(opens > 0, "expected at least one opening span, got: {html}");
+        assert_eq!(
+            opens, closes,
+            "every opened classed span must be balanced by a closing tag (opens={opens}, closes={closes}); cleanup tag count likely wrong"
+        );
+
+        // The output must end with \n (the outer source.rust meta-scope
+        // closing tag plus the trailing newline that ended the input line).
+        // This is the most direct test of the cleanup-emit-end_scope path.
+        assert!(
+            html.ends_with("\n") || html.ends_with(""),
+            "expected output to end with the outermost , got: {html:?}"
+        );
+    }
+
+    #[test]
+    fn write_returns_full_buffer_length_for_nonempty_input() {
+        // Pin the `io::Write` contract: `write` must return the number
+        // of bytes consumed from the input buffer, which always equals
+        // the input length for the streaming highlighter (it never
+        // partially-accepts a write). An empty input must short-circuit
+        // and return 0 without engaging the line-buffer machinery — a
+        // bug there would either return a constant length, or
+        // accidentally re-process the empty buffer as if a complete
+        // line had arrived.
+        use crate::parsing::{SyntaxDefinition, SyntaxSetBuilder};
+
+        let syntax_str = r#"
+name: PingNoop
+scope: source.ping
+contexts:
+  main:
+    - match: 'ping'
+      scope: kw.ping
+"#;
+        let syntax = SyntaxDefinition::load_from_str(syntax_str, true, None).unwrap();
+        let mut builder = SyntaxSetBuilder::new();
+        builder.add(syntax);
+        let ss = builder.build();
+        let syntax_ref = &ss.syntaxes()[0];
+
+        let (capture, _events) = CapturingMarkup::new();
+        let mut w = HighlightedWriter::from_markup(syntax_ref, &ss, capture).build();
+        let buf = b"ping\n";
+        let n = w.write(buf).unwrap();
+        assert_eq!(n, buf.len());
+
+        // Empty buffer must return 0 (and NOT accidentally process anything).
+        let n_empty = w.write(b"").unwrap();
+        assert_eq!(n_empty, 0);
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 182fb426..572b70fe 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -34,7 +34,11 @@ mod escape;
 pub mod highlighting;
 #[cfg(feature = "html")]
 pub mod html;
+#[cfg(feature = "parsing")]
+pub mod io;
 pub mod parsing;
+#[cfg(feature = "parsing")]
+pub mod rendering;
 pub mod util;
 mod utils;
 
diff --git a/src/parsing/metadata.rs b/src/parsing/metadata.rs
index 0a7a22fe..c24c06ac 100644
--- a/src/parsing/metadata.rs
+++ b/src/parsing/metadata.rs
@@ -500,7 +500,9 @@ mod tests {
         let r = RawMetadataEntry::load(comments_file);
         assert!(r.is_ok());
 
-        let indent_file: &str = "testdata/Packages/Go/Indentation Rules.tmPreferences";
+        // v4202 still has the upstream "Indention" typo; renamed to
+        // "Indentation" by `1c86d7f9` post-v4202 (#4519).
+        let indent_file: &str = "testdata/Packages/Go/Indention Rules.tmPreferences";
         assert!(Path::new(indent_file).exists());
 
         let r = RawMetadataEntry::load(indent_file).unwrap();
diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs
deleted file mode 100644
index f8102e32..00000000
--- a/src/parsing/parser.rs
+++ /dev/null
@@ -1,6045 +0,0 @@
-// Suppression of a false positive clippy lint. Upstream issue:
-//
-//   mutable_key_type false positive for raw pointers
-//   https://github.com/rust-lang/rust-clippy/issues/6745
-//
-// We use `*const MatchPattern` as key in our `SearchCache` hash map.
-// Clippy thinks this is a problem since `MatchPattern` has interior mutability
-// via `MatchPattern::regex::regex` which is an `AtomicLazyCell`.
-// But raw pointers are hashed via the pointer itself, not what is pointed to.
-// See https://github.com/rust-lang/rust/blob/1.54.0/library/core/src/hash/mod.rs#L717-L725
-#![allow(clippy::mutable_key_type)]
-
-use super::regex::{Regex, Region};
-use super::scope::*;
-use super::syntax_definition::*;
-use crate::parsing::syntax_definition::ContextId;
-use crate::parsing::syntax_set::{SyntaxReference, SyntaxSet};
-use fnv::FnvHasher;
-use regex_syntax::escape;
-use std::collections::HashMap;
-use std::hash::BuildHasherDefault;
-
-/// Errors that can occur while parsing.
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
-pub enum ParsingError {
-    #[error("Somehow main context was popped from the stack")]
-    MissingMainContext,
-    /// A context is missing. Usually caused by a syntax referencing a another
-    /// syntax that is not known to syntect. See e.g. 
-    #[error("Missing context with ID '{0:?}'")]
-    MissingContext(ContextId),
-    #[error("Bad index to match_at: {0}")]
-    BadMatchIndex(usize),
-    #[error("Tried to use a ContextReference that has not bee resolved yet: {0:?}")]
-    UnresolvedContextReference(ContextReference),
-}
-
-/// Keeps the current parser state (the internal syntax interpreter stack) between lines of parsing.
-///
-/// If you are parsing an entire file you create one of these at the start and use it
-/// all the way to the end.
-///
-/// # Caching
-///
-/// One reason this is exposed is that since it implements `Clone` you can actually cache
-/// these (probably along with a [`HighlightState`]) and only re-start parsing from the point of a change.
-/// See the docs for [`HighlightState`] for more in-depth discussion of caching.
-///
-/// This state doesn't keep track of the current scope stack and parsing only returns changes to this stack
-/// so if you want to construct scope stacks you'll need to keep track of that as well.
-/// Note that [`HighlightState`] contains exactly this as a public field that you can use.
-///
-/// **Note:** Caching is for advanced users who have tons of time to maximize performance or want to do so eventually.
-/// It is not recommended that you try caching the first time you implement highlighting.
-///
-/// [`HighlightState`]: ../highlighting/struct.HighlightState.html
-/// Output of [`ParseState::parse_line`].
-///
-/// `ops` contains the scope-stack operations for the current line, as before.
-/// `replayed` is non-empty only after a cross-line `fail` fires: it contains
-/// the corrected ops for each buffered line (in chronological order) so that
-/// callers who want full cross-line accuracy can re-apply them.
-///
-/// Callers that do not need cross-line accuracy can use `.ops` directly,
-/// which behaves identically to the old `Vec<(usize, ScopeStackOp)>` return.
-#[derive(Debug, Clone, Default)]
-pub struct ParseLineOutput {
-    /// Ops for the current line.
-    pub ops: Vec<(usize, ScopeStackOp)>,
-    /// Ops for previously buffered lines that have now been corrected, in order.
-    /// Non-empty only when a cross-line `fail` just resolved.
-    pub replayed: Vec>,
-    /// Warnings collected during parsing (e.g. branch point expiry).
-    pub warnings: Vec,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct ParseState {
-    stack: Vec,
-    first_line: bool,
-    // See issue #101. Contains indices of frames pushed by `with_prototype`s.
-    // Doesn't look at `with_prototype`s below top of stack.
-    proto_starts: Vec,
-    /// Active branch points for backtracking support.
-    branch_points: Vec,
-    /// Line counter for 128-line branch point expiry.
-    line_number: usize,
-    /// Line strings buffered while branch points are active, for potential
-    /// cross-line `fail` replay. Only the strings are stored; the ops are
-    /// returned to callers immediately (same as before).
-    pending_lines: Vec,
-    /// Corrected ops produced by a cross-line `fail` replay, to be returned
-    /// as `ParseLineOutput::replayed` at the end of `parse_line`.
-    flushed_ops: Vec>,
-    /// Warnings accumulated during parsing, drained into `ParseLineOutput`.
-    warnings: Vec,
-    /// Active escape patterns from embed operations. The escape regex takes
-    /// strict precedence over normal patterns — it is checked first and can
-    /// truncate the search region.
-    escape_stack: Vec,
-}
-
-/// A resolved escape pattern from an `embed` operation, stored on the escape stack.
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct EscapeEntry {
-    /// The resolved escape regex (backrefs substituted at push time).
-    regex: Regex,
-    /// Capture mapping for escape_captures scopes.
-    captures: Option,
-    /// Stack depth at the time of the embed push — when escape fires,
-    /// pop down to this depth.
-    stack_depth: usize,
-}
-
-/// Snapshot of parser state at a branch point, used for backtracking.
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct BranchPoint {
-    name: String,
-    /// Index of the next alternative to try (0 = first alt already tried).
-    next_alternative: usize,
-    alternatives: Vec,
-    stack_snapshot: Vec,
-    proto_starts_snapshot: Vec,
-    /// Character position to rewind to. Despite the name, this is the
-    /// branch match's *end* position — where the parser resumes from.
-    match_start: usize,
-    /// Real start of the branch_point match text. Together with
-    /// `match_start` (above, which is the match end) this bounds the
-    /// span on which `pat_scope` applies. Used to re-emit the keyword
-    /// scope — e.g. `keyword.operator.comparison.sql` on `LIKE` —
-    /// after a `fail` rewind, since the original Push/Pop pair was
-    /// truncated off `ops` along with `alt[0]`'s subsequent work.
-    trigger_match_start: usize,
-    /// Scopes declared on the branch_point match itself (re-emitted on
-    /// fail-retry over the [`trigger_match_start`, `match_start`) span).
-    pat_scope: Vec,
-    /// Line number when the branch was created (for 128-line limit).
-    line_number: usize,
-    /// Length of ops vec at snapshot time — truncation point on fail.
-    ops_snapshot_len: usize,
-    /// Stack depth at creation — if stack shrinks below this, branch is invalid.
-    stack_depth: usize,
-    non_consuming_push_at_snapshot: (usize, usize),
-    first_line_snapshot: bool,
-    with_prototype: Option,
-    /// `pending_lines.len()` at snapshot time, for cross-line replay truncation.
-    pending_lines_snapshot_len: usize,
-    escape_stack_snapshot: Vec,
-    /// Number of contexts to pop before pushing the alternative (for pop + branch).
-    pop_count: usize,
-    /// Ops emitted on the branch-creation line before the branch match.
-    /// Used by cross-line fail replay to reconstruct the first buffered
-    /// line without re-parsing its pre-branch prefix under the new
-    /// alternative (which would misattribute pre-branch content to
-    /// rules of the new alternative — e.g. in multi-line SQL `LIKE …
-    /// ESCAPE …`, every non-whitespace before the `LIKE` fires
-    /// `else-pop` in the escape-alternative, derailing the stack).
-    prefix_ops: Vec<(usize, ScopeStackOp)>,
-    /// Capture Push/Pop ops emitted alongside the branch_point match's
-    /// `pat_scope`. Re-emitted on fail-retry between the pat_scope
-    /// Push and Pop so captures like `keyword.declaration.data.haskell`
-    /// on the first capture group of `(data)(?:\s+(family|instance))?`
-    /// survive a branch swap — without this, a `data CtxCls ctx => …`
-    /// (where `alt[0]` `data-signature` fails into `alt[1]` `data-context`)
-    /// drops the keyword scope from the `data` token.
-    capture_ops: Vec<(usize, ScopeStackOp)>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct StateLevel {
-    context: ContextId,
-    prototypes: Vec,
-    captures: Option<(Region, String)>,
-}
-
-#[derive(Debug)]
-struct RegexMatch<'a> {
-    regions: Region,
-    context: &'a Context,
-    pat_index: usize,
-    from_with_prototype: bool,
-    would_loop: bool,
-    /// For escape matches (pat_index == usize::MAX): index into escape_stack.
-    escape_index: usize,
-}
-
-/// Maps the pattern to the start index, which is -1 if not found.
-type SearchCache = HashMap<*const MatchPattern, Option, BuildHasherDefault>;
-
-/// Build the ordered Push/Pop ops for a match's `captures:` mapping over
-/// its regex `regions`. Captures can appear in arbitrary source order
-/// (e.g. `((bob)|(hi))*` matching `hibob` — the outer group must Push
-/// before any inner group). Empty captures are skipped because they'd
-/// otherwise sort a Pop before its Push. The returned ops are already
-/// position-ordered and safe to append to a parser ops vec.
-fn build_capture_ops(capture_map: &CaptureMapping, regions: &Region) -> Vec<(usize, ScopeStackOp)> {
-    let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new();
-    for &(cap_index, ref scopes) in capture_map.iter() {
-        if let Some((cap_start, cap_end)) = regions.pos(cap_index) {
-            if cap_start == cap_end {
-                continue;
-            }
-            for scope in scopes.iter() {
-                map.push((
-                    (cap_start, -((cap_end - cap_start) as i32)),
-                    ScopeStackOp::Push(*scope),
-                ));
-            }
-            map.push(((cap_end, i32::MIN), ScopeStackOp::Pop(scopes.len())));
-        }
-    }
-    map.sort_by(|a, b| a.0.cmp(&b.0));
-    map.into_iter().map(|((i, _), op)| (i, op)).collect()
-}
-
-// To understand the implementation of this, here's an introduction to how
-// Sublime Text syntax definitions work.
-//
-// Let's say we have the following made-up syntax definition:
-//
-//     contexts:
-//       main:
-//         - match: A
-//           scope: scope.a.first
-//           push: context-a
-//         - match: b
-//           scope: scope.b
-//         - match: \w+
-//           scope: scope.other
-//       context-a:
-//         - match: a+
-//           scope: scope.a.rest
-//         - match: (?=.)
-//           pop: true
-//
-// There are two contexts, `main` and `context-a`. Each context contains a list
-// of match rules with instructions for how to proceed.
-//
-// Let's say we have the input string " Aaaabxxx". We start at position 0 in
-// the string. We keep a stack of contexts, which at the beginning is just main.
-//
-// So we start by looking at the top of the context stack (main), and look at
-// the rules in order. The rule that wins is the first one that matches
-// "earliest" in the input string. In our example:
-//
-// 1. The first one matches "A". Note that matches are not anchored, so this
-//    matches at position 1.
-// 2. The second one matches "b", so position 5. The first rule is winning.
-// 3. The third one matches "\w+", so also position 1. But because the first
-//    rule comes first, it wins.
-//
-// So now we execute the winning rule. Whenever we matched some text, we assign
-// the scope (if there is one) to the matched text and advance our position to
-// after the matched text. The scope is "scope.a.first" and our new position is
-// after the "A", so 2. The "push" means that we should change our stack by
-// pushing `context-a` on top of it.
-//
-// In the next step, we repeat the above, but now with the rules in `context-a`.
-// The result is that we match "a+" and assign "scope.a.rest" to "aaa", and our
-// new position is now after the "aaa". Note that there was no instruction for
-// changing the stack, so we stay in that context.
-//
-// In the next step, the first rule doesn't match anymore, so we go to the next
-// rule where "(?=.)" matches. The instruction is to "pop", which means we
-// pop the top of our context stack, which means we're now back in main.
-//
-// This time in main, we match "b", and in the next step we match the rest with
-// "\w+", and we're done.
-//
-//
-// ## Preventing loops
-//
-// These are the basics of how matching works. Now, you saw that you can write
-// patterns that result in an empty match and don't change the position. These
-// are called non-consuming matches. The problem with them is that they could
-// result in infinite loops. Let's look at a syntax where that is the case:
-//
-//     contexts:
-//       main:
-//         - match: (?=.)
-//           push: test
-//       test:
-//         - match: \w+
-//           scope: word
-//         - match: (?=.)
-//           pop: true
-//
-// This is a bit silly, but it's a minimal example for explaining how matching
-// works in that case.
-//
-// Let's say we have the input string " hello". In `main`, our rule matches and
-// we go into `test` and stay at position 0. Now, the best match is the rule
-// with "pop". But if we used that rule, we'd pop back to `main` and would still
-// be at the same position we started at! So this would be an infinite loop,
-// which we don't want.
-//
-// So what Sublime Text does in case a looping rule "won":
-//
-// * If there's another rule that matches at the same position and does not
-//   result in a loop, use that instead.
-// * Otherwise, go to the next position and go through all the rules in the
-//   current context again. Note that it means that the "pop" could again be the
-//   winning rule, but that's ok as it wouldn't result in a loop anymore.
-//
-// So in our input string, we'd skip one character and try to match the rules
-// again. This time, the "\w+" wins because it comes first.
-
-impl ParseState {
-    /// Creates a state from a syntax definition, keeping its own reference-counted point to the
-    /// main context of the syntax
-    pub fn new(syntax: &SyntaxReference) -> ParseState {
-        let start_state = StateLevel {
-            context: syntax.context_ids()["__start"],
-            prototypes: Vec::new(),
-            captures: None,
-        };
-        ParseState {
-            stack: vec![start_state],
-            first_line: true,
-            proto_starts: Vec::new(),
-            branch_points: Vec::new(),
-            line_number: 0,
-            pending_lines: Vec::new(),
-            flushed_ops: Vec::new(),
-            warnings: Vec::new(),
-            escape_stack: Vec::new(),
-        }
-    }
-
-    /// Parses a single line of the file. Because of the way regex engines work you unfortunately
-    /// have to pass in a single line contiguous in memory. This can be bad for really long lines.
-    /// Sublime Text avoids this by just not highlighting lines that are too long (thousands of characters).
-    ///
-    /// For efficiency reasons this returns only the changes to the current scope at each point in the line.
-    /// You can use [`ScopeStack::apply`] on each operation in succession to get the stack for a given point.
-    /// Look at the code in `highlighter.rs` for an example of doing this for highlighting purposes.
-    ///
-    /// The returned vector is in order both by index to apply at (the `usize`) and also by order to apply them at a
-    /// given index (e.g popping old scopes before pushing new scopes).
-    ///
-    /// The [`SyntaxSet`] has to be the one that contained the syntax that was used to construct
-    /// this [`ParseState`], or an extended version of it. Otherwise the parsing would return the
-    /// wrong result or even panic. The reason for this is that contexts within the [`SyntaxSet`]
-    /// are referenced via indexes.
-    ///
-    /// [`ScopeStack::apply`]: struct.ScopeStack.html#method.apply
-    /// [`SyntaxSet`]: struct.SyntaxSet.html
-    /// [`ParseState`]: struct.ParseState.html
-    pub fn parse_line(
-        &mut self,
-        line: &str,
-        syntax_set: &SyntaxSet,
-    ) -> Result {
-        if self.stack.is_empty() {
-            return Err(ParsingError::MissingMainContext);
-        }
-
-        // Prune branch points older than 128 lines
-        let cur_line = self.line_number;
-        let warnings = &mut self.warnings;
-        self.branch_points.retain(|bp| {
-            let alive = cur_line.saturating_sub(bp.line_number) <= 128;
-            if !alive {
-                warnings.push(format!(
-                    "branch point '{}' expired (exceeded 128-line rewind limit)",
-                    bp.name
-                ));
-            }
-            alive
-        });
-        self.line_number += 1;
-
-        let ops = self.parse_line_inner(line, syntax_set)?;
-
-        // Collect any corrected ops produced by a cross-line `fail` during the
-        // parse above.  These are stored by `handle_fail` in `self.flushed_ops`.
-        let replayed = std::mem::take(&mut self.flushed_ops);
-
-        // Keep the line string for potential future cross-line replay.
-        if !self.branch_points.is_empty() {
-            self.pending_lines.push(line.to_string());
-        } else {
-            // No active branch points: any buffered strings are stale.
-            self.pending_lines.clear();
-        }
-
-        let warnings = std::mem::take(&mut self.warnings);
-
-        Ok(ParseLineOutput {
-            ops,
-            replayed,
-            warnings,
-        })
-    }
-
-    /// Returns `true` when the parser is inside a `branch_point` and the
-    /// result of `parse_line` may be revised by a future `fail` action.
-    /// Once the branch resolves (or if no branch was entered), this returns
-    /// `false` and all ops emitted so far are final.
-    pub fn is_speculative(&self) -> bool {
-        !self.branch_points.is_empty()
-    }
-
-    /// Inner parsing loop: processes `line` with the current parser state and
-    /// returns the scope-stack operations.  Does **not** touch `pending_lines`
-    /// or `flushed_ops`, so it is safe to call recursively from `handle_fail`
-    /// for cross-line replay without re-entrancy issues.
-    fn parse_line_inner(
-        &mut self,
-        line: &str,
-        syntax_set: &SyntaxSet,
-    ) -> Result, ParsingError> {
-        self.parse_line_inner_from(line, syntax_set, 0)
-    }
-
-    /// Parse `line` starting at `start_at` rather than column 0. Used by
-    /// cross-line `fail` replay: the first buffered line's pre-branch
-    /// prefix was correctly parsed under the pre-branch state, so the
-    /// replay resumes *after* the branch match under the new alternative.
-    /// When `start_at > 0` the `first_line` bookkeeping is skipped — the
-    /// caller has already emitted (or preserved) the initial
-    /// meta_content_scope push.
-    fn parse_line_inner_from(
-        &mut self,
-        line: &str,
-        syntax_set: &SyntaxSet,
-        start_at: usize,
-    ) -> Result, ParsingError> {
-        let mut match_start = start_at;
-        let mut res = Vec::new();
-
-        if start_at == 0 && self.first_line {
-            let cur_level = &self.stack[self.stack.len() - 1];
-            let context = syntax_set.get_context(&cur_level.context)?;
-            if !context.meta_content_scope.is_empty() {
-                res.push((0, ScopeStackOp::Push(context.meta_content_scope[0])));
-            }
-            self.first_line = false;
-        }
-
-        let mut regions = Region::new();
-        let fnv = BuildHasherDefault::::default();
-        let mut search_cache: SearchCache = HashMap::with_capacity_and_hasher(128, fnv);
-        // Used for detecting loops with push/pop, see long comment above.
-        let mut non_consuming_push_at = (0, 0);
-
-        while self.parse_next_token(
-            line,
-            syntax_set,
-            &mut match_start,
-            &mut search_cache,
-            &mut regions,
-            &mut non_consuming_push_at,
-            &mut res,
-        )? {}
-
-        Ok(res)
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    fn parse_next_token(
-        &mut self,
-        line: &str,
-        syntax_set: &SyntaxSet,
-        start: &mut usize,
-        search_cache: &mut SearchCache,
-        regions: &mut Region,
-        non_consuming_push_at: &mut (usize, usize),
-        ops: &mut Vec<(usize, ScopeStackOp)>,
-    ) -> Result {
-        let check_pop_loop = {
-            let (pos, stack_depth) = *non_consuming_push_at;
-            pos == *start && stack_depth == self.stack.len()
-        };
-
-        // Trim proto_starts that are no longer valid
-        while self
-            .proto_starts
-            .last()
-            .map(|start| *start >= self.stack.len())
-            .unwrap_or(false)
-        {
-            self.proto_starts.pop();
-        }
-
-        let best_match = self.find_best_match(
-            line,
-            *start,
-            syntax_set,
-            search_cache,
-            regions,
-            check_pop_loop,
-        )?;
-
-        if let Some(reg_match) = best_match {
-            // Check if this is an escape match (sentinel pat_index)
-            if reg_match.pat_index == usize::MAX {
-                let (match_start, match_end) = reg_match.regions.pos(0).unwrap();
-                *start = match_end;
-                self.exec_escape(
-                    reg_match.escape_index,
-                    match_start,
-                    match_end,
-                    ®_match.regions,
-                    syntax_set,
-                    ops,
-                )?;
-                search_cache.clear();
-                return Ok(true);
-            }
-
-            if reg_match.would_loop {
-                // A push that doesn't consume anything (a regex that resulted
-                // in an empty match at the current position) can not be
-                // followed by a non-consuming pop. Otherwise we're back where
-                // we started and would try the same sequence of matches again,
-                // resulting in an infinite loop. In this case, Sublime Text
-                // advances one character and tries again, thus preventing the
-                // loop.
-
-                // println!("pop_would_loop for match {:?}, start {}", reg_match, *start);
-
-                // nth(1) gets the next character if there is one. Need to do
-                // this instead of just += 1 because we have byte indices and
-                // unicode characters can be more than 1 byte.
-                if let Some((i, _)) = line[*start..].char_indices().nth(1) {
-                    *start += i;
-                    return Ok(true);
-                } else {
-                    // End of line, no character to advance and no point trying
-                    // any more patterns.
-                    return Ok(false);
-                }
-            }
-
-            let match_end = reg_match.regions.pos(0).unwrap().1;
-
-            // Check if this is a Fail operation — handle before advancing start
-            let context = reg_match.context;
-            let match_pattern = context.match_at(reg_match.pat_index)?;
-            if let MatchOperation::Fail(_) = match_pattern.operation {
-                let level_context = {
-                    let id = &self.stack[self.stack.len() - 1].context;
-                    syntax_set.get_context(id)?
-                };
-                return self.exec_pattern(
-                    line,
-                    ®_match,
-                    level_context,
-                    syntax_set,
-                    start,
-                    non_consuming_push_at,
-                    ops,
-                    search_cache,
-                );
-            }
-
-            let consuming = match_end > *start;
-            if !consuming {
-                // The match doesn't consume any characters. If this is a
-                // "push", remember the position and stack size so that we can
-                // check the next "pop" for loops. Otherwise leave the state,
-                // e.g. non-consuming "set" could also result in a loop.
-                if matches!(
-                    match_pattern.operation,
-                    MatchOperation::Push(_)
-                        | MatchOperation::Branch { .. }
-                        | MatchOperation::Embed { .. }
-                ) {
-                    *non_consuming_push_at = (match_end, self.stack.len() + 1);
-                }
-            }
-
-            *start = match_end;
-
-            // ignore `with_prototype`s below this if a context is pushed
-            if reg_match.from_with_prototype {
-                // use current height, since we're before the actual push
-                self.proto_starts.push(self.stack.len());
-            }
-
-            let level_context = {
-                let id = &self.stack[self.stack.len() - 1].context;
-                syntax_set.get_context(id)?
-            };
-            self.exec_pattern(
-                line,
-                ®_match,
-                level_context,
-                syntax_set,
-                start,
-                non_consuming_push_at,
-                ops,
-                search_cache,
-            )?;
-
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn find_best_match<'a>(
-        &self,
-        line: &str,
-        start: usize,
-        syntax_set: &'a SyntaxSet,
-        search_cache: &mut SearchCache,
-        regions: &mut Region,
-        check_pop_loop: bool,
-    ) -> Result>, ParsingError> {
-        let cur_level = &self.stack[self.stack.len() - 1];
-        let context = syntax_set.get_context(&cur_level.context)?;
-        let prototype = if let Some(ref p) = context.prototype {
-            Some(p)
-        } else {
-            None
-        };
-
-        // Build an iterator for the contexts we want to visit in order
-        let context_chain = {
-            let proto_start = self.proto_starts.last().cloned().unwrap_or(0);
-            // Sublime applies with_prototypes from bottom to top
-            let with_prototypes = self.stack[proto_start..].iter().flat_map(|lvl| {
-                lvl.prototypes
-                    .iter()
-                    .map(move |ctx| (true, ctx, lvl.captures.as_ref()))
-            });
-            let cur_prototype = prototype.into_iter().map(|ctx| (false, ctx, None));
-            let cur_context =
-                Some((false, &cur_level.context, cur_level.captures.as_ref())).into_iter();
-            with_prototypes.chain(cur_prototype).chain(cur_context)
-        };
-
-        // println!("{:#?}", cur_level);
-        // println!("token at {} on {}", start, line.trim_right());
-
-        // Check escape patterns first — they take strict precedence.
-        // If an escape matches at `start`, return it immediately as a synthetic match.
-        // If it matches later, truncate the search region for normal patterns.
-        let mut search_end = line.len();
-        let mut escape_match: Option<(usize, Region)> = None; // (escape_stack_index, region)
-
-        for (ei, entry) in self.escape_stack.iter().enumerate() {
-            let mut esc_regions = Region::new();
-            if entry
-                .regex
-                .search(line, start, line.len(), Some(&mut esc_regions), true)
-            {
-                let (esc_start, _esc_end) = esc_regions.pos(0).unwrap();
-                if esc_start < search_end {
-                    search_end = esc_start;
-                    escape_match = Some((ei, esc_regions));
-                }
-            }
-        }
-
-        // If escape matches right at `start`, it wins immediately — no need to
-        // search normal patterns.
-        if let Some((ei, ref esc_region)) = escape_match {
-            let esc_start = esc_region.pos(0).unwrap().0;
-            if esc_start == start {
-                return Ok(Some(RegexMatch {
-                    regions: esc_region.clone(),
-                    context: syntax_set.get_context(&cur_level.context)?,
-                    pat_index: usize::MAX, // sentinel for escape match
-                    from_with_prototype: false,
-                    would_loop: false,
-                    escape_index: ei,
-                }));
-            }
-        }
-
-        let mut min_start = usize::MAX;
-        let mut best_match: Option> = None;
-        let mut pop_would_loop = false;
-
-        for (from_with_proto, ctx, captures) in context_chain {
-            for (pat_context, pat_index) in context_iter(syntax_set, syntax_set.get_context(ctx)?) {
-                let match_pat = pat_context.match_at(pat_index)?;
-
-                if let Some(match_region) = self.search_with_end(
-                    line,
-                    start,
-                    search_end,
-                    match_pat,
-                    captures,
-                    search_cache,
-                    regions,
-                ) {
-                    let (match_start, match_end) = match_region.pos(0).unwrap();
-
-                    // println!("matched pattern {:?} at start {} end {} (pop would loop: {}, min start: {}, initial start: {}, check_pop_loop: {}, stack_len: {})", match_pat, match_start, match_end, pop_would_loop, min_start, start, check_pop_loop, self.stack.len());
-
-                    if match_start < min_start || (match_start == min_start && pop_would_loop) {
-                        // New match is earlier in text than old match,
-                        // or old match was a looping pop at the same
-                        // position.
-
-                        // println!("setting as current match");
-
-                        min_start = match_start;
-
-                        let consuming = match_end > start;
-                        // A non-consuming `pop: N` after a non-consuming push
-                        // only loops when N == 1 — that restores the exact
-                        // pre-push stack, so the push rule fires again. With
-                        // N >= 2 the stack drops strictly below the pre-push
-                        // depth, so the outer context no longer has the same
-                        // trigger in scope (e.g. Haskell's `immediately-pop2`
-                        // as the fallback branch alternative for
-                        // `declaration-type-end`).
-                        pop_would_loop = check_pop_loop
-                            && !consuming
-                            && matches!(match_pat.operation, MatchOperation::Pop(1));
-
-                        let push_too_deep = matches!(
-                            match_pat.operation,
-                            MatchOperation::Push(_)
-                                | MatchOperation::Branch { .. }
-                                | MatchOperation::Embed { .. }
-                        ) && self.stack.len() >= 100;
-
-                        if push_too_deep {
-                            return Ok(None);
-                        }
-
-                        best_match = Some(RegexMatch {
-                            regions: match_region,
-                            context: pat_context,
-                            pat_index,
-                            from_with_prototype: from_with_proto,
-                            would_loop: pop_would_loop,
-                            escape_index: 0, // not an escape match
-                        });
-
-                        if match_start == start && !pop_would_loop {
-                            // We're not gonna find a better match after this,
-                            // so as an optimization we can stop matching now.
-                            return Ok(best_match);
-                        }
-                    }
-                }
-            }
-        }
-
-        // If no normal match was found before the escape position, or escape
-        // position is earlier, use the escape match.
-        if let Some((ei, esc_region)) = escape_match {
-            let esc_start = esc_region.pos(0).unwrap().0;
-            if esc_start < min_start || (esc_start == min_start && pop_would_loop) {
-                return Ok(Some(RegexMatch {
-                    regions: esc_region,
-                    context: syntax_set.get_context(&cur_level.context)?,
-                    pat_index: usize::MAX, // sentinel for escape match
-                    from_with_prototype: false,
-                    would_loop: false,
-                    escape_index: ei,
-                }));
-            }
-        }
-
-        Ok(best_match)
-    }
-
-    fn search_with_end(
-        &self,
-        line: &str,
-        start: usize,
-        search_end: usize,
-        match_pat: &MatchPattern,
-        captures: Option<&(Region, String)>,
-        search_cache: &mut SearchCache,
-        regions: &mut Region,
-    ) -> Option {
-        // println!("{} - {:?} - {:?}", match_pat.regex_str, match_pat.has_captures, cur_level.captures.is_some());
-        let match_ptr = match_pat as *const MatchPattern;
-
-        if let Some(maybe_region) = search_cache.get(&match_ptr) {
-            if let Some(ref region) = *maybe_region {
-                let (cached_start, cached_end) = region.pos(0).unwrap();
-                if cached_start >= start && cached_end <= search_end {
-                    // Cached match is valid within the truncated region.
-                    return Some(region.clone());
-                } else if cached_start >= start && cached_start < search_end {
-                    // Match starts within range but extends past search_end.
-                    // Can't use cache — need to re-search. Fall through below.
-                } else if cached_start >= search_end {
-                    // Cached match is beyond our search end — treat as no match
-                    return None;
-                }
-                // cached_start < start: cache miss, re-search below
-            } else {
-                // Didn't find a match earlier, so no point trying to match it again
-                return None;
-            }
-        }
-
-        let (regex, can_cache) = match (match_pat.has_captures, captures) {
-            (true, Some(captures)) => {
-                let (region, s) = captures;
-                (&match_pat.regex_with_refs(region, s), false)
-            }
-            _ => (match_pat.regex(), true),
-        };
-        // Only `MatchOperation::None` patterns must avoid zero-length matches; every other
-        // operation legitimately needs them (lookaheads with branch/fail, empty patterns with
-        // pop/set, etc.). The regex engine handles this via its `FIND_NOT_EMPTY` option.
-        let allow_empty = !matches!(match_pat.operation, MatchOperation::None);
-        // print!("  executing regex: {:?} at pos {} on line {}", regex.regex_str(), start, line);
-        let matched = regex.search(line, start, search_end, Some(regions), allow_empty);
-
-        if matched {
-            let (match_start, match_end) = regions.pos(0).unwrap();
-            // this is necessary to avoid infinite looping on dumb patterns
-            let does_something = match match_pat.operation {
-                MatchOperation::None => match_start != match_end,
-                MatchOperation::Push(_)
-                | MatchOperation::Branch { .. }
-                | MatchOperation::Embed { .. } => self.stack.len() < 100,
-                _ => true,
-            };
-            if can_cache && does_something && search_end == line.len() {
-                // Only cache when searching the full line — truncated searches
-                // could give different results for later positions.
-                search_cache.insert(match_pat, Some(regions.clone()));
-            }
-            if does_something {
-                // print!("catch {} at {} on {}", match_pat.regex_str, match_start, line);
-                return Some(regions.clone());
-            }
-        } else if can_cache && search_end == line.len() {
-            search_cache.insert(match_pat, None);
-        }
-        None
-    }
-
-    /// Returns true if the stack was changed.
-    /// For `Fail` operations, returns `Ok(true)` if backtracking was performed
-    /// (caller should continue parsing from the rewound position).
-    fn exec_pattern<'a>(
-        &mut self,
-        line: &str,
-        reg_match: &RegexMatch<'a>,
-        level_context: &'a Context,
-        syntax_set: &'a SyntaxSet,
-        start: &mut usize,
-        non_consuming_push_at: &mut (usize, usize),
-        ops: &mut Vec<(usize, ScopeStackOp)>,
-        search_cache: &mut SearchCache,
-    ) -> Result {
-        let (match_start, match_end) = reg_match.regions.pos(0).unwrap();
-        let context = reg_match.context;
-        let pat = context.match_at(reg_match.pat_index)?;
-
-        // Handle Fail: attempt backtracking
-        if let MatchOperation::Fail(ref name) = pat.operation {
-            return self.handle_fail(
-                name,
-                line,
-                start,
-                non_consuming_push_at,
-                ops,
-                search_cache,
-                syntax_set,
-            );
-        }
-
-        // For Branch, we need to snapshot state before executing, then synthesize a Push.
-        let is_branch = matches!(pat.operation, MatchOperation::Branch { .. });
-        let synthetic_op;
-
-        if is_branch {
-            if let MatchOperation::Branch {
-                ref name,
-                ref alternatives,
-                pop_count,
-            } = pat.operation
-            {
-                // Snapshot current state.
-                //
-                // NOTE on field naming: `match_start` here stores the
-                // position the parser should *resume* from on fail —
-                // which is the branch match's end position (since the
-                // parser has already consumed the match). `match_end`
-                // and `pat_scope` carry the *real* match span plus the
-                // keyword's own scopes so a same-line fail rewind can
-                // re-emit them (they were truncated off `ops` along
-                // with the alt[0]'s subsequent work).
-                let bp = BranchPoint {
-                    name: name.clone(),
-                    next_alternative: 1, // 0 is about to be pushed
-                    alternatives: alternatives.clone(),
-                    stack_snapshot: self.stack.clone(),
-                    proto_starts_snapshot: self.proto_starts.clone(),
-                    match_start: *start, // position before this match's advance
-                    trigger_match_start: match_start,
-                    pat_scope: pat.scope.clone(),
-                    line_number: self.line_number.saturating_sub(1), // current line (already incremented)
-                    ops_snapshot_len: ops.len(),
-                    stack_depth: self.stack.len(),
-                    non_consuming_push_at_snapshot: *non_consuming_push_at,
-                    first_line_snapshot: self.first_line,
-                    with_prototype: pat.with_prototype.clone(),
-                    pending_lines_snapshot_len: self.pending_lines.len(),
-                    escape_stack_snapshot: self.escape_stack.clone(),
-                    pop_count,
-                    prefix_ops: ops.clone(),
-                    capture_ops: pat
-                        .captures
-                        .as_ref()
-                        .map(|m| build_capture_ops(m, ®_match.regions))
-                        .unwrap_or_default(),
-                };
-                self.branch_points.push(bp);
-                // When pop_count > 0 (pop + branch), use Set semantics to
-                // pop the current context before pushing the first alternative.
-                synthetic_op = if pop_count > 0 {
-                    MatchOperation::Set {
-                        ctx_refs: vec![alternatives[0].clone()],
-                        pop_count,
-                    }
-                } else {
-                    MatchOperation::Push(vec![alternatives[0].clone()])
-                };
-            } else {
-                unreachable!()
-            }
-        } else {
-            synthetic_op = pat.operation.clone();
-        }
-
-        let op_to_use = if is_branch {
-            &synthetic_op
-        } else {
-            &pat.operation
-        };
-
-        self.push_meta_ops(true, match_start, level_context, op_to_use, syntax_set, ops)?;
-        for s in &pat.scope {
-            ops.push((match_start, ScopeStackOp::Push(*s)));
-        }
-        let capture_ops = pat
-            .captures
-            .as_ref()
-            .map(|m| build_capture_ops(m, ®_match.regions))
-            .unwrap_or_default();
-        ops.extend(capture_ops.iter().cloned());
-        if !pat.scope.is_empty() {
-            ops.push((match_end, ScopeStackOp::Pop(pat.scope.len())));
-        }
-        self.push_meta_ops(false, match_end, level_context, op_to_use, syntax_set, ops)?;
-
-        if is_branch {
-            // Execute the synthetic Push through perform_op
-            let synthetic_pat = MatchPattern::new(
-                pat.has_captures,
-                pat.regex.regex_str().to_string(),
-                pat.scope.clone(),
-                pat.captures.clone(),
-                synthetic_op,
-                pat.with_prototype.clone(),
-            );
-            self.perform_op(line, ®_match.regions, &synthetic_pat, syntax_set)
-        } else {
-            self.perform_op(line, ®_match.regions, pat, syntax_set)
-        }
-    }
-
-    /// Handle a `fail` operation by rewinding to the named branch point.
-    /// Returns Ok(true) if backtracking happened (caller should continue from rewound position).
-    /// Returns Ok(false) if the fail had no effect.
-    fn handle_fail(
-        &mut self,
-        name: &str,
-        line: &str,
-        start: &mut usize,
-        non_consuming_push_at: &mut (usize, usize),
-        ops: &mut Vec<(usize, ScopeStackOp)>,
-        search_cache: &mut SearchCache,
-        syntax_set: &SyntaxSet,
-    ) -> Result {
-        // Find the branch point by name (most recent first), skipping
-        // records whose alternative's pushed frame is no longer on
-        // the stack. The alternative lives at
-        // `bp.stack_depth - bp.pop_count + 1`, so `stack.len() >
-        // bp.stack_depth - bp.pop_count` means the frame is still
-        // present. Without this skip, a nested `branch_point` with
-        // the same name whose inner alternative popped cleanly would
-        // shadow an enclosing record via `rposition`, rewinding to
-        // the inner branch position instead of the outer one
-        // (Haskell's raw-string QQ `[r|[a-zA-Z]|]`).
-        let stack_len = self.stack.len();
-        let bp_index = self.branch_points.iter().rposition(|bp| {
-            bp.name == name && stack_len > bp.stack_depth.saturating_sub(bp.pop_count)
-        });
-        let bp_index = match bp_index {
-            Some(i) => i,
-            None => return Ok(false), // No such branch point, fail is no-op
-        };
-
-        let cur_line = self.line_number.saturating_sub(1);
-        let bp = &self.branch_points[bp_index];
-
-        // Check validity: not >128 lines old
-        if cur_line.saturating_sub(bp.line_number) > 128 {
-            let bp = self.branch_points.remove(bp_index);
-            self.warnings.push(format!(
-                "branch point '{}' expired (exceeded 128-line rewind limit)",
-                bp.name
-            ));
-            return Ok(false);
-        }
-
-        // Check validity: stack depth still >= branch's stack_depth
-        if self.stack.len() < bp.stack_depth {
-            self.branch_points.remove(bp_index);
-            return Ok(false);
-        }
-
-        // Check if there are more alternatives
-        if bp.next_alternative >= bp.alternatives.len() {
-            // All alternatives exhausted: restore parser state to the
-            // pre-branch snapshot so the stuck alternative's pushed
-            // contexts and emitted ops are discarded, then advance
-            // past the branch_point match position by one character so
-            // we don't immediately re-enter the same branch_point and
-            // loop. Before this, the branch_point was silently removed
-            // while its last alternative's contexts remained on the
-            // stack — the cause of the "scope stack stays in
-            // `meta.interpolation.brace.shell`" cascade in Zsh when
-            // both `brace-interpolation-sequence` and
-            // `brace-interpolation-series` failed and there was no
-            // fallback alternative (Zsh explicitly excludes
-            // `brace-interpolation-fallback`).
-            //
-            // Cross-line exhaustion takes the same shape, plus a replay
-            // of the buffered lines under the pre-branch state so
-            // callers see corrected ops for lines they've already been
-            // handed. Without this, the unterminated TypeScript type
-            // expression at `sublimehq/Packages#3598`
-            // (`type x = { bar: (cb: (\n};`) left the inner
-            // `ts-type-function-parameter-list-body` on the stack
-            // forever, contaminating every subsequent line's scope
-            // stack with `meta.type.js, meta.group.js` — 274 cascading
-            // assertion failures in `syntax_test_typescript.ts`.
-            let is_cross_line = bp.line_number < cur_line;
-            let stack_snapshot = bp.stack_snapshot.clone();
-            let proto_starts_snapshot = bp.proto_starts_snapshot.clone();
-            let escape_stack_snapshot = bp.escape_stack_snapshot.clone();
-            let first_line_snapshot = bp.first_line_snapshot;
-            let non_consuming_push_at_snapshot = bp.non_consuming_push_at_snapshot;
-            let ops_snapshot_len = bp.ops_snapshot_len;
-            let match_start_pos = bp.match_start;
-            let pending_lines_snapshot_len = bp.pending_lines_snapshot_len;
-            let prefix_ops = bp.prefix_ops.clone();
-            self.branch_points.remove(bp_index);
-
-            self.stack = stack_snapshot;
-            self.proto_starts = proto_starts_snapshot;
-            self.escape_stack = escape_stack_snapshot;
-            self.first_line = first_line_snapshot;
-            *non_consuming_push_at = non_consuming_push_at_snapshot;
-            ops.truncate(ops_snapshot_len.min(ops.len()));
-
-            if is_cross_line {
-                // Re-parse each buffered line under the restored (pre-branch)
-                // state so `parse_line` can surface the corrected ops via
-                // `ParseLineOutput::replayed`. The first buffered line is the
-                // branch-creation line: emit its saved `prefix_ops` (the ops
-                // emitted before the branch match) verbatim, then advance past
-                // the branch match by one character before resuming — otherwise
-                // the same branch_point would fire again at the original match
-                // position and we'd loop.
-                //
-                // Keep `pending_lines` intact (don't drain): if an outer
-                // branch_point on this same line also fails after this
-                // exhaustion replay, its own replay needs access to the same
-                // buffered lines.
-                let truncated_lines: Vec =
-                    self.pending_lines[pending_lines_snapshot_len..].to_vec();
-                let mut replayed_ops: Vec> =
-                    Vec::with_capacity(truncated_lines.len());
-                for (i, replay_line) in truncated_lines.iter().enumerate() {
-                    let line_ops = if i == 0 {
-                        let mut first_line_ops = prefix_ops.clone();
-                        let resume_at = if let Some((j, _)) =
-                            replay_line[match_start_pos..].char_indices().nth(1)
-                        {
-                            match_start_pos + j
-                        } else {
-                            replay_line.len()
-                        };
-                        let tail_ops =
-                            self.parse_line_inner_from(replay_line, syntax_set, resume_at)?;
-                        first_line_ops.extend(tail_ops);
-                        first_line_ops
-                    } else {
-                        self.parse_line_inner(replay_line, syntax_set)?
-                    };
-                    replayed_ops.push(line_ops);
-                }
-                self.flushed_ops.extend(replayed_ops);
-
-                // Restart the current line from the beginning under the
-                // restored state.
-                ops.clear();
-                *start = 0;
-                *non_consuming_push_at = (0, 0);
-                search_cache.clear();
-                return Ok(true);
-            }
-
-            // Same-line exhaustion: advance one char past the branch_point match
-            // to avoid immediately re-matching the same `(?=...)` lookahead.
-            if let Some((i, _)) = line[match_start_pos..].char_indices().nth(1) {
-                *start = match_start_pos + i;
-            } else {
-                // End of line — no character to advance past.
-                *start = line.len();
-            }
-            search_cache.clear();
-            return Ok(true);
-        }
-
-        // Determine if this is a cross-line fail (branch was created on a previous line).
-        let is_cross_line = bp.line_number < cur_line;
-
-        // Extract everything we need from bp before mutating self.
-        let next_alt_index = bp.next_alternative;
-        let next_alt = bp.alternatives[next_alt_index].clone();
-        let match_start_pos = bp.match_start;
-        let trigger_match_start = bp.trigger_match_start;
-        let trigger_pat_scope = bp.pat_scope.clone();
-        let trigger_capture_ops = bp.capture_ops.clone();
-        let stack_snapshot = bp.stack_snapshot.clone();
-        let proto_starts_snapshot = bp.proto_starts_snapshot.clone();
-        let first_line_snapshot = bp.first_line_snapshot;
-        let non_consuming_push_at_snapshot = bp.non_consuming_push_at_snapshot;
-        let ops_snapshot_len = bp.ops_snapshot_len;
-        let pending_lines_snapshot_len = bp.pending_lines_snapshot_len;
-        let escape_stack_snapshot = bp.escape_stack_snapshot.clone();
-        let prefix_ops = bp.prefix_ops.clone();
-        // bp borrow ends here.
-
-        let pop_count = self.branch_points[bp_index].pop_count;
-
-        // Restore parser state to the snapshot.
-        self.stack = stack_snapshot;
-        self.proto_starts = proto_starts_snapshot;
-        self.escape_stack = escape_stack_snapshot;
-        self.first_line = first_line_snapshot;
-        *non_consuming_push_at = non_consuming_push_at_snapshot;
-
-        // Update the branch point record before popping/pushing
-        // (must happen before the pop which may invalidate indices).
-        self.branch_points[bp_index].next_alternative = next_alt_index + 1;
-
-        // For pop + branch: re-pop the contexts (snapshot was taken pre-pop).
-        if pop_count > 0 {
-            for _ in 0..pop_count {
-                self.stack.pop();
-            }
-        }
-
-        // Push the next alternative onto the stack.
-        let with_prototype = self.branch_points[bp_index].with_prototype.clone();
-        let context_id = next_alt.id()?;
-        let context = syntax_set.get_context(&context_id)?;
-        let captures = None; // no captures available at rewind time
-
-        let proto_ids = match with_prototype {
-            Some(ref p) => vec![p.id()?],
-            None => Vec::new(),
-        };
-
-        self.stack.push(StateLevel {
-            context: context_id,
-            prototypes: proto_ids,
-            captures,
-        });
-
-        if is_cross_line {
-            // Cross-line fail: the ops for lines since the branch was created
-            // have already been returned to callers.  Re-parse those lines under
-            // the new alternative and store the corrected ops in `flushed_ops`
-            // so that `parse_line` can surface them via `ParseLineOutput::replayed`.
-            //
-            // The first buffered line is the branch-creation line. Its
-            // pre-branch prefix (cols 0..trigger_match_start) was correctly
-            // parsed under the *pre-branch* state — not the new alternative.
-            // Re-parsing it from column 0 with the new alternative on the
-            // stack would misattribute that prefix to the new alternative's
-            // rules (observed on multi-line SQL `LIKE … ESCAPE …`: every
-            // non-whitespace before `LIKE` fires `else-pop` in the
-            // escape-alternative, derailing the stack). Instead, reuse the
-            // prefix_ops saved at branch-creation time, manually emit the
-            // branch trigger's pat.scope and the new alternative's meta
-            // scope ops, then resume parsing from match_end with the new
-            // alternative on the stack via `parse_line_inner_from`.
-            // Keep `pending_lines` intact (don't drain): if a second branch_point
-            // on the current line also fails after this retry, its own replay
-            // needs access to the same buffered lines. Nested branches from the
-            // same earlier line share the buffer.
-            let truncated_lines: Vec =
-                self.pending_lines[pending_lines_snapshot_len..].to_vec();
-
-            let mut replayed_ops: Vec> =
-                Vec::with_capacity(truncated_lines.len());
-            for (i, replay_line) in truncated_lines.iter().enumerate() {
-                let line_ops = if i == 0 {
-                    // First buffered line: compose prefix + branch ops + resume.
-                    let mut first_line_ops = prefix_ops.clone();
-                    // Re-emit the trigger's pat.scope and the new
-                    // alternative's meta scope ops in the same order
-                    // the non-fail push path uses: clear_scopes and
-                    // meta_scope at `trigger_match_start` (so the
-                    // matched text sees them), then pat.scope at the
-                    // same position, popped at `match_start_pos`.
-                    // meta_content_scope only applies after the
-                    // matched text, so it lands at `match_start_pos`.
-                    if let Some(clear_amount) = context.clear_scopes {
-                        first_line_ops
-                            .push((trigger_match_start, ScopeStackOp::Clear(clear_amount)));
-                    }
-                    for scope in context.meta_scope.iter() {
-                        first_line_ops.push((trigger_match_start, ScopeStackOp::Push(*scope)));
-                    }
-                    for scope in &trigger_pat_scope {
-                        first_line_ops.push((trigger_match_start, ScopeStackOp::Push(*scope)));
-                    }
-                    // See matching comment in the same-line branch below —
-                    // re-emit the trigger match's captures inside the
-                    // pat_scope brackets so they survive the branch swap.
-                    first_line_ops.extend(trigger_capture_ops.iter().cloned());
-                    if !trigger_pat_scope.is_empty() {
-                        first_line_ops
-                            .push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len())));
-                    }
-                    for scope in context.meta_content_scope.iter() {
-                        first_line_ops.push((match_start_pos, ScopeStackOp::Push(*scope)));
-                    }
-                    // Resume parsing from the branch match's end position.
-                    let tail_ops =
-                        self.parse_line_inner_from(replay_line, syntax_set, match_start_pos)?;
-                    first_line_ops.extend(tail_ops);
-                    first_line_ops
-                } else {
-                    self.parse_line_inner(replay_line, syntax_set)?
-                };
-                replayed_ops.push(line_ops);
-            }
-            // Append (rather than overwrite) in case multiple cross-line fails
-            // fire on the same parse_line call.
-            self.flushed_ops.extend(replayed_ops);
-
-            // Restart the current line from the beginning.
-            ops.clear();
-            *start = 0;
-            *non_consuming_push_at = (0, 0);
-
-            // Guard: the replayed `parse_line_inner` calls above can
-            // mutate `self.branch_points` (adding new branches,
-            // removing expired or exhausted ones), which can shift or
-            // invalidate `bp_index`. Indexing with the stale position
-            // previously panicked outright on files that exercise
-            // nested cross-line branching (observed on
-            // `JavaScript/syntax_test_js.js` and
-            // `syntax_test_typescript.ts`). Skip the bookkeeping if
-            // the branch point has been removed — the replay already
-            // completed, which is the essential work of the fail.
-            if bp_index < self.branch_points.len() {
-                self.branch_points[bp_index].ops_snapshot_len = 0;
-            }
-        } else {
-            // Same-line fail: truncate ops back to the snapshot point and rewind.
-            ops.truncate(ops_snapshot_len.min(ops.len()));
-            *start = match_start_pos;
-
-            // Keep `ops_snapshot_len` pointing at the pre-branch state.
-            // Subsequent fails on the same branch_point must truncate
-            // back to *here* — not to the position after the pat.scope
-            // re-emit below — otherwise a second fail would preserve
-            // the first re-emit's (Push at trigger_match_start) while
-            // appending another re-emit, producing the disordered
-            // sequence (trigger, Push), (match_end, Pop), (trigger,
-            // Push), (match_end, Pop). `ScopeRegionIterator` then
-            // panics in `easy.rs` because position goes backwards.
-            self.branch_points[bp_index].ops_snapshot_len = ops.len();
-
-            // Re-emit the branch_point match's own scopes over their
-            // original span. Without this, keywords that trigger a
-            // branch (e.g. `LIKE` with
-            // `scope: keyword.operator.comparison.sql`) lose their
-            // scope whenever alt[0] fails and alt[1..] succeeds,
-            // because the original Push/Pop pair was truncated off
-            // `ops` together with alt[0]'s subsequent work.
-            //
-            // The new alternative's `clear_scopes` and `meta_scope`
-            // are emitted at `trigger_match_start` *before* the
-            // trigger's `pat.scope`, mirroring the non-fail push path
-            // in `push_meta_ops` (initial phase): meta_scope sits
-            // below the match scope on the stack so the matched text
-            // sees both. Placing them at `match_start_pos` would mean
-            // the trigger character (e.g. `(` of `for (var i = 0; …)`)
-            // never sees the alternative's `meta_scope`. The
-            // `meta_content_scope` legitimately stays at
-            // `match_start_pos` — mcs only applies after the matched
-            // text.
-            if let Some(clear_amount) = context.clear_scopes {
-                ops.push((trigger_match_start, ScopeStackOp::Clear(clear_amount)));
-            }
-            for scope in context.meta_scope.iter() {
-                ops.push((trigger_match_start, ScopeStackOp::Push(*scope)));
-            }
-            for scope in &trigger_pat_scope {
-                ops.push((trigger_match_start, ScopeStackOp::Push(*scope)));
-            }
-            // Captures emitted alongside the original pat.scope (e.g.
-            // `keyword.declaration.data.haskell` on the first capture of
-            // `(data)(?:\s+(family|instance))?`) were truncated off with
-            // alt[0]'s ops. Re-emit them inside the pat_scope brackets so
-            // the keyword scope survives the branch swap.
-            ops.extend(trigger_capture_ops.iter().cloned());
-            if !trigger_pat_scope.is_empty() {
-                ops.push((match_start_pos, ScopeStackOp::Pop(trigger_pat_scope.len())));
-            }
-            for scope in context.meta_content_scope.iter() {
-                ops.push((match_start_pos, ScopeStackOp::Push(*scope)));
-            }
-        }
-
-        // Clear search cache since we're rewinding.
-        search_cache.clear();
-
-        Ok(true)
-    }
-
-    /// Get the syntax version for the current parse state
-    fn current_syntax_version(&self, syntax_set: &SyntaxSet) -> u32 {
-        if let Some(level) = self.stack.last() {
-            let syntax_index = level.context.syntax_index;
-            syntax_set
-                .syntaxes()
-                .get(syntax_index)
-                .map_or(1, |s| s.version)
-        } else {
-            1
-        }
-    }
-
-    fn push_meta_ops(
-        &self,
-        initial: bool,
-        index: usize,
-        cur_context: &Context,
-        match_op: &MatchOperation,
-        syntax_set: &SyntaxSet,
-        ops: &mut Vec<(usize, ScopeStackOp)>,
-    ) -> Result<(), ParsingError> {
-        let version = self.current_syntax_version(syntax_set);
-        // println!("metas ops for {:?}, initial: {}",
-        //          match_op,
-        //          initial);
-        // println!("{:?}", cur_context.meta_scope);
-        match *match_op {
-            MatchOperation::Pop(n) => {
-                // For `pop: N` with N > 1, every context being popped
-                // contributes scope atoms on the scope stack that must
-                // be unwound in LIFO order. The TOP context's trigger
-                // text must not see its own `meta_content_scope`, so
-                // that one is popped in the initial phase; all other
-                // scope unwinding (top context's `meta_scope`, then
-                // each deeper context's `meta_content_scope` followed
-                // by its `meta_scope`) happens in the non-initial
-                // phase, immediately after the match text's own
-                // scope has been popped.
-                //
-                // Before this fix only the top context's scopes were
-                // ever popped, leaving the N-1 deeper contexts'
-                // `meta_scope` / `meta_content_scope` atoms orphaned —
-                // the cause of the "scope stack grows unboundedly"
-                // cascade in Makefile and Zsh (Category A).
-                let stack_len = self.stack.len();
-                let pop_count = n.min(stack_len);
-                if initial {
-                    // v2: if the context immediately below the top has
-                    // embed_scope_replaces, cur_context's meta_content_scope
-                    // was never pushed, so don't generate a Pop for it.
-                    let skip = version >= 2
-                        && stack_len >= 2
-                        && syntax_set
-                            .get_context(&self.stack[stack_len - 2].context)
-                            .map(|c| c.embed_scope_replaces)
-                            .unwrap_or(false);
-                    if !skip && !cur_context.meta_content_scope.is_empty() {
-                        ops.push((
-                            index,
-                            ScopeStackOp::Pop(cur_context.meta_content_scope.len()),
-                        ));
-                    }
-                } else {
-                    // Top context's meta_scope comes off first (it sat
-                    // immediately below the trigger text's scope on the
-                    // stack).
-                    if !cur_context.meta_scope.is_empty() {
-                        ops.push((index, ScopeStackOp::Pop(cur_context.meta_scope.len())));
-                    }
-                    // Each deeper context's scopes are popped in
-                    // top-to-bottom order: meta_content_scope first
-                    // (pushed after its own meta_scope, hence above on
-                    // the stack), then meta_scope, then any Restore
-                    // paired with that context's own `clear_scopes`
-                    // (mirrors the push order Clear → meta_scope → mcs
-                    // in reverse). Without the Restore, a `pop: N`
-                    // from the top that also unwinds a deeper context
-                    // with `clear_scopes` leaves the originally cleared
-                    // atoms orphaned — observed on Python f/t-string
-                    // interpolation close, where `f-string-replacement-end`
-                    // fires `pop: 2` that also pops
-                    // `f-string-replacement-meta`.
-                    for depth in 1..pop_count {
-                        let level_idx = stack_len - 1 - depth;
-                        let ctx = syntax_set.get_context(&self.stack[level_idx].context)?;
-                        let skip_content = version >= 2
-                            && level_idx >= 1
-                            && syntax_set
-                                .get_context(&self.stack[level_idx - 1].context)
-                                .map(|c| c.embed_scope_replaces)
-                                .unwrap_or(false);
-                        if !skip_content && !ctx.meta_content_scope.is_empty() {
-                            ops.push((index, ScopeStackOp::Pop(ctx.meta_content_scope.len())));
-                        }
-                        if !ctx.meta_scope.is_empty() {
-                            ops.push((index, ScopeStackOp::Pop(ctx.meta_scope.len())));
-                        }
-                        if ctx.clear_scopes.is_some() {
-                            ops.push((index, ScopeStackOp::Restore));
-                        }
-                    }
-                }
-
-                // cleared scopes are restored after the scopes from match pattern that invoked the pop are applied
-                if !initial && cur_context.clear_scopes.is_some() {
-                    ops.push((index, ScopeStackOp::Restore))
-                }
-            }
-            // for some reason the ST3 behaviour of set is convoluted and is inconsistent with the docs and other ops
-            // - the meta_content_scope of the current context is applied to the matched thing, unlike pop
-            // - the clear_scopes are applied after the matched token, unlike push
-            // - the interaction with meta scopes means that the token has the meta scopes of both the current scope and the new scope.
-            MatchOperation::Push(ref context_refs)
-            | MatchOperation::Set {
-                ctx_refs: ref context_refs,
-                ..
-            } => {
-                let is_set = matches!(*match_op, MatchOperation::Set { .. });
-                let set_pop_count = match *match_op {
-                    MatchOperation::Set { pop_count, .. } => pop_count.max(1),
-                    _ => 1,
-                };
-                // a match pattern that "set"s keeps the meta_content_scope and meta_scope from the previous context
-                if initial {
-                    // v2: pop parent's meta_content_scope so matched text does not see it
-                    if is_set && version >= 2 && !cur_context.meta_content_scope.is_empty() {
-                        ops.push((
-                            index,
-                            ScopeStackOp::Pop(cur_context.meta_content_scope.len()),
-                        ));
-                    }
-                    // NOTE: cur_context.clear_scopes Restore is emitted in the
-                    // non-initial phase below, AFTER Pop(cur.meta_scope + target.meta_scope)
-                    // has run. Restoring here (pre-match) would place the cleared
-                    // scopes on top of the stack above the target.meta_scope push,
-                    // and the non-initial Pop would then remove the restored scopes
-                    // instead of the intended meta_scopes — dropping cur's cleared
-                    // state on the floor. Observed as duplicate
-                    // `meta.mapping.value.json` atoms in nested JSON objects.
-                    // add each context's meta scope
-                    if version >= 2 {
-                        // Push: emit Clear for every pushed context that has
-                        // `clear_scopes`, at its own index position in the
-                        // push order — same as v1. Sublime permits at most
-                        // one `clear_scopes` per push list, but when it sits
-                        // on a non-topmost entry (e.g. Python's
-                        // `f-string-replacement-meta` at index 0 of a
-                        // 3-context push) restricting to `i == last_idx`
-                        // silently drops it, leaking the parent's
-                        // `meta_content_scope` atoms into interpolation
-                        // content.
-                        //
-                        // Single-context `set:` with clear_scopes on the
-                        // target: emit Clear here — before target.meta_scope
-                        // is pushed and before the trigger match scope — so
-                        // the matched text sees the cleared stack.
-                        // Observed on Lisp's `(defun fn (...)`: the
-                        // parameter-list `(` otherwise kept the enclosing
-                        // `meta.function.lisp` alongside
-                        // `meta.function.parameters.lisp` because Clear
-                        // fired only after the match in the non-initial
-                        // phase. See
-                        // `v2_set_to_target_with_clear_scopes_clears_parent_meta_content_scope`.
-                        //
-                        // Multi-context `set:` keeps Clear in the non-initial
-                        // phase (emitted inline after preceding contexts'
-                        // mcs pushes). Moving it to the initial phase here
-                        // would strip atoms from below the outer mcs rather
-                        // than from the top of the just-pushed inner mcs
-                        // stack — Makefile's `set: [value-to-be-defined,
-                        // eat-whitespace-then-pop]` relies on Clear eating
-                        // the last-pushed mcs atom, which Restore then
-                        // replaces when eat-whitespace-then-pop pops.
-                        let single_context_set_clear = is_set && context_refs.len() == 1;
-                        for r in context_refs.iter() {
-                            let ctx = r.resolve(syntax_set)?;
-
-                            let emit_clear_here = !is_set || single_context_set_clear;
-                            if emit_clear_here {
-                                if let Some(clear_amount) = ctx.clear_scopes {
-                                    ops.push((index, ScopeStackOp::Clear(clear_amount)));
-                                }
-                            }
-
-                            for scope in ctx.meta_scope.iter() {
-                                ops.push((index, ScopeStackOp::Push(*scope)));
-                            }
-                        }
-                    } else {
-                        for r in context_refs.iter() {
-                            let ctx = r.resolve(syntax_set)?;
-
-                            if !is_set {
-                                if let Some(clear_amount) = ctx.clear_scopes {
-                                    ops.push((index, ScopeStackOp::Clear(clear_amount)));
-                                }
-                            }
-
-                            for scope in ctx.meta_scope.iter() {
-                                ops.push((index, ScopeStackOp::Push(*scope)));
-                            }
-                        }
-                    }
-                } else {
-                    // `pop: N + set:` (set_pop_count > 1) unwinds N-1 deeper
-                    // contexts in addition to the usual set-replace semantics;
-                    // their meta_scope / meta_content_scope atoms sitting on
-                    // the scope stack must be popped off, so force repush to
-                    // fire even if the immediate contexts had no mcs/ms.
-                    let repush = (is_set
-                        && (set_pop_count > 1
-                            || !cur_context.meta_scope.is_empty()
-                            || !cur_context.meta_content_scope.is_empty()
-                            // cur has clear_scopes but no meta_scope/mcs: we still
-                            // need to Pop the target.meta_scope pushed in initial,
-                            // Restore cur.clear_scopes, and re-push target.meta_scope
-                            // + target.meta_content_scope in the correct order.
-                            || cur_context.clear_scopes.is_some()))
-                        || context_refs.iter().any(|r| {
-                            let ctx = r.resolve(syntax_set).unwrap();
-
-                            !ctx.meta_content_scope.is_empty()
-                                || (ctx.clear_scopes.is_some() && is_set)
-                        });
-                    if repush {
-                        // remove previously pushed meta scopes, so that meta content scopes will be applied in the correct order
-                        let mut num_to_pop: usize = context_refs
-                            .iter()
-                            .map(|r| {
-                                let ctx = r.resolve(syntax_set).unwrap();
-                                ctx.meta_scope.len()
-                            })
-                            .sum();
-
-                        // also pop off the original context's meta scopes
-                        if is_set {
-                            if version >= 2 {
-                                // v2: set excludes parent meta_content_scope from matched text
-                                num_to_pop += cur_context.meta_scope.len();
-                            } else {
-                                num_to_pop += cur_context.meta_content_scope.len()
-                                    + cur_context.meta_scope.len();
-                            }
-                            // For `pop: N + set:` (set_pop_count > 1), also
-                            // unwind the N-1 deeper contexts' scope atoms. They
-                            // were pushed when those contexts entered; without
-                            // this they'd linger on the stack under the new
-                            // target's meta_scope.
-                            if set_pop_count > 1 {
-                                let stack_len = self.stack.len();
-                                for depth in 1..set_pop_count.min(stack_len) {
-                                    let level_idx = stack_len - 1 - depth;
-                                    let ctx =
-                                        syntax_set.get_context(&self.stack[level_idx].context)?;
-                                    num_to_pop +=
-                                        ctx.meta_content_scope.len() + ctx.meta_scope.len();
-                                }
-                            }
-                        }
-
-                        // do all the popping as one operation
-                        if num_to_pop > 0 {
-                            ops.push((index, ScopeStackOp::Pop(num_to_pop)));
-                        }
-
-                        // Restore scopes cleared by the leaving context, now that
-                        // cur.meta_scope and the initial phase's target.meta_scope
-                        // push have been popped off. The restored atoms land below
-                        // the target's upcoming meta_scope / meta_content_scope push.
-                        if is_set && cur_context.clear_scopes.is_some() {
-                            ops.push((index, ScopeStackOp::Restore));
-                        }
-
-                        // now we push meta scope and meta context scope for each context pushed
-                        if version >= 2 {
-                            // v2: For multi-context `set:`, Clear is emitted
-                            // here so it strips the topmost just-pushed mcs
-                            // atom (as Sublime does for multi-context set).
-                            // Single-context `set:` emitted its Clear earlier
-                            // in the initial phase (so the trigger token sees
-                            // the cleared stack); re-emitting here would
-                            // double-push onto clear_stack and cause Pop
-                            // underflow when the context unwinds.
-                            //
-                            // Clear is emitted per-context (not only for the
-                            // topmost) because `clear_scopes` on a non-topmost
-                            // context is a real pattern: Bash's
-                            // `set: [def-function-body, def-function-params,
-                            // def-function-name]` has `clear_scopes: 1` on
-                            // def-function-params (middle). Each Clear is
-                            // placed just before that context's own mcs/ms
-                            // pushes so it strips the previous iteration's
-                            // last-pushed atom, matching Sublime's semantics.
-                            let single_context_set_clear = is_set && context_refs.len() == 1;
-                            let mut prev_embed_scope_replaces = false;
-                            for r in context_refs.iter() {
-                                let ctx = r.resolve(syntax_set)?;
-
-                                if is_set && !single_context_set_clear {
-                                    if let Some(clear_amount) = ctx.clear_scopes {
-                                        ops.push((index, ScopeStackOp::Clear(clear_amount)));
-                                    }
-                                }
-
-                                for scope in ctx.meta_scope.iter() {
-                                    ops.push((index, ScopeStackOp::Push(*scope)));
-                                }
-                                // v2: if the previous context has embed_scope_replaces,
-                                // skip this context's meta_content_scope (the embedded
-                                // syntax's top-level scope is replaced by embed_scope)
-                                if !prev_embed_scope_replaces {
-                                    for scope in ctx.meta_content_scope.iter() {
-                                        ops.push((index, ScopeStackOp::Push(*scope)));
-                                    }
-                                }
-                                prev_embed_scope_replaces = ctx.embed_scope_replaces;
-                            }
-                        } else {
-                            for r in context_refs {
-                                let ctx = r.resolve(syntax_set)?;
-
-                                // for some reason, contrary to my reading of the docs, set does this after the token
-                                if is_set {
-                                    if let Some(clear_amount) = ctx.clear_scopes {
-                                        ops.push((index, ScopeStackOp::Clear(clear_amount)));
-                                    }
-                                }
-
-                                for scope in ctx.meta_scope.iter() {
-                                    ops.push((index, ScopeStackOp::Push(*scope)));
-                                }
-                                for scope in ctx.meta_content_scope.iter() {
-                                    ops.push((index, ScopeStackOp::Push(*scope)));
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            MatchOperation::Embed {
-                ref contexts,
-                pop_count,
-                ..
-            } => {
-                // When pop_count > 0 (pop + embed), use Set semantics to handle
-                // popping the current context's meta scopes before pushing the
-                // embedded contexts' meta scopes.
-                let synthetic = if pop_count > 0 {
-                    MatchOperation::Set {
-                        ctx_refs: contexts.clone(),
-                        pop_count,
-                    }
-                } else {
-                    MatchOperation::Push(contexts.clone())
-                };
-                return self.push_meta_ops(
-                    initial,
-                    index,
-                    cur_context,
-                    &synthetic,
-                    syntax_set,
-                    ops,
-                );
-            }
-            MatchOperation::None | MatchOperation::Fail(_) => (),
-            MatchOperation::Branch {
-                ref alternatives,
-                pop_count,
-                ..
-            } => {
-                // Branch acts like Push for meta ops purposes (or Set when pop_count > 0).
-                // At exec time, Branch is transformed into a synthetic Push/Set before
-                // calling push_meta_ops, so this arm is a safety fallback.
-                let synthetic = if pop_count > 0 {
-                    MatchOperation::Set {
-                        ctx_refs: alternatives.clone(),
-                        pop_count,
-                    }
-                } else {
-                    MatchOperation::Push(alternatives.clone())
-                };
-                return self.push_meta_ops(
-                    initial,
-                    index,
-                    cur_context,
-                    &synthetic,
-                    syntax_set,
-                    ops,
-                );
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Returns true if the stack was changed
-    fn perform_op(
-        &mut self,
-        line: &str,
-        regions: &Region,
-        pat: &MatchPattern,
-        syntax_set: &SyntaxSet,
-    ) -> Result {
-        let (ctx_refs, old_proto_ids, is_embed) = match pat.operation {
-            MatchOperation::Push(ref ctx_refs) => (ctx_refs, None, false),
-            MatchOperation::Embed {
-                ref contexts,
-                pop_count,
-                ..
-            } => {
-                if pop_count > 0 {
-                    for _ in 0..pop_count {
-                        self.stack.pop();
-                    }
-                    self.branch_points
-                        .retain(|bp| bp.stack_depth <= self.stack.len());
-                    self.escape_stack
-                        .retain(|e| e.stack_depth < self.stack.len());
-                }
-                (contexts, None, true)
-            }
-            MatchOperation::Set {
-                ref ctx_refs,
-                pop_count,
-            } => {
-                // a `with_prototype` stays active when the context is `set`
-                // until the context layer in the stack (where the `with_prototype`
-                // was initially applied) is popped off. With `pop: N + set:`
-                // (pop_count > 1), the topmost popped frame's prototypes are
-                // what carry forward onto the new push.
-                let pops = pop_count.max(1);
-                let old_proto_ids = self.stack.pop().map(|s| s.prototypes);
-                for _ in 1..pops {
-                    self.stack.pop();
-                }
-                // Prune branch_points / escape_stack against the *final* stack
-                // length (after the common push loop below), so a branch_point
-                // captured at the pre-set depth survives a pop-1 + push-1 set
-                // — the depth the bp references is still valid after the push.
-                let final_len = self.stack.len() + ctx_refs.len();
-                self.branch_points.retain(|bp| bp.stack_depth <= final_len);
-                self.escape_stack.retain(|e| e.stack_depth < final_len);
-                (ctx_refs, old_proto_ids, false)
-            }
-            MatchOperation::Pop(n) => {
-                for _ in 0..n {
-                    self.stack.pop();
-                }
-                // Invalidate branch points whose stack depth is now above current stack
-                self.branch_points
-                    .retain(|bp| bp.stack_depth <= self.stack.len());
-                // Remove escape entries whose stack_depth >= current stack
-                self.escape_stack
-                    .retain(|e| e.stack_depth < self.stack.len());
-                return Ok(true);
-            }
-            MatchOperation::None => return Ok(false),
-            MatchOperation::Branch { .. } | MatchOperation::Fail(_) => {
-                // Branch and Fail are handled in exec_pattern, not here
-                return Ok(false);
-            }
-        };
-
-        // Record stack depth before pushing (for Embed escape entry)
-        let stack_depth_before = self.stack.len();
-
-        for (i, r) in ctx_refs.iter().enumerate() {
-            let mut proto_ids = if i == 0 {
-                // it is only necessary to preserve the old prototypes
-                // at the first stack frame pushed
-                old_proto_ids.clone().unwrap_or_else(Vec::new)
-            } else {
-                Vec::new()
-            };
-            if i == ctx_refs.len() - 1 {
-                // if a with_prototype was specified, and multiple contexts were pushed,
-                // then the with_prototype applies only to the last context pushed, i.e.
-                // top most on the stack after all the contexts are pushed - this is also
-                // referred to as the "target" of the push by sublimehq - see
-                // https://forum.sublimetext.com/t/dev-build-3111/19240/17 for more info
-                if let Some(ref p) = pat.with_prototype {
-                    proto_ids.push(p.id()?);
-                }
-            }
-            let context_id = r.id()?;
-            let context = syntax_set.get_context(&context_id)?;
-            let captures = {
-                let mut uses_backrefs = context.uses_backrefs;
-                if !proto_ids.is_empty() {
-                    uses_backrefs = uses_backrefs
-                        || proto_ids
-                            .iter()
-                            .any(|id| syntax_set.get_context(id).unwrap().uses_backrefs);
-                }
-                if uses_backrefs {
-                    Some((regions.clone(), line.to_owned()))
-                } else {
-                    None
-                }
-            };
-            self.stack.push(StateLevel {
-                context: context_id,
-                prototypes: proto_ids,
-                captures,
-            });
-        }
-
-        // For Embed: push an EscapeEntry with the resolved escape regex
-        if is_embed {
-            if let MatchOperation::Embed { ref escape, .. } = pat.operation {
-                let resolved_regex = if escape.has_captures {
-                    // Resolve backrefs in escape regex using the triggering match's captures
-                    let new_regex_str =
-                        substitute_backrefs_in_regex(escape.escape_regex.regex_str(), |i| {
-                            regions.pos(i).map(|(s, e)| escape_str(&line[s..e]))
-                        });
-                    Regex::new(new_regex_str)
-                } else {
-                    escape.escape_regex.clone()
-                };
-                self.escape_stack.push(EscapeEntry {
-                    regex: resolved_regex,
-                    captures: escape.escape_captures.clone(),
-                    stack_depth: stack_depth_before,
-                });
-            }
-        }
-
-        Ok(true)
-    }
-
-    /// Execute an escape match: apply escape_captures, pop stack down to
-    /// the embed's stack_depth, and remove the escape entry.
-    fn exec_escape(
-        &mut self,
-        escape_idx: usize,
-        match_start: usize,
-        _match_end: usize,
-        regions: &Region,
-        syntax_set: &SyntaxSet,
-        ops: &mut Vec<(usize, ScopeStackOp)>,
-    ) -> Result<(), ParsingError> {
-        let entry = &self.escape_stack[escape_idx];
-        let target_depth = entry.stack_depth;
-        let escape_captures = entry.captures.clone();
-
-        // Pop all stack levels down to target_depth, emitting proper meta scope pops
-        while self.stack.len() > target_depth {
-            let level = &self.stack[self.stack.len() - 1];
-            let ctx = syntax_set.get_context(&level.context)?;
-
-            // Pop meta_content_scope.  If the context below has
-            // embed_scope_replaces (it's a v2 embed_scope wrapper), the top
-            // context is the embedded syntax's main — whose mcs was never
-            // pushed on the way in — so skip the pop here too.  Gating this
-            // on `current_syntax_version >= 2` would be wrong: the version
-            // is read from the top context (the embedded syntax), but
-            // embed_scope_replaces is set only by the v2 host syntax.  A v2
-            // host embedding a v1 grammar (e.g. Rails HTML embedding Ruby)
-            // would otherwise Pop a scope that was never pushed, misaligning
-            // every scope below until the escape closes.
-            if !ctx.meta_content_scope.is_empty() {
-                let skip = self.stack.len() >= 2
-                    && syntax_set
-                        .get_context(&self.stack[self.stack.len() - 2].context)
-                        .map(|c| c.embed_scope_replaces)
-                        .unwrap_or(false);
-                if !skip {
-                    ops.push((match_start, ScopeStackOp::Pop(ctx.meta_content_scope.len())));
-                }
-            }
-
-            // Pop meta_scope
-            if !ctx.meta_scope.is_empty() {
-                ops.push((match_start, ScopeStackOp::Pop(ctx.meta_scope.len())));
-            }
-
-            // Restore cleared scopes
-            if ctx.clear_scopes.is_some() {
-                ops.push((match_start, ScopeStackOp::Restore));
-            }
-
-            self.stack.pop();
-        }
-
-        // Apply escape_captures scopes
-        if let Some(ref capture_map) = escape_captures {
-            let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new();
-            for &(cap_index, ref scopes) in capture_map.iter() {
-                if let Some((cap_start, cap_end)) = regions.pos(cap_index) {
-                    if cap_start == cap_end {
-                        continue;
-                    }
-                    for scope in scopes.iter() {
-                        map.push((
-                            (cap_start, -((cap_end - cap_start) as i32)),
-                            ScopeStackOp::Push(*scope),
-                        ));
-                    }
-                    map.push(((cap_end, i32::MIN), ScopeStackOp::Pop(scopes.len())));
-                }
-            }
-            map.sort_by(|a, b| a.0.cmp(&b.0));
-            for ((index, _), op) in map.into_iter() {
-                ops.push((index, op));
-            }
-        }
-
-        // Remove this escape entry and any inner (later) escape entries
-        self.escape_stack.truncate(escape_idx);
-
-        // Invalidate branch points whose stack depth is now above current stack
-        self.branch_points
-            .retain(|bp| bp.stack_depth <= self.stack.len());
-
-        Ok(())
-    }
-}
-
-/// Escape a string for use in regex substitution (re-export for use in escape resolution).
-fn escape_str(s: &str) -> String {
-    escape(s)
-}
-
-#[cfg(feature = "yaml-load")]
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::parsing::ScopeStackOp::{Pop, Push};
-    use crate::parsing::{Scope, ScopeStack, SyntaxSet, SyntaxSetBuilder};
-    use crate::util::debug_print_ops;
-    use crate::utils::testdata;
-
-    const TEST_SYNTAX: &str = include_str!("../../testdata/parser_tests.sublime-syntax");
-    #[test]
-    fn can_parse_simple() {
-        let ss = &*testdata::PACKAGES_SYN_SET;
-        let mut state = {
-            let syntax = ss.find_syntax_by_name("Ruby (Rails)").unwrap();
-            ParseState::new(syntax)
-        };
-
-        let ops1 = ops(&mut state, "module Bob::Wow::Troll::Five; 5; end", ss);
-        // `source.ruby.rails` is pushed once — the file's top-level
-        // scope. Earlier versions of `add_initial_contexts` inserted
-        // the scope into `main.meta_content_scope` twice (once on
-        // initial load, once again after `resolve_extends` re-ran),
-        // which showed up here as a duplicate Push. That duplication
-        // also broke assertions of the form `- source source` in the
-        // Diff test fixtures; the initial-contexts fix removes it.
-        let test_ops1 = vec![
-            (0, Push(Scope::new("source.ruby.rails").unwrap())),
-            (0, Push(Scope::new("meta.namespace.ruby").unwrap())),
-            (
-                0,
-                Push(Scope::new("keyword.declaration.namespace.ruby").unwrap()),
-            ),
-            (6, Pop(1)),
-            (7, Pop(1)),
-            (7, Push(Scope::new("meta.namespace.ruby").unwrap())),
-            (7, Push(Scope::new("entity.name.namespace.ruby").unwrap())),
-            (7, Push(Scope::new("support.other.namespace.ruby").unwrap())),
-        ];
-        assert_eq!(&ops1[0..test_ops1.len()], &test_ops1[..]);
-
-        let ops2 = ops(&mut state, "def lol(wow = 5)", ss);
-        let test_ops2 = [
-            (0, Push(Scope::new("meta.function.ruby").unwrap())),
-            (
-                0,
-                Push(Scope::new("keyword.declaration.function.ruby").unwrap()),
-            ),
-            (3, Pop(2)),
-            (3, Push(Scope::new("meta.function.ruby").unwrap())),
-            (4, Push(Scope::new("entity.name.function.ruby").unwrap())),
-            (7, Pop(1)),
-        ];
-        assert_eq!(&ops2[0..test_ops2.len()], &test_ops2[..]);
-    }
-
-    #[test]
-    fn can_parse_yaml() {
-        let ps = &*testdata::PACKAGES_SYN_SET;
-        let mut state = {
-            let syntax = ps.find_syntax_by_name("YAML").unwrap();
-            ParseState::new(syntax)
-        };
-
-        assert_eq!(
-            ops(&mut state, "key: value\n", ps),
-            vec![
-                (0, Push(Scope::new("source.yaml").unwrap())),
-                (0, Push(Scope::new("meta.mapping.key.yaml").unwrap())),
-                (0, Push(Scope::new("meta.string.yaml").unwrap())),
-                (
-                    0,
-                    Push(Scope::new("string.unquoted.plain.out.yaml").unwrap())
-                ),
-                (3, Pop(2)),
-                (3, Pop(1)),
-                (3, Push(Scope::new("meta.mapping.yaml").unwrap())),
-                (
-                    3,
-                    Push(Scope::new("punctuation.separator.key-value.mapping.yaml").unwrap())
-                ),
-                (4, Pop(2)),
-                (5, Push(Scope::new("meta.string.yaml").unwrap())),
-                (
-                    5,
-                    Push(Scope::new("string.unquoted.plain.out.yaml").unwrap())
-                ),
-                (10, Pop(2)),
-            ]
-        );
-    }
-
-    #[test]
-    fn can_parse_includes() {
-        let ss = &*testdata::PACKAGES_SYN_SET;
-        let mut state = {
-            let syntax = ss.find_syntax_by_name("HTML (Rails)").unwrap();
-            ParseState::new(syntax)
-        };
-
-        let ops = ops(&mut state, "