Extract the parser's pure interpreter state into Core

.gitignore

@@ -17,3 +17,14 @@ target/
 
 # VS Code
 .vscode/
+
+# Generated by examples/synhtml-css-classes
+synhtml-css-classes.html
+synhtml-css-classes.css
+theme-dark.css
+theme-light.css
+
+# cargo-mutants
+mutants.out/
+mutants.out.old/
+master.diff

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "testdata/Packages"]
 	path = testdata/Packages
 	url = https://github.com/sublimehq/Packages
+	branch = v4202
 [submodule "testdata/InspiredGitHub.tmtheme"]
 	path = testdata/InspiredGitHub.tmtheme
 	url = https://github.com/sethlopezme/InspiredGitHub.tmtheme.git

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Breaking changes
 
+- `HighlightLines` and `HighlightFile` deprecated since 6.0.0 — use `syntect::io::HighlightedWriter` instead [#627]
+- `line_tokens_to_classed_spans` (`syntect::html`) deprecated since 6.0.0 — cannot correctly handle cross-line branch-point failures (the rendered output for earlier lines is already in the caller's hands when the parser retroactively replays). Use `ClassedHTMLGenerator` or `HighlightedWriter::from_markup` instead, both of which buffer rendered output during speculative parsing and replay corrected ops [#627]
+- `SCOPE_REPO` removed from public API (was already deprecated since 5.3.0) [#627]
 - `ParseState::parse_line` now returns `Result<ParseLineOutput, ParsingError>` instead of a bare `Vec` [#614]
 - `MatchOperation` enum: new `Branch`, `Fail` [#614], and `Embed` [#615] variants
 - `Pattern` enum: new `IncludeWithPrototype` variant [#610]
@@ -15,12 +18,33 @@
 
 ### Improvements
 
+- Add `syntect::io` module containing `HighlightedWriter<'a, R, W>`, a streaming highlighter that implements `std::io::Write` and handles branch-point backtracking, generic over the output sink (`Vec<u8>` by default, or any `io::Write` for streaming). Constructed via the new `HighlightedWriterBuilder`: pick a renderer category with `HighlightedWriter::from_themed` / `from_markup` / `from_renderer`, then chain `.with_output(...)` / `.with_state(...)` and finish with `.build()`. End-of-input cleanup runs implicitly via `Drop` on a best-effort basis; call `into_inner` explicitly when you need the inner sink back or want errors propagated [#627]
+- Add `syntect::rendering` module with a layered renderer trait design [#627]:
+    - `ScopeMarkup` — slim trait for stateless renderers that map scope structure 1:1 to output structure (e.g. CSS-classed HTML); receives only pre-resolved atom strings
+    - `StyledOutput` — small trait for theme-aware renderers (e.g. ANSI, inline-styled HTML, LaTeX `\textcolor`); only describes how to begin/end a styled span and write text
+    - `ThemedRenderer<O>` — adapter that turns any `StyledOutput` into a `ScopeRenderer` by managing the `Highlighter`, the style stack, and adjacent-token style merging
+    - `ScopeRenderer` — low-level engine trait kept as an escape hatch for advanced cases that need raw `Scope` / `&[Scope]` access
+    - `AnsiStyledOutput` — built-in 24-bit ANSI colour emitter; pair with `HighlightedWriter::from_themed` for terminal output
+- Add `ClassedHTMLScopeRenderer` (`syntect::html`) implementing `ScopeMarkup` for `<span class="...">` output with CSS classes derived from scope atoms [#627]
+- Add `HtmlStyledOutput` (`syntect::html`) implementing `StyledOutput` for theme-aware inline-styled `<span style="...">` HTML output with whitespace-merge optimization [#627]
+- Add `Scope::with_atom_strs()` closure-based API for reading atom strings without direct repository access [#627]
+- `ClassedHTMLGenerator` preserved as backward-compatible wrapper around `HighlightedWriter` (now driven by `MarkupAdapter<ClassedHTMLScopeRenderer>`) [#627]
+- `highlighted_html_for_string`/`highlighted_html_for_file` reworked to use `HighlightedWriter::from_themed` and `HtmlStyledOutput` [#627]
 - Implement sublime-syntax Build 4075 features: `extends` (single and multiple inheritance), `meta_prepend`/`meta_append`, `apply_prototype`, `version: 2`, and `first_line_match` variable resolution [#610] [#612]
 - Implement `branch_point`/`branch`/`fail` backtracking with cross-line replay support [#614]
 - Implement native `embed`/`escape` support, fixing cases where embedded context patterns could consume escape characters first [#615]
 - Gracefully skip syntax files that fail to load and hide syntaxes with unresolved `extends` parents [#610]
 - Add `warnings()` API on `SyntaxSetBuilder` and `SyntaxSet` to replace `eprintln!` warnings with programmatic access [#619]
 
+[#512]: https://github.com/trishume/syntect/pull/512
+[#576]: https://github.com/trishume/syntect/pull/576
+[#610]: https://github.com/trishume/syntect/pull/610
+[#612]: https://github.com/trishume/syntect/pull/612
+[#614]: https://github.com/trishume/syntect/pull/614
+[#615]: https://github.com/trishume/syntect/pull/615
+[#619]: https://github.com/trishume/syntect/pull/619
+[#627]: https://github.com/trishume/syntect/pull/627
+
 ## [Version 5.3.0](https://github.com/trishume/syntect/compare/v5.2.0...v5.3.0)
 
 ### Improvements

benches/highlight_utils/mod.rs

@@ -1,6 +1,8 @@
-use syntect::easy::HighlightLines;
+use std::io::Write;
 use syntect::highlighting::Theme;
+use syntect::io::HighlightedWriter;
 use syntect::parsing::{SyntaxReference, SyntaxSet};
+use syntect::rendering::AnsiStyledOutput;
 
 /// Common helper for benchmarking highlighting.
 pub fn do_highlight(
@@ -9,11 +11,11 @@ pub fn do_highlight(
     syntax: &SyntaxReference,
     theme: &Theme,
 ) -> usize {
-    let mut h = HighlightLines::new(syntax, theme);
-    let mut count = 0;
+    let mut highlight =
+        HighlightedWriter::from_themed(syntax, syntax_set, theme, AnsiStyledOutput::Transparent)
+            .build();
     for line in s.lines() {
-        let regions = h.highlight_line(line, syntax_set).unwrap();
-        count += regions.len();
+        writeln!(highlight, "{}", line).unwrap();
     }
-    count
+    highlight.into_inner().unwrap().len()
 }

examples/latex-demo.rs

@@ -1,22 +1,64 @@
-use syntect::easy::HighlightLines;
+use std::fmt::Write as FmtWrite;
+use std::io::Write as IoWrite;
 use syntect::highlighting::{Style, ThemeSet};
+use syntect::io::HighlightedWriter;
 use syntect::parsing::SyntaxSet;
-use syntect::util::{as_latex_escaped, LinesWithEndings};
+use syntect::rendering::StyledOutput;
+
+/// A [`StyledOutput`] that produces LaTeX `\textcolor[RGB]{r,g,b}{...}`
+/// output. Wrap with [`ThemedRenderer`] (or pass to
+/// [`HighlightedWriter::with_themed`]) to plug it into a highlighter.
+struct LatexStyledOutput;
+
+impl StyledOutput for LatexStyledOutput {
+    fn begin_style(&mut self, style: Style, output: &mut String) {
+        write!(
+            output,
+            "\\textcolor[RGB]{{{},{},{}}}{{",
+            style.foreground.r, style.foreground.g, style.foreground.b
+        )
+        .unwrap();
+    }
+
+    fn end_style(&mut self, output: &mut String) {
+        output.push('}');
+    }
+
+    fn write_text(&mut self, text: &str, output: &mut String) {
+        // The `ThemedRenderer` adapter guarantees `text` never contains
+        // '\n' (line breaks are split out and emitted between styled
+        // spans), so we only need to escape LaTeX's three special
+        // characters.
+        for ch in text.chars() {
+            match ch {
+                '\\' => output.push_str("\\\\"),
+                '{' => output.push_str("\\{"),
+                '}' => output.push_str("\\}"),
+                _ => output.push(ch),
+            }
+        }
+    }
+}
 
 fn main() {
     // Load these once at the start of your program
     let ps = SyntaxSet::load_defaults_newlines();
     let ts = ThemeSet::load_defaults();
 
     let syntax = ps.find_syntax_by_extension("rs").unwrap();
-    let s = "pub struct Wow { hi: u64 }\nfn blah() -> u64 {}\n";
-
-    let mut h = HighlightLines::new(syntax, &ts.themes["InspiredGitHub"]);
-    for line in LinesWithEndings::from(s) {
-        // LinesWithEndings enables use of newlines mode
-        let ranges: Vec<(Style, &str)> = h.highlight_line(line, &ps).unwrap();
-        let escaped = as_latex_escaped(&ranges[..]);
-        println!("\n{:?}", line);
-        println!("\n{}", escaped);
-    }
+    // No explicit `into_inner` needed at the end: we don't need the
+    // `StdoutLock` back, and `Drop` runs the close-scopes / partial-line
+    // cleanup on a best-effort basis when `highlight` falls out of scope.
+    // Examples that *do* need the inner sink back (e.g. `parsyncat` collecting
+    // a `Vec<u8>` per file) call `into_inner` explicitly to propagate errors.
+    let mut highlight = HighlightedWriter::from_themed(
+        syntax,
+        &ps,
+        &ts.themes["InspiredGitHub"],
+        LatexStyledOutput,
+    )
+    .with_output(std::io::stdout().lock())
+    .build();
+    writeln!(highlight, "pub struct Wow {{ hi: u64 }}").unwrap();
+    writeln!(highlight, "fn blah() -> u64 {{}}").unwrap();
 }

examples/parsyncat.rs

@@ -2,12 +2,13 @@
 //! Prints the highlighted output to stdout.
 
 use rayon::prelude::*;
-use syntect::easy::HighlightFile;
-use syntect::highlighting::{Style, ThemeSet};
+use syntect::highlighting::ThemeSet;
+use syntect::io::HighlightedWriter;
 use syntect::parsing::SyntaxSet;
+use syntect::rendering::AnsiStyledOutput;
 
 use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::io::{self, Write};
 
 fn main() {
     let files: Vec<String> = std::env::args().skip(1).collect();
@@ -20,51 +21,33 @@ fn main() {
     let syntax_set = SyntaxSet::load_defaults_newlines();
     let theme_set = ThemeSet::load_defaults();
 
-    // We first collect the contents of the files...
-    let contents: Vec<Vec<String>> = files
+    // Highlight each file in parallel, buffering each result into its own
+    // `Vec<u8>` so we can stream them out in order at the end.
+    let outputs: Vec<Vec<u8>> = files
         .par_iter()
         .map(|filename| {
-            let mut lines = Vec::new();
-            // We use `String::new()` and `read_line()` instead of `BufRead::lines()`
-            // in order to preserve the newlines and get better highlighting.
-            let mut line = String::new();
-            let mut reader = BufReader::new(File::open(filename).unwrap());
-            while reader.read_line(&mut line).unwrap() > 0 {
-                lines.push(line);
-                line = String::new();
-            }
-            lines
-        })
-        .collect();
-
-    // ...so that the highlighted regions have valid lifetimes...
-    let regions: Vec<Vec<(Style, &str)>> = files
-        .par_iter()
-        .zip(&contents)
-        .map(|(filename, contents)| {
-            let mut regions = Vec::new();
             let theme = &theme_set.themes["base16-ocean.dark"];
-            let mut highlighter = HighlightFile::new(filename, &syntax_set, theme).unwrap();
-
-            for line in contents {
-                for region in highlighter
-                    .highlight_lines
-                    .highlight_line(line, &syntax_set)
-                    .unwrap()
-                {
-                    regions.push(region);
-                }
-            }
-
-            regions
+            let syntax = syntax_set
+                .find_syntax_for_file(filename)
+                .unwrap()
+                .unwrap_or_else(|| syntax_set.find_syntax_plain_text());
+            let mut f = File::open(filename).unwrap();
+            let mut w = HighlightedWriter::from_themed(
+                syntax,
+                &syntax_set,
+                theme,
+                AnsiStyledOutput::Transparent,
+            )
+            .build();
+            io::copy(&mut f, &mut w).unwrap();
+            w.into_inner().unwrap()
         })
         .collect();
 
     // ...and then print them all out.
-    for file_regions in regions {
-        print!(
-            "{}",
-            syntect::util::as_24_bit_terminal_escaped(&file_regions[..], true)
-        );
+    let stdout = io::stdout();
+    let mut out = stdout.lock();
+    for output in outputs {
+        out.write_all(&output).unwrap();
     }
 }