From 27936d0333779cb500486b9efa87d8ae6bf1a4c9 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 11:49:59 +0100 Subject: [PATCH 1/7] feat: support character token sources --- src/sources.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/sources.rs b/src/sources.rs index 71017cb..71fb965 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -20,6 +20,11 @@ pub fn words(data: &str) -> Words<'_> { Words(data) } +/// Returns a [`TokenSource`] that uses the characters in `data` as Tokens +pub fn chars(data: &str) -> impl Iterator + Clone + '_ { + data.chars() +} + /// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline /// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing /// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is @@ -134,6 +139,20 @@ impl<'a> TokenSource for Words<'a> { } } +impl<'a> TokenSource for std::str::Chars<'a> { + type Token = char; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + self.clone() + } + + fn estimate_tokens(&self) -> u32 { + self.as_str().len() as u32 + } +} + /// A [`TokenSource`] that returns the lines of a byte slice as tokens. See [`byte_lines`] /// for details. #[derive(Clone, Copy, PartialEq, Eq)] From bd9d33720765c799386ca40ed54f5bb3077b2747 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 11:49:59 +0100 Subject: [PATCH 2/7] feat: add `Hunk#char_diff` --- src/lib.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5f0860a..bd583e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,11 +138,12 @@ //! ); //! ``` +use std::hash::Hash; use std::ops::Range; use std::slice; use crate::{ - sources::words, + sources::{chars, words}, util::{strip_common_postfix, strip_common_prefix}, }; @@ -400,18 +401,44 @@ impl Hunk { diff_input: &mut InternedInput<&'a str>, diff: &mut Diff, ) { + self.granular_diff(words, input, diff_input, diff) + } + + /// Performs a character-diff of the hunk + pub fn char_diff( + &self, + input: &InternedInput<&str>, + diff_input: &mut InternedInput, + diff: &mut Diff, + ) { + self.granular_diff(chars, input, diff_input, diff) + } + + /// Performs a granular diff of the hunk based on a given tokenizer. For + /// instance, this can be used to compute a word-diff. + pub fn granular_diff<'a, F, I, T>( + &self, + tokenizer: F, + input: &InternedInput<&'a str>, + diff_input: &mut InternedInput, + diff: &mut Diff, + ) where + F: Fn(&'a str) -> I, + I: Iterator, + T: Eq + Hash, + { let Hunk { before, after } = self.clone(); diff_input.update_before( before .map(|index| input.before[index as usize]) .map(|token| input.interner[token]) - .flat_map(|line| words(line)), + .flat_map(|line| tokenizer(line)), ); diff_input.update_after( after .map(|index| input.after[index as usize]) .map(|token| input.interner[token]) - .flat_map(|line| words(line)), + .flat_map(|line| tokenizer(line)), ); diff.removed.clear(); diff.removed.resize(diff_input.before.len(), false); From 0c8ca26f4fefa8b88ccc396799035145682d65fb Mon Sep 17 00:00:00 2001 From: Steffen Trog Date: Tue, 3 Feb 2026 10:12:24 +0100 Subject: [PATCH 3/7] style: fmt --- src/sources.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index 2c8332c..edca29e 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -27,7 +27,7 @@ pub fn words(data: &str) -> Words<'_> { /// Returns a [`TokenSource`] that uses the characters in `data` as Tokens pub fn chars(data: &str) -> impl Iterator + Clone + '_ { - data.chars() + data.chars() } /// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline From 991fd86b068e29d32e84292823c532bcbf45526e Mon Sep 17 00:00:00 2001 From: Steffen Trog Date: Tue, 3 Feb 2026 09:24:29 +0000 Subject: [PATCH 4/7] docs: fix latin_word_diff docs --- src/lib.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4bb0d82..4c1e5c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -417,17 +417,18 @@ impl Hunk { /// Performs a word-diff on this hunk. /// - /// This requires passing the original [`input`](InternedInput) in order to look up - /// the tokens of the current hunk, which typically are lines. - /// Each token is split into words using the built-in [`words`] tokenizer. - /// The resulting word tokens are stored in a second [`diff_input`](InternedInput), - /// and a [`diff`](Diff) is computed on them, with basic post-processing applied. + /// This requires passing the original [`input`](InternedInput) in order to + /// look up the tokens of the current hunk, which typically are lines. Each + /// token is split into words using the built-in [`words`] tokenizer. The + /// resulting word tokens are stored in a second + /// [`word_tokens`](InternedInput), and a [`diff`](Diff) is computed on + /// them, with basic post-processing applied. /// - /// For performance reasons, this second [`diff_input`](InternedInput) as well as - /// the computed [`diff`](Diff) need to be passed as parameters so that they can be - /// re-used when iterating over hunks. Note that word tokens are always - /// added but never removed from the interner. Consider clearing it if you expect - /// your input to have a large vocabulary. + /// For performance reasons, this second [`word_tokens`](InternedInput) as + /// well as the computed [`diff`](Diff) need to be passed as parameters so + /// that they can be re-used when iterating over hunks. Note that word + /// tokens are always added but never removed from the interner. Consider + /// clearing it if you expect your input to have a large vocabulary. /// /// # Examples /// From 08bae1a5f51f21ab23c6d863177680606a5c359a Mon Sep 17 00:00:00 2001 From: Steffen Trog Date: Tue, 3 Feb 2026 09:24:51 +0000 Subject: [PATCH 5/7] fix: make char_diff impl and docs consistent --- src/lib.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4c1e5c3..daa787e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -459,17 +459,55 @@ impl Hunk { word_tokens: &mut InternedInput<&'a str>, diff: &mut Diff, ) { - self.granular_diff(words, input, diff_input, diff) + self.granular_diff(words, input, word_tokens, diff) } /// Performs a character-diff of the hunk + /// + /// This requires passing the original [`input`](InternedInput) in order to + /// look up the tokens of the current hunk, which typically are lines or + /// words. Each token is split into characters using the built-in [`chars`] + /// tokenizer which simply calls `str::chars` directly. The resulting + /// character tokens are stored in a second [`char_tokens`](InternedInput), + /// and a [`diff`](Diff) is computed on them, with basic post-processing + /// applied. + /// + /// For performance reasons, this second [`char_tokens`](InternedInput) as + /// well as the computed [`diff`](Diff) need to be passed as parameters so + /// that they can be re-used when iterating over hunks. Note that character + /// tokens are always added but never removed from the interner. Consider + /// clearing it if you expect your input to have a large alphabet. + /// + /// # Examples + /// + /// ``` + /// # use imara_diff::{InternedInput, Diff, Algorithm}; + /// // Compute diff normally + /// let before = "before text"; + /// let after = "after text"; + /// let mut lines = InternedInput::new(before, after); + /// let mut diff = Diff::compute(Algorithm::Histogram, &lines); + /// diff.postprocess_lines(&lines); + /// + /// // Compute char-diff per hunk, reusing allocations across iterations + /// let mut hunk_diff_input = InternedInput::default(); + /// let mut hunk_diff = Diff::default(); + /// for hunk in diff.hunks() { + /// hunk.char_diff(&lines, &mut hunk_diff_input, &mut hunk_diff); + /// let added = hunk_diff.count_additions(); + /// let removed = hunk_diff.count_removals(); + /// println!("char-diff of this hunk has {added} additions and {removed} removals"); + /// // optionally, clear the interner: + /// hunk_diff_input.clear(); + /// } + /// ``` pub fn char_diff( &self, input: &InternedInput<&str>, - diff_input: &mut InternedInput, + char_tokens: &mut InternedInput, diff: &mut Diff, ) { - self.granular_diff(chars, input, diff_input, diff) + self.granular_diff(chars, input, char_tokens, diff) } /// Performs a granular diff of the hunk based on a given tokenizer. For @@ -486,22 +524,22 @@ impl Hunk { T: Eq + Hash, { let Hunk { before, after } = self.clone(); - word_tokens.update_before( + diff_input.update_before( before .map(|index| input.before[index as usize]) .map(|token| input.interner[token]) .flat_map(|line| tokenizer(line)), ); - word_tokens.update_after( + diff_input.update_after( after .map(|index| input.after[index as usize]) .map(|token| input.interner[token]) .flat_map(|line| tokenizer(line)), ); diff.removed.clear(); - diff.removed.resize(word_tokens.before.len(), false); + diff.removed.resize(diff_input.before.len(), false); diff.added.clear(); - diff.added.resize(word_tokens.after.len(), false); + diff.added.resize(diff_input.after.len(), false); if self.is_pure_removal() { diff.removed.fill(true); } else if self.is_pure_insertion() { @@ -509,11 +547,11 @@ impl Hunk { } else { diff.compute_with( Algorithm::Myers, - &word_tokens.before, - &word_tokens.after, - word_tokens.interner.num_tokens(), + &diff_input.before, + &diff_input.after, + diff_input.interner.num_tokens(), ); - diff.postprocess_no_heuristic(word_tokens); + diff.postprocess_no_heuristic(diff_input); } } } From d3fcadeaf660bbdcc81dfa2796911f86e2f7bbcc Mon Sep 17 00:00:00 2001 From: Steffen Trog Date: Tue, 3 Feb 2026 09:25:35 +0000 Subject: [PATCH 6/7] fix: redundant closure lint --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index daa787e..e1d6abb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -528,13 +528,13 @@ impl Hunk { before .map(|index| input.before[index as usize]) .map(|token| input.interner[token]) - .flat_map(|line| tokenizer(line)), + .flat_map(&tokenizer), ); diff_input.update_after( after .map(|index| input.after[index as usize]) .map(|token| input.interner[token]) - .flat_map(|line| tokenizer(line)), + .flat_map(&tokenizer), ); diff.removed.clear(); diff.removed.resize(diff_input.before.len(), false); From 0e61b784e54beadbe8c712975be3ae9b3d05fc45 Mon Sep 17 00:00:00 2001 From: Steffen Trog Date: Tue, 3 Feb 2026 09:26:50 +0000 Subject: [PATCH 7/7] docs: add missing . character --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e1d6abb..b8d0cff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -462,7 +462,7 @@ impl Hunk { self.granular_diff(words, input, word_tokens, diff) } - /// Performs a character-diff of the hunk + /// Performs a character-diff of the hunk. /// /// This requires passing the original [`input`](InternedInput) in order to /// look up the tokens of the current hunk, which typically are lines or