From 969dde8053ec12f0ebf3050d2ea19dcc04bd3abf Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 27 Sep 2025 13:41:28 +0200 Subject: [PATCH 01/21] feat: define bytes token source --- src/sources.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/sources.rs b/src/sources.rs index d0506e8..417a825 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -12,6 +12,11 @@ pub fn lines(data: &str) -> Lines<'_> { Lines(ByteLines(data.as_bytes())) } +/// Returns a [`TokenSource`] that uses the bytes in `data` as Tokens. +pub fn bytes(data: &[u8]) -> Bytes<'_> { + Bytes(data) +} + /// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline /// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing /// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is @@ -79,6 +84,37 @@ impl<'a> TokenSource for Lines<'a> { } } +/// A [`TokenSource`] that returns the bytes of a byte slice as tokens. See [`bytes`] +/// for details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Bytes<'a>(&'a [u8]); + +impl<'a> Iterator for Bytes<'a> { + type Item = &'a u8; + + fn next(&mut self) -> Option { + if let Some((head, rem)) = self.0.split_first() { + self.0 = rem; + Some(head) + } else { + None + } + } +} +impl<'a> TokenSource for Bytes<'a> { + type Token = &'a u8; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + 0xFF + } +} + /// A [`TokenSource`] that returns the lines of a byte slice as tokens. See [`byte_lines`] /// for details. #[derive(Clone, Copy, PartialEq, Eq)] From a625abcd1c2bf120b2fcf2e4b4697f75c8a15061 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 27 Sep 2025 13:41:39 +0200 Subject: [PATCH 02/21] feat: implement byte-based word diff --- src/lib.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 2ac3c58..965fb82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,7 +141,10 @@ use std::ops::Range; use std::slice; -use crate::util::{strip_common_postfix, strip_common_prefix}; +use crate::{ + sources::bytes, + util::{strip_common_postfix, strip_common_prefix}, +}; pub use crate::slider_heuristic::{ IndentHeuristic, IndentLevel, NoSliderHeuristic, SliderHeuristic, @@ -389,6 +392,35 @@ impl Hunk { pub fn is_pure_removal(&self) -> bool { self.after.is_empty() } + + /// Performs a byte-diff of the hunk + pub fn byte_diff(&self, diff: &mut Diff, input: &InternedInput<&str>) { + let Hunk { before, after } = self.clone(); + let before_bytes: &[u8] = &before + .flat_map(|token| input.interner[input.before[token as usize]].bytes()) + .collect::>(); + let after_bytes: &[u8] = &after + .flat_map(|token| input.interner[input.after[token as usize]].bytes()) + .collect::>(); + diff.removed.clear(); + diff.removed.resize(before_bytes.len(), false); + diff.added.clear(); + diff.added.resize(after_bytes.len(), false); + if self.is_pure_removal() { + diff.removed.fill(true); + } else if self.is_pure_insertion() { + diff.added.fill(true); + } else { + let input = InternedInput::new(bytes(before_bytes), bytes(after_bytes)); + diff.compute_with( + Algorithm::Myers, + &input.before, + &input.after, + input.interner.num_tokens(), + ); + diff.postprocess_no_heuristic(&input); + } + } } /// Yields all [`Hunk`]s in a file in monotonically increasing order. From 1bf3ccf70eacd6f9e9069fd67eb6aee4b252ed3e Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 27 Sep 2025 13:41:50 +0200 Subject: [PATCH 03/21] test: cover word diffs --- src/tests.rs | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/tests.rs b/src/tests.rs index c19f8c8..4316a4c 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -203,6 +203,120 @@ fn simple_insert() { } } +#[test] +fn hunk_byte_diff_pure() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println("hello world") +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in Algorithm::ALL { + println!("{algorithm:?}"); + let mut d = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.byte_diff(&mut d, &input); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + assert_eq!(first.before, 0..0); + assert_eq!( + first.after, + 0.." println(\"hello world\")\n".len() as u32 + ); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.byte_diff(&mut d, &input); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + assert_eq!( + first.before, + 0.." println(\"hello world\")\n".len() as u32 + ); + assert_eq!(first.after, 0..0); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + swap(&mut input.before, &mut input.after); + } +} + +#[test] +fn hunk_byte_diff_modify() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar{ + let mut foo = 12.0; + foo += 100 / 2; +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in Algorithm::ALL { + println!("{algorithm:?}"); + let mut d = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.byte_diff(&mut d, &input); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + let off = r#" let mut foo = "#.len() as u32; + assert_eq!(first.before, off..off); + assert_eq!(first.after, off..1 + off); + let second = h.next().expect("missing second inner hunk"); + let off = r#" let mut foo = 2.0; + foo "# + .len() as u32; + assert_eq!(second.before, off..1 + off); + assert_eq!(second.after, 1 + off..2 + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.byte_diff(&mut d, &input); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + let off = r#" let mut foo = "#.len() as u32; + assert_eq!(first.before, off..1 + off); + assert_eq!(first.after, off..off); + let second = h.next().expect("missing second inner hunk"); + let off = r#" let mut foo = 2.0; + foo "# + .len() as u32; + assert_eq!(second.before, 1 + off..2 + off); + assert_eq!(second.after, off..1 + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + } +} + pub fn project_root() -> PathBuf { let dir = env!("CARGO_MANIFEST_DIR"); let mut res = PathBuf::from(dir); From 3dba63fec3c24dec8078692263a875fbb0701cdf Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 27 Sep 2025 13:41:57 +0200 Subject: [PATCH 04/21] docs: fix typo --- src/intern.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intern.rs b/src/intern.rs index 54fc1a8..10cfbda 100644 --- a/src/intern.rs +++ b/src/intern.rs @@ -163,7 +163,7 @@ impl Interner { self.tokens.reserve(capacity); } - /// Intern `token` and return a the interned integer. + /// Intern `token` and return the interned integer. pub fn intern(&mut self, token: T) -> Token { let hash = self.hasher.hash_one(&token); match self.table.entry( From 2682999cd3bb8d757ad5dd87827f9b3e89af9103 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Fri, 7 Nov 2025 18:18:55 +0100 Subject: [PATCH 05/21] feat: tokenize words not bytes --- src/lib.rs | 39 +++++++++++++++++++++++++-------------- src/sources.rs | 45 ++++++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 965fb82..ec6dbd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,7 +142,7 @@ use std::ops::Range; use std::slice; use crate::{ - sources::bytes, + sources::words, util::{strip_common_postfix, strip_common_prefix}, }; @@ -393,32 +393,43 @@ impl Hunk { self.after.is_empty() } - /// Performs a byte-diff of the hunk - pub fn byte_diff(&self, diff: &mut Diff, input: &InternedInput<&str>) { + /// Performs a word-diff of the hunk + pub fn word_diff<'a>( + &self, + input: &InternedInput<&'a str>, + diff_input: &mut InternedInput<&'a str>, + diff: &mut Diff, + ) { let Hunk { before, after } = self.clone(); - let before_bytes: &[u8] = &before - .flat_map(|token| input.interner[input.before[token as usize]].bytes()) + let before_words = before + .map(|index| input.before[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)) .collect::>(); - let after_bytes: &[u8] = &after - .flat_map(|token| input.interner[input.after[token as usize]].bytes()) + let after_words = after + .map(|index| input.after[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)) .collect::>(); + diff_input.update_before(before_words.into_iter()); + diff_input.update_after(after_words.into_iter()); + diff.removed.clear(); - diff.removed.resize(before_bytes.len(), false); + diff.removed.resize(diff_input.before.len(), false); diff.added.clear(); - diff.added.resize(after_bytes.len(), false); + diff.added.resize(diff_input.after.len(), false); if self.is_pure_removal() { diff.removed.fill(true); } else if self.is_pure_insertion() { diff.added.fill(true); } else { - let input = InternedInput::new(bytes(before_bytes), bytes(after_bytes)); diff.compute_with( Algorithm::Myers, - &input.before, - &input.after, - input.interner.num_tokens(), + &diff_input.before, + &diff_input.after, + diff_input.interner.num_tokens(), ); - diff.postprocess_no_heuristic(&input); + diff.postprocess_no_heuristic(&diff_input); } } } diff --git a/src/sources.rs b/src/sources.rs index 417a825..040c8bf 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -12,9 +12,11 @@ pub fn lines(data: &str) -> Lines<'_> { Lines(ByteLines(data.as_bytes())) } -/// Returns a [`TokenSource`] that uses the bytes in `data` as Tokens. -pub fn bytes(data: &[u8]) -> Bytes<'_> { - Bytes(data) +/// Returns a [`TokenSource`] that uses the words in `data` as Tokens. A word is +/// a sequence of alphanumeric characters as determined by +/// `char::is_alphanumeric`. Any other characters are their own word. +pub fn words(data: &str) -> Words<'_> { + Words(data) } /// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline @@ -84,25 +86,38 @@ impl<'a> TokenSource for Lines<'a> { } } -/// A [`TokenSource`] that returns the bytes of a byte slice as tokens. See [`bytes`] -/// for details. +/// A [`TokenSource`] that returns the words of a string as tokens. See +/// [`bytes`] for details. #[derive(Clone, Copy, PartialEq, Eq)] -pub struct Bytes<'a>(&'a [u8]); +pub struct Words<'a>(&'a str); -impl<'a> Iterator for Bytes<'a> { - type Item = &'a u8; +impl<'a> Iterator for Words<'a> { + type Item = &'a str; fn next(&mut self) -> Option { - if let Some((head, rem)) = self.0.split_first() { + if self.0.is_empty() { + return None; + } + + let initial = self.0.chars().next().unwrap(); + if !initial.is_alphanumeric() { + let (char, rem) = self.0.split_at(initial.len_utf8()); self.0 = rem; - Some(head) - } else { - None + return Some(char); } + + let end_index = self + .0 + .char_indices() + .find(|(_, c)| !c.is_alphanumeric()) + .map_or(self.0.len(), |(index, _)| index); + let (word, rem) = self.0.split_at(end_index); + self.0 = rem; + Some(word) } } -impl<'a> TokenSource for Bytes<'a> { - type Token = &'a u8; +impl<'a> TokenSource for Words<'a> { + type Token = &'a str; type Tokenizer = Self; @@ -111,7 +126,7 @@ impl<'a> TokenSource for Bytes<'a> { } fn estimate_tokens(&self) -> u32 { - 0xFF + (self.0.len() / 3) as u32 } } From a175626acfeed9fe66d35efe02d4522ac6981d5f Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Fri, 7 Nov 2025 18:19:14 +0100 Subject: [PATCH 06/21] test: cover word diffs --- src/tests.rs | 66 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/src/tests.rs b/src/tests.rs index 4316a4c..1c4cad2 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -7,6 +7,7 @@ use expect_test::{expect, expect_file}; // use git_repository as git; use crate::intern::InternedInput; +use crate::sources::words; use crate::unified_diff::BasicLineDiffPrinter; use crate::{Algorithm, Diff, UnifiedDiffConfig}; @@ -204,7 +205,7 @@ fn simple_insert() { } #[test] -fn hunk_byte_diff_pure() { +fn hunk_word_diff_pure() { let before = r#"fn foo() -> Bar{ let mut foo = 2.0; foo *= 100 / 2; @@ -216,21 +217,24 @@ fn hunk_byte_diff_pure() { }"#; let mut input = InternedInput::new(before, after); for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); + let mut diff_input = InternedInput::default(); let mut d = Diff::default(); + println!("{algorithm:?}"); + let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.byte_diff(&mut d, &input); + hunk.word_diff(&input, &mut diff_input, &mut d); let mut h = d.hunks(); let first = h.next().expect("missing first inner hunk"); assert!(first.is_pure_insertion()); assert_eq!(first.before, 0..0); assert_eq!( first.after, - 0.." println(\"hello world\")\n".len() as u32 + 0..words(" println(\"hello world\")\n").count() as u32 ); assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); @@ -239,55 +243,63 @@ fn hunk_byte_diff_pure() { let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.byte_diff(&mut d, &input); + hunk.word_diff(&input, &mut diff_input, &mut d); let mut h = d.hunks(); let first = h.next().expect("missing first inner hunk"); assert!(first.is_pure_removal()); assert_eq!( first.before, - 0.." println(\"hello world\")\n".len() as u32 + 0..words(" println(\"hello world\")\n").count() as u32 ); assert_eq!(first.after, 0..0); assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); + swap(&mut input.before, &mut input.after); } } #[test] -fn hunk_byte_diff_modify() { - let before = r#"fn foo() -> Bar{ +fn hunk_word_diff_modify() { + let before = r#"fn foo() -> Bar { let mut foo = 2.0; foo *= 100 / 2; }"#; - let after = r#"fn foo() -> Bar{ - let mut foo = 12.0; + let after = r#"fn foo() -> Bar { + let mut foo = 3.0 * 2.0; foo += 100 / 2; }"#; let mut input = InternedInput::new(before, after); for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); + let mut diff_input = InternedInput::default(); let mut d = Diff::default(); + println!("{algorithm:?}"); + let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.byte_diff(&mut d, &input); + hunk.word_diff(&input, &mut diff_input, &mut d); let mut h = d.hunks(); let first = h.next().expect("missing first inner hunk"); assert!(first.is_pure_insertion()); - let off = r#" let mut foo = "#.len() as u32; + let off = words(" let mut foo = ").count() as u32; assert_eq!(first.before, off..off); - assert_eq!(first.after, off..1 + off); + let ins = words("3.0 * ").count() as u32; + assert_eq!(first.after, off..ins + off); let second = h.next().expect("missing second inner hunk"); - let off = r#" let mut foo = 2.0; - foo "# - .len() as u32; + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; assert_eq!(second.before, off..1 + off); - assert_eq!(second.after, 1 + off..2 + off); + assert_eq!(second.after, ins + off..1 + ins + off); assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); @@ -295,20 +307,24 @@ fn hunk_byte_diff_modify() { let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.byte_diff(&mut d, &input); + hunk.word_diff(&input, &mut diff_input, &mut d); let mut h = d.hunks(); let first = h.next().expect("missing first inner hunk"); assert!(first.is_pure_removal()); - let off = r#" let mut foo = "#.len() as u32; - assert_eq!(first.before, off..1 + off); + let off = words(" let mut foo = ").count() as u32; + let rem = words("3.0 * ").count() as u32; + assert_eq!(first.before, off..rem + off); assert_eq!(first.after, off..off); let second = h.next().expect("missing second inner hunk"); - let off = r#" let mut foo = 2.0; - foo "# - .len() as u32; - assert_eq!(second.before, 1 + off..2 + off); + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; + assert_eq!(second.before, rem + off..1 + rem + off); assert_eq!(second.after, off..1 + off); assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); From ae627f554ab684913550cd54b50eec91fad46b46 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Fri, 7 Nov 2025 18:23:24 +0100 Subject: [PATCH 07/21] docs: fix oversight --- src/sources.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index 040c8bf..9449428 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -87,7 +87,7 @@ impl<'a> TokenSource for Lines<'a> { } /// A [`TokenSource`] that returns the words of a string as tokens. See -/// [`bytes`] for details. +/// [`words`] for details. #[derive(Clone, Copy, PartialEq, Eq)] pub struct Words<'a>(&'a str); From acdad54da239b9247949b1351dc07b5cc477434c Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Fri, 7 Nov 2025 18:28:36 +0100 Subject: [PATCH 08/21] fix: lint about needless `&` --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ec6dbd4..1056164 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -429,7 +429,7 @@ impl Hunk { &diff_input.after, diff_input.interner.num_tokens(), ); - diff.postprocess_no_heuristic(&diff_input); + diff.postprocess_no_heuristic(diff_input); } } } From 337bbdbf80e1b134b5cd623cd3151baf6feab780 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 00:06:02 +0100 Subject: [PATCH 09/21] perf: no need to copy word vec --- src/lib.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1056164..5f0860a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -401,19 +401,18 @@ impl Hunk { diff: &mut Diff, ) { let Hunk { before, after } = self.clone(); - let before_words = before - .map(|index| input.before[index as usize]) - .map(|token| input.interner[token]) - .flat_map(|line| words(line)) - .collect::>(); - let after_words = after - .map(|index| input.after[index as usize]) - .map(|token| input.interner[token]) - .flat_map(|line| words(line)) - .collect::>(); - diff_input.update_before(before_words.into_iter()); - diff_input.update_after(after_words.into_iter()); - + diff_input.update_before( + before + .map(|index| input.before[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); + diff_input.update_after( + after + .map(|index| input.after[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); diff.removed.clear(); diff.removed.resize(diff_input.before.len(), false); diff.added.clear(); From 236ae4222947f8ed510cc60779df5e11c8e53b19 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 00:46:08 +0100 Subject: [PATCH 10/21] refactor: only branch on word length --- src/sources.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/sources.rs b/src/sources.rs index 9449428..e9392fc 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -100,18 +100,16 @@ impl<'a> Iterator for Words<'a> { } let initial = self.0.chars().next().unwrap(); - if !initial.is_alphanumeric() { - let (char, rem) = self.0.split_at(initial.len_utf8()); - self.0 = rem; - return Some(char); - } + let word_len = if initial.is_alphanumeric() { + self.0 + .char_indices() + .find(|(_, c)| !c.is_alphanumeric()) + .map_or(self.0.len(), |(index, _)| index) + } else { + initial.len_utf8() + }; - let end_index = self - .0 - .char_indices() - .find(|(_, c)| !c.is_alphanumeric()) - .map_or(self.0.len(), |(index, _)| index); - let (word, rem) = self.0.split_at(end_index); + let (word, rem) = self.0.split_at(word_len); self.0 = rem; Some(word) } From 4c149116630374ef7e5c72acdbc321c7e3ecee73 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 01:00:37 +0100 Subject: [PATCH 11/21] perf: merge contiguous blocks of space chars --- src/sources.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index e9392fc..aeff0e6 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -100,7 +100,12 @@ impl<'a> Iterator for Words<'a> { } let initial = self.0.chars().next().unwrap(); - let word_len = if initial.is_alphanumeric() { + let word_len = if initial == ' ' { + self.0 + .char_indices() + .find(|(_, c)| *c != ' ') + .map_or(self.0.len(), |(index, _)| index) + } else if initial.is_alphanumeric() { self.0 .char_indices() .find(|(_, c)| !c.is_alphanumeric()) From 83fe6bd89e7f097b747bf0e67261dfc399510be8 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 10:20:45 +0100 Subject: [PATCH 12/21] docs: improve `words` docs w.r.t. space handling --- src/sources.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index aeff0e6..71017cb 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -14,7 +14,8 @@ pub fn lines(data: &str) -> Lines<'_> { /// Returns a [`TokenSource`] that uses the words in `data` as Tokens. A word is /// a sequence of alphanumeric characters as determined by -/// `char::is_alphanumeric`. Any other characters are their own word. +/// `char::is_alphanumeric`, or a sequence of just the space character ' '. Any +/// other characters are their own word. pub fn words(data: &str) -> Words<'_> { Words(data) } From b24e52c325dab920cccca2ee1428a0ca31cf4bcc Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 8 Nov 2025 10:20:52 +0100 Subject: [PATCH 13/21] test: cover words tokenizer --- src/tests.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tests.rs b/src/tests.rs index a736c62..eb8cbd1 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -11,6 +11,16 @@ use crate::sources::words; use crate::unified_diff::BasicLineDiffPrinter; use crate::{Algorithm, Diff, UnifiedDiffConfig}; +#[test] +fn words_tokenizer() { + let text = "Hello, imara!\n (foo-bar)"; + let tokens = words(text).collect::>(); + assert_eq!( + tokens, + vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar", ")"] + ); +} + #[test] fn postprocess() { let before = r#" @@ -450,6 +460,27 @@ fn hunk_word_diff_modify() { } } +#[test] +fn large_file() { + println!("reading files"); + let before = std::fs::read_to_string("/tmp/before.html").expect("bad file read"); + let after = std::fs::read_to_string("/tmp/after.html").expect("bad file read"); + println!("interning"); + let input = InternedInput::new(before.as_str(), after.as_str()); + println!("initial diff"); + let diff = Diff::compute(Algorithm::Myers, &input); + + let mut word_input = InternedInput::default(); + let mut word_diff = Diff::default(); + for (i, hunk) in diff.hunks().enumerate() { + println!("+++ Hunk {i}"); + hunk.word_diff(&input, &mut word_input, &mut word_diff); + println!("word diff count {}", word_diff.hunks().count()); + println!("--- Hunk {i}"); + } + println!("done"); +} + pub fn project_root() -> PathBuf { let dir = env!("CARGO_MANIFEST_DIR"); let mut res = PathBuf::from(dir); From 92900668abe1c4728cb5b143ac8d8d173de09ea5 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Fri, 21 Nov 2025 23:04:07 +0100 Subject: [PATCH 14/21] fix: consider `_` as part of a word Co-authored-by: Pascal Kuthe --- src/sources.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index 71017cb..16f38a5 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -109,7 +109,7 @@ impl<'a> Iterator for Words<'a> { } else if initial.is_alphanumeric() { self.0 .char_indices() - .find(|(_, c)| !c.is_alphanumeric()) + .find(|(_, c)| !c.is_alphanumeric() && c != '_') .map_or(self.0.len(), |(index, _)| index) } else { initial.len_utf8() From 30d2f41f6ecd09ad103746c030d1457259e4a1d4 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 22 Nov 2025 09:05:21 +0100 Subject: [PATCH 15/21] fix: add missing deref --- src/sources.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources.rs b/src/sources.rs index 16f38a5..b4fd1d8 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -109,7 +109,7 @@ impl<'a> Iterator for Words<'a> { } else if initial.is_alphanumeric() { self.0 .char_indices() - .find(|(_, c)| !c.is_alphanumeric() && c != '_') + .find(|(_, c)| !c.is_alphanumeric() && *c != '_') .map_or(self.0.len(), |(index, _)| index) } else { initial.len_utf8() From 936bbcf95ea0305b7ef30580c7ce8a8f84b32910 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 22 Nov 2025 09:45:25 +0100 Subject: [PATCH 16/21] test: cover _ tokenization --- src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests.rs b/src/tests.rs index eb8cbd1..a42c185 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -13,11 +13,11 @@ use crate::{Algorithm, Diff, UnifiedDiffConfig}; #[test] fn words_tokenizer() { - let text = "Hello, imara!\n (foo-bar)"; + let text = "Hello, imara!\n (foo-bar_baz)"; let tokens = words(text).collect::>(); assert_eq!( tokens, - vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar", ")"] + vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar_baz", ")"] ); } From 0194b5042367f7af691ba69b30ac05b0851c840c Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 22 Nov 2025 09:45:31 +0100 Subject: [PATCH 17/21] test: remove lost test case --- src/tests.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/tests.rs b/src/tests.rs index a42c185..b11cde2 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -460,27 +460,6 @@ fn hunk_word_diff_modify() { } } -#[test] -fn large_file() { - println!("reading files"); - let before = std::fs::read_to_string("/tmp/before.html").expect("bad file read"); - let after = std::fs::read_to_string("/tmp/after.html").expect("bad file read"); - println!("interning"); - let input = InternedInput::new(before.as_str(), after.as_str()); - println!("initial diff"); - let diff = Diff::compute(Algorithm::Myers, &input); - - let mut word_input = InternedInput::default(); - let mut word_diff = Diff::default(); - for (i, hunk) in diff.hunks().enumerate() { - println!("+++ Hunk {i}"); - hunk.word_diff(&input, &mut word_input, &mut word_diff); - println!("word diff count {}", word_diff.hunks().count()); - println!("--- Hunk {i}"); - } - println!("done"); -} - pub fn project_root() -> PathBuf { let dir = env!("CARGO_MANIFEST_DIR"); let mut res = PathBuf::from(dir); From cd204974a32d0deede7ab926952db360b15e00a2 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 20 Dec 2025 23:26:06 +0100 Subject: [PATCH 18/21] docs: elaborate on `word_diff` usage --- src/lib.rs | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index a408cb0..1ccc346 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -420,7 +420,42 @@ impl Hunk { self.after.is_empty() } - /// Performs a word-diff of the hunk + /// Performs a word-diff of the hunk. + /// + /// This requires passing the original [`InternedInput`] in order to look up + /// the tokens of the current hunk. Each token is split into words using the + /// built-in [`words`] tokenizer. The resulting word tokens are stored in a + /// second [`InternedInput`], and a [`Diff`] is computed on them. + /// + /// For performance reasons, this second [`InternedInput`] as well as the + /// computed [`Diff`] need to be passed as parameters so that they can be + /// re-used when iterating over hunks. Note that word tokens are always + /// added but never removed from the interner. Consider clearing it every + /// `n` iterations if you expect your input to have a large vocabulary. + /// + /// # Examples + /// + /// ``` + /// # use imara_diff::{InternedInput, Diff, Algorithm}; + /// // Compute diff normally + /// let before = "before text"; + /// let after = "after text"; + /// let mut input = InternedInput::new(before, after); + /// let mut diff = Diff::compute(Algorithm::Histogram, &input); + /// diff.postprocess_lines(&input); + /// + /// // Compute word-diff per hunk, reusing allocations across iterations + /// let mut hunk_diff_input = InternedInput::default(); + /// let mut hunk_diff = Diff::default(); + /// for hunk in diff.hunks() { + /// hunk.word_diff(&input, &mut hunk_diff_input, &mut hunk_diff); + /// let added = hunk_diff.count_additions(); + /// let removed = hunk_diff.count_removals(); + /// println!("word-diff of this hunk has {added} additions and {removed} removals"); + /// // optionally, clear the interner: + /// hunk_diff_input.clear(); + /// } + /// ``` pub fn word_diff<'a>( &self, input: &InternedInput<&'a str>, From d1937720898a70eab41328a021bef189335886be Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Dec 2025 08:24:10 +0100 Subject: [PATCH 19/21] refactor - rename `word_diff` to `latin_word_diff` --- src/lib.rs | 55 +++++++------- src/tests.rs | 200 ++++++++++++++++++++++++++------------------------- 2 files changed, 129 insertions(+), 126 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1ccc346..f4ae517 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -421,34 +421,35 @@ impl Hunk { } /// Performs a word-diff of the hunk. - /// - /// This requires passing the original [`InternedInput`] in order to look up - /// the tokens of the current hunk. Each token is split into words using the - /// built-in [`words`] tokenizer. The resulting word tokens are stored in a - /// second [`InternedInput`], and a [`Diff`] is computed on them. - /// - /// For performance reasons, this second [`InternedInput`] as well as the - /// computed [`Diff`] need to be passed as parameters so that they can be + /// + /// This requires passing the original [`input`](InternedInput) in order to look up + /// the tokens of the current hunk, which typically are lines. + /// Each token is split into words using the built-in [`words`] tokenizer. + /// The resulting word tokens are stored in a second [`diff_input`](InternedInput), + /// and a [`diff`](Diff) is computed on them, with basic post-processing applied. + /// + /// For performance reasons, this second [`diff_input`](InternedInput) as well as + /// the computed [`diff`](Diff) need to be passed as parameters so that they can be /// re-used when iterating over hunks. Note that word tokens are always - /// added but never removed from the interner. Consider clearing it every - /// `n` iterations if you expect your input to have a large vocabulary. - /// + /// added but never removed from the interner. Consider clearing it if you expect + /// your input to have a large vocabulary. + /// /// # Examples - /// + /// /// ``` /// # use imara_diff::{InternedInput, Diff, Algorithm}; /// // Compute diff normally /// let before = "before text"; /// let after = "after text"; - /// let mut input = InternedInput::new(before, after); - /// let mut diff = Diff::compute(Algorithm::Histogram, &input); - /// diff.postprocess_lines(&input); - /// + /// let mut lines = InternedInput::new(before, after); + /// let mut diff = Diff::compute(Algorithm::Histogram, &lines); + /// diff.postprocess_lines(&lines); + /// /// // Compute word-diff per hunk, reusing allocations across iterations /// let mut hunk_diff_input = InternedInput::default(); /// let mut hunk_diff = Diff::default(); /// for hunk in diff.hunks() { - /// hunk.word_diff(&input, &mut hunk_diff_input, &mut hunk_diff); + /// hunk.latin_word_diff(&lines, &mut hunk_diff_input, &mut hunk_diff); /// let added = hunk_diff.count_additions(); /// let removed = hunk_diff.count_removals(); /// println!("word-diff of this hunk has {added} additions and {removed} removals"); @@ -456,29 +457,29 @@ impl Hunk { /// hunk_diff_input.clear(); /// } /// ``` - pub fn word_diff<'a>( + pub fn latin_word_diff<'a>( &self, input: &InternedInput<&'a str>, - diff_input: &mut InternedInput<&'a str>, + word_tokens: &mut InternedInput<&'a str>, diff: &mut Diff, ) { let Hunk { before, after } = self.clone(); - diff_input.update_before( + word_tokens.update_before( before .map(|index| input.before[index as usize]) .map(|token| input.interner[token]) .flat_map(|line| words(line)), ); - diff_input.update_after( + word_tokens.update_after( after .map(|index| input.after[index as usize]) .map(|token| input.interner[token]) .flat_map(|line| words(line)), ); diff.removed.clear(); - diff.removed.resize(diff_input.before.len(), false); + diff.removed.resize(word_tokens.before.len(), false); diff.added.clear(); - diff.added.resize(diff_input.after.len(), false); + diff.added.resize(word_tokens.after.len(), false); if self.is_pure_removal() { diff.removed.fill(true); } else if self.is_pure_insertion() { @@ -486,11 +487,11 @@ impl Hunk { } else { diff.compute_with( Algorithm::Myers, - &diff_input.before, - &diff_input.after, - diff_input.interner.num_tokens(), + &word_tokens.before, + &word_tokens.after, + word_tokens.interner.num_tokens(), ); - diff.postprocess_no_heuristic(diff_input); + diff.postprocess_no_heuristic(word_tokens); } } } diff --git a/src/tests.rs b/src/tests.rs index 3c8844f..5083dea 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -331,132 +331,134 @@ i } } -#[test] -fn hunk_word_diff_pure() { - let before = r#"fn foo() -> Bar{ +mod latin_word_diff { + use crate::sources::words; + use crate::{Algorithm, Diff, InternedInput}; + use std::mem::swap; + + #[test] + fn pure() { + let before = r#"fn foo() -> Bar{ let mut foo = 2.0; foo *= 100 / 2; }"#; - let after = r#"fn foo() -> Bar{ + let after = r#"fn foo() -> Bar{ let mut foo = 2.0; foo *= 100 / 2; println("hello world") }"#; - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); - - println!("{algorithm:?}"); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_insertion()); - assert_eq!(first.before, 0..0); - assert_eq!( - first.after, - 0..words(" println(\"hello world\")\n").count() as u32 - ); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); + let mut input = InternedInput::new(before, after); + for algorithm in Algorithm::ALL { + let mut diff_input = InternedInput::default(); + let mut d = Diff::default(); - swap(&mut input.before, &mut input.after); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + assert_eq!(first.before, 0..0); + assert_eq!( + first.after, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_removal()); - assert_eq!( - first.before, - 0..words(" println(\"hello world\")\n").count() as u32 - ); - assert_eq!(first.after, 0..0); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); - swap(&mut input.before, &mut input.after); + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + assert_eq!( + first.before, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(first.after, 0..0); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + } } -} -#[test] -fn hunk_word_diff_modify() { - let before = r#"fn foo() -> Bar { + #[test] + fn modify() { + let before = r#"fn foo() -> Bar { let mut foo = 2.0; foo *= 100 / 2; }"#; - let after = r#"fn foo() -> Bar { + let after = r#"fn foo() -> Bar { let mut foo = 3.0 * 2.0; foo += 100 / 2; }"#; - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); + let mut input = InternedInput::new(before, after); + for algorithm in Algorithm::ALL { + let mut diff_input = InternedInput::default(); + let mut d = Diff::default(); - println!("{algorithm:?}"); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_insertion()); - let off = words(" let mut foo = ").count() as u32; - assert_eq!(first.before, off..off); - let ins = words("3.0 * ").count() as u32; - assert_eq!(first.after, off..ins + off); - let second = h.next().expect("missing second inner hunk"); - let off = words( - r#" let mut foo = 2.0; + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + let off = words(" let mut foo = ").count() as u32; + assert_eq!(first.before, off..off); + let ins = words("3.0 * ").count() as u32; + assert_eq!(first.after, off..ins + off); + let second = h.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; foo "#, - ) - .count() as u32; - assert_eq!(second.before, off..1 + off); - assert_eq!(second.after, ins + off..1 + ins + off); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); + ) + .count() as u32; + assert_eq!(second.before, off..1 + off); + assert_eq!(second.after, ins + off..1 + ins + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); - swap(&mut input.before, &mut input.after); + swap(&mut input.before, &mut input.after); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_removal()); - let off = words(" let mut foo = ").count() as u32; - let rem = words("3.0 * ").count() as u32; - assert_eq!(first.before, off..rem + off); - assert_eq!(first.after, off..off); - let second = h.next().expect("missing second inner hunk"); - let off = words( - r#" let mut foo = 2.0; + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + let off = words(" let mut foo = ").count() as u32; + let rem = words("3.0 * ").count() as u32; + assert_eq!(first.before, off..rem + off); + assert_eq!(first.after, off..off); + let second = h.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; foo "#, - ) - .count() as u32; - assert_eq!(second.before, rem + off..1 + rem + off); - assert_eq!(second.after, off..1 + off); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); + ) + .count() as u32; + assert_eq!(second.before, rem + off..1 + rem + off); + assert_eq!(second.after, off..1 + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); - swap(&mut input.before, &mut input.after); + swap(&mut input.before, &mut input.after); + } } } From 77249093da2e13dd3f3e377f649e16c769516cc5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Dec 2025 08:41:02 +0100 Subject: [PATCH 20/21] refactor tests Move all tests that don't validate private state are on integration level. Reorganise fixtures while at it. --- Cargo.lock | 12 +- src/lib.rs | 10 +- src/tests.rs | 611 ++------------------------------------ tests/integration/main.rs | 457 ++++++++++++++++++++++++++++ 4 files changed, 490 insertions(+), 600 deletions(-) create mode 100644 tests/integration/main.rs diff --git a/Cargo.lock b/Cargo.lock index 59140cd..1829ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "cov-mark" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1d92727879fb4f24cec33a35e3bff74035541326cbc12ad44ba8886d1927b0" +checksum = "90863d8442510cddf7f46618c4f92413774635771a3e80830c8b30d183420b14" [[package]] name = "dissimilar" @@ -32,9 +32,9 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "once_cell" diff --git a/src/lib.rs b/src/lib.rs index f4ae517..9150d31 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -163,13 +163,12 @@ mod myers; mod postprocess; mod slider_heuristic; pub mod sources; +#[cfg(test)] +mod tests; #[cfg(feature = "unified_diff")] mod unified_diff; mod util; -#[cfg(test)] -mod tests; - /// `imara-diff` supports multiple different algorithms /// for computing an edit sequence. /// These algorithms have different performance and all produce different output. @@ -228,11 +227,6 @@ pub enum Algorithm { MyersMinimal, } -impl Algorithm { - #[cfg(test)] - const ALL: [Self; 2] = [Algorithm::Histogram, Algorithm::Myers]; -} - /// Represents the difference between two sequences of tokens. /// /// A `Diff` stores which tokens were removed from the first sequence and which tokens were added to the second sequence. diff --git a/src/tests.rs b/src/tests.rs index 5083dea..a0e9355 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,165 +1,6 @@ -use std::fs::read_to_string; -use std::mem::swap; -use std::path::PathBuf; +use crate::{Algorithm, BasicLineDiffPrinter, Diff, InternedInput, UnifiedDiffConfig}; +use expect_test::expect; -use expect_test::{expect, expect_file}; -// use git::bstr::BStr; -// use git_repository as git; - -use crate::intern::InternedInput; -use crate::sources::words; -use crate::unified_diff::BasicLineDiffPrinter; -use crate::{Algorithm, Diff, UnifiedDiffConfig}; - -#[test] -fn words_tokenizer() { - let text = "Hello, imara!\n (foo-bar_baz)"; - let tokens = words(text).collect::>(); - assert_eq!( - tokens, - vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar_baz", ")"] - ); -} - -#[test] -fn postprocess() { - let before = r#" - /* - * Stay on the safe side. if read_directory() has run once on - * "dir", some sticky flag may have been left. Clear them all. - */ - clear_sticky(dir); - - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - * subset of positive ones, which has no impacts on - * foo - * bar - * test - */ - foo - "#; - let after = r#" - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - * subset of positive ones, which has no impacts on - * foo - * bar - * test - */ - foo - "#; - - let input = InternedInput::new(before, after); - for algorithm in [Algorithm::Histogram, Algorithm::Myers] { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - let diff = diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(); - println!("{diff:?}"); - expect![[r#" - @@ -1,10 +1,4 @@ - - - /* - - * Stay on the safe side. if read_directory() has run once on - - * "dir", some sticky flag may have been left. Clear them all. - - */ - - clear_sticky(dir); - - - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - "#]] - .assert_eq(&diff); - } -} - -#[test] -fn replace() { - let before = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println!("hello world") -} -"#; - - let after = r#"const TEST: i32 = 0; -fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println!("hello world"); - println!("hello foo {TEST}"); -} - -"#; - let input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,5 +1,8 @@ - +const TEST: i32 = 0; - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - - println!("hello world") - + println!("hello world"); - + println!("hello foo {TEST}"); - } - + - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -#[test] -fn myers_is_odd() { - let before = "a\nb\nx\ny\nx\n"; - let after = "b\na\nx\ny\n"; - - cov_mark::check!(ODD_SPLIT); - // if the check for odd doesn't work then - // we still find the correct result but the number of search - // iterations increases - cov_mark::check_count!(SPLIT_SEARCH_ITER, 9); - let input = InternedInput::new(before, after); - let diff = Diff::compute(Algorithm::Myers, &input); - expect![[r#" - @@ -1,5 +1,4 @@ - -a - b - +a - x - y - -x - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); -} #[test] fn myers_is_even() { let before = "a\nb\nx\nx\ny\n"; @@ -195,435 +36,33 @@ fn myers_is_even() { } #[test] -fn identical_files() { - let file = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; -}"#; +fn myers_is_odd() { + let before = "a\nb\nx\ny\nx\n"; + let after = "b\na\nx\ny\n"; - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let input = InternedInput::new(file, file); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - assert_eq!( - diff.unified_diff( + cov_mark::check!(ODD_SPLIT); + // if the check for odd doesn't work then + // we still find the correct result but the number of search + // iterations increases + cov_mark::check_count!(SPLIT_SEARCH_ITER, 9); + let input = InternedInput::new(before, after); + let diff = Diff::compute(Algorithm::Myers, &input); + expect![[r#" + @@ -1,5 +1,4 @@ + -a + b + +a + x + y + -x + "#]] + .assert_eq( + &diff + .unified_diff( &BasicLineDiffPrinter(&input.interner), UnifiedDiffConfig::default(), &input, ) .to_string(), - "" - ); - } -} - -#[test] -fn simple_insert() { - let before = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; -}"#; - - let after = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println("hello world") -}"#; - - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,4 +1,5 @@ - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - + println("hello world") - } - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - - swap(&mut input.before, &mut input.after); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,5 +1,4 @@ - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - - println("hello world") - } - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - swap(&mut input.before, &mut input.after); - } -} - -#[test] -fn unified_diff_context_lines_near_input_start_and_end() { - let before = r#"a -b -c -d -e -f -g -h -i -"#; - - let after = r#"a -b -c -d -edit -f -g -h -i -"#; - - let input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -2,7 +2,7 @@ - b - c - d - -e - +edit - f - g - h - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -mod latin_word_diff { - use crate::sources::words; - use crate::{Algorithm, Diff, InternedInput}; - use std::mem::swap; - - #[test] - fn pure() { - let before = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; -}"#; - let after = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println("hello world") -}"#; - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_insertion()); - assert_eq!(first.before, 0..0); - assert_eq!( - first.after, - 0..words(" println(\"hello world\")\n").count() as u32 - ); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); - - swap(&mut input.before, &mut input.after); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_removal()); - assert_eq!( - first.before, - 0..words(" println(\"hello world\")\n").count() as u32 - ); - assert_eq!(first.after, 0..0); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); - - swap(&mut input.before, &mut input.after); - } - } - - #[test] - fn modify() { - let before = r#"fn foo() -> Bar { - let mut foo = 2.0; - foo *= 100 / 2; -}"#; - let after = r#"fn foo() -> Bar { - let mut foo = 3.0 * 2.0; - foo += 100 / 2; -}"#; - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_insertion()); - let off = words(" let mut foo = ").count() as u32; - assert_eq!(first.before, off..off); - let ins = words("3.0 * ").count() as u32; - assert_eq!(first.after, off..ins + off); - let second = h.next().expect("missing second inner hunk"); - let off = words( - r#" let mut foo = 2.0; - foo "#, - ) - .count() as u32; - assert_eq!(second.before, off..1 + off); - assert_eq!(second.after, ins + off..1 + ins + off); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); - - swap(&mut input.before, &mut input.after); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - - let mut hunks = diff.hunks(); - let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); - assert!(first.is_pure_removal()); - let off = words(" let mut foo = ").count() as u32; - let rem = words("3.0 * ").count() as u32; - assert_eq!(first.before, off..rem + off); - assert_eq!(first.after, off..off); - let second = h.next().expect("missing second inner hunk"); - let off = words( - r#" let mut foo = 2.0; - foo "#, - ) - .count() as u32; - assert_eq!(second.before, rem + off..1 + rem + off); - assert_eq!(second.after, off..1 + off); - assert_eq!(h.next(), None); - assert_eq!(hunks.next(), None); - - swap(&mut input.before, &mut input.after); - } - } -} - -pub fn project_root() -> PathBuf { - let dir = env!("CARGO_MANIFEST_DIR"); - let mut res = PathBuf::from(dir); - while !res.join("README.md").exists() { - res = res - .parent() - .expect("reached fs root without finding project root") - .to_owned() - } - res -} - -#[test] -#[cfg(not(miri))] -fn hand_checked_unidiffs() { - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let test_dir = project_root().join("tests"); - let file = "helix_syntax.rs"; - let path_before = test_dir.join(format!("{file}.before")); - let path_after = test_dir.join(format!("{file}.after")); - let path_diff = test_dir.join(format!("{file}.{algorithm:?}.diff")); - let before = read_to_string(path_before).unwrap(); - let after = read_to_string(path_after).unwrap(); - let input = InternedInput::new(&*before, &*after); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect_file![path_diff].assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -#[test] -#[cfg(not(miri))] -fn complex_diffs() { - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let test_dir = project_root().join("tests"); - for (file1, file2) in [ - ("test1.json", "test2.json"), - ("helix_syntax.rs.Histogram.diff", "helix_syntax.rs.after"), - ] { - let path_before = test_dir.join(file1); - let path_diff = test_dir.join(file2); - let before = read_to_string(path_before).unwrap(); - let after = read_to_string(path_diff).unwrap(); - let input = InternedInput::new(&*before, &*after); - let mut diff = Diff::compute(algorithm, &input); - println!("start postprocess {file1}"); - diff.postprocess_lines(&input); - println!("-{} +{}", diff.count_removals(), diff.count_additions()) - } - } + ); } - -// fn git_diff( -// algo: Algorithm, -// repo: &git::Repository, -// rev1: &git::bstr::BStr, -// rev2: &git::bstr::BStr, -// ) -> String { -// let commit1 = repo -// .rev_parse_single(rev1) -// .unwrap() -// .object() -// .unwrap() -// .peel_to_kind(git::object::Kind::Commit) -// .unwrap() -// .into_commit(); -// let commit2 = repo -// .rev_parse_single(rev2) -// .unwrap() -// .object() -// .unwrap() -// .peel_to_kind(git::object::Kind::Commit) -// .unwrap() -// .into_commit(); -// let mut res = String::new(); -// commit1 -// .tree() -// .unwrap() -// .changes() -// .track_path() -// .for_each_to_obtain_tree( -// &commit2.tree().unwrap(), -// |change| -> Result<_, fmt::Error> { -// match change.event { -// git::object::tree::diff::change::Event::Addition { id, .. } => { -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// writeln!(&mut res, "@@")?; -// for line in blob.data.as_slice().tokenize() { -// write!(&mut res, "+{}", BStr::new(line))?; -// } -// } -// git::object::tree::diff::change::Event::Deletion { id, .. } => { -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// writeln!(&mut res, "@@")?; -// for line in blob.data.as_slice().tokenize() { -// write!(&mut res, "-{}", BStr::new(line))?; -// } -// if !res.ends_with('\n') { -// writeln!(&mut res)?; -// } -// } -// git::object::tree::diff::change::Event::Modification { -// previous_id, -// id, -// .. -// } => { -// let prev_blob = previous_id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// let mut input = InternedInput::default(); -// input.reserve( -// prev_blob.data.as_slice().estimate_tokens(), -// blob.data.as_slice().estimate_tokens(), -// ); -// input.update_before(prev_blob.data.as_slice().tokenize().map(BStr::new)); -// input.update_after(blob.data.as_slice().tokenize().map(BStr::new)); -// let mut diff = Diff::compute(algo, &input); -// diff.postprocess(&input); -// write!( -// &mut res, -// "{}", -// diff.unified_diff( -// &SimpleLineDiff(&input.interner), -// UnifiedDiffConfig::default(), -// &input, -// ) -// )?; -// } -// } -// if !res.ends_with('\n') { -// writeln!(&mut res)?; -// } - -// Ok(git::object::tree::diff::Action::Continue) -// }, -// ) -// .unwrap(); - -// res -// } diff --git a/tests/integration/main.rs b/tests/integration/main.rs new file mode 100644 index 0000000..3bcdbfe --- /dev/null +++ b/tests/integration/main.rs @@ -0,0 +1,457 @@ +use std::fs::read_to_string; +use std::mem::swap; +use std::path::PathBuf; + +use expect_test::{expect, expect_file}; +// use git::bstr::BStr; +// use git_repository as git; + +use imara_diff::sources::words; +use imara_diff::BasicLineDiffPrinter; +use imara_diff::InternedInput; +use imara_diff::{Algorithm, Diff, UnifiedDiffConfig}; + +const ALL_ALGORITHMS: [Algorithm; 2] = [Algorithm::Histogram, Algorithm::Myers]; + +#[test] +fn words_tokenizer() { + let text = "Hello, imara!\n (foo-bar_baz)"; + let tokens = words(text).collect::>(); + assert_eq!( + tokens, + vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar_baz", ")"] + ); +} + +#[test] +fn replace() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println!("hello world") +} +"#; + + let after = r#"const TEST: i32 = 0; +fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println!("hello world"); + println!("hello foo {TEST}"); +} + +"#; + let input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,5 +1,8 @@ + +const TEST: i32 = 0; + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + - println!("hello world") + + println!("hello world"); + + println!("hello foo {TEST}"); + } + + + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +#[test] +fn identical_files() { + let file = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + + for algorithm in ALL_ALGORITHMS { + let input = InternedInput::new(file, file); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + assert_eq!( + diff.unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + "" + ); + } +} + +#[test] +fn simple_insert() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + + let after = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println("hello world") +}"#; + + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,4 +1,5 @@ + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + + println("hello world") + } + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,5 +1,4 @@ + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + - println("hello world") + } + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + swap(&mut input.before, &mut input.after); + } +} + +#[test] +fn unified_diff_context_lines_near_input_start_and_end() { + let before = r#"a +b +c +d +e +f +g +h +i +"#; + + let after = r#"a +b +c +d +edit +f +g +h +i +"#; + + let input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -2,7 +2,7 @@ + b + c + d + -e + +edit + f + g + h + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +mod latin_word_diff { + use crate::ALL_ALGORITHMS; + use imara_diff::sources::words; + use imara_diff::{Diff, InternedInput}; + use std::mem::swap; + + #[test] + fn pure() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println("hello world") +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff_input = InternedInput::default(); + let mut d = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + assert_eq!(first.before, 0..0); + assert_eq!( + first.after, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + assert_eq!( + first.before, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(first.after, 0..0); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + } + } + + #[test] + fn modify() { + let before = r#"fn foo() -> Bar { + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar { + let mut foo = 3.0 * 2.0; + foo += 100 / 2; +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff_input = InternedInput::default(); + let mut d = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + let off = words(" let mut foo = ").count() as u32; + assert_eq!(first.before, off..off); + let ins = words("3.0 * ").count() as u32; + assert_eq!(first.after, off..ins + off); + let second = h.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; + assert_eq!(second.before, off..1 + off); + assert_eq!(second.after, ins + off..1 + ins + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut d); + let mut h = d.hunks(); + let first = h.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + let off = words(" let mut foo = ").count() as u32; + let rem = words("3.0 * ").count() as u32; + assert_eq!(first.before, off..rem + off); + assert_eq!(first.after, off..off); + let second = h.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; + assert_eq!(second.before, rem + off..1 + rem + off); + assert_eq!(second.after, off..1 + off); + assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + } + } +} + +pub fn project_root() -> PathBuf { + let dir = env!("CARGO_MANIFEST_DIR"); + let mut res = PathBuf::from(dir); + while !res.join("README.md").exists() { + res = res + .parent() + .expect("reached fs root without finding project root") + .to_owned() + } + res +} + +#[test] +#[cfg(not(miri))] +fn hand_checked_unidiffs() { + for algorithm in ALL_ALGORITHMS { + println!("{algorithm:?}"); + let test_dir = project_root().join("tests"); + let file = "helix_syntax.rs"; + let path_before = test_dir.join(format!("{file}.before")); + let path_after = test_dir.join(format!("{file}.after")); + let path_diff = test_dir.join(format!("{file}.{algorithm:?}.diff")); + let before = read_to_string(path_before).unwrap(); + let after = read_to_string(path_after).unwrap(); + let input = InternedInput::new(&*before, &*after); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect_file![path_diff].assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +#[test] +#[cfg(not(miri))] +#[ignore = "visually assert diffs, maybe removable if hand-checked unidiffs are the same"] +fn complex_diffs() { + for algorithm in ALL_ALGORITHMS { + let test_dir = project_root().join("tests"); + for (file1, file2) in [ + ("test1.json", "test2.json"), + ("helix_syntax.rs.Histogram.diff", "helix_syntax.rs.after"), + ] { + let path_before = test_dir.join(file1); + let path_diff = test_dir.join(file2); + let before = read_to_string(path_before).unwrap(); + let after = read_to_string(path_diff).unwrap(); + let input = InternedInput::new(&*before, &*after); + let mut diff = Diff::compute(algorithm, &input); + println!("start postprocess {file1}"); + diff.postprocess_lines(&input); + println!("-{} +{}", diff.count_removals(), diff.count_additions()) + } + } +} + +#[test] +fn postprocess() { + let before = r#" + /* + * Stay on the safe side. if read_directory() has run once on + * "dir", some sticky flag may have been left. Clear them all. + */ + clear_sticky(dir); + + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * foo + * bar + * test + */ + foo + "#; + let after = r#" + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * foo + * bar + * test + */ + foo + "#; + + let input = InternedInput::new(before, after); + for algorithm in [Algorithm::Histogram, Algorithm::Myers] { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let diff = diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(); + expect![[r#" + @@ -1,10 +1,4 @@ + + - /* + - * Stay on the safe side. if read_directory() has run once on + - * "dir", some sticky flag may have been left. Clear them all. + - */ + - clear_sticky(dir); + - + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + "#]] + .assert_eq(&diff); + } +} From e92b3c6b8c06199f0abe832c2c97c8d3ba5b6aef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Dec 2025 09:07:02 +0100 Subject: [PATCH 21/21] Add more tests for the word diff to understand postprocessing --- src/lib.rs | 2 +- tests/integration/main.rs | 97 +++++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9150d31..6a6eea6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -414,7 +414,7 @@ impl Hunk { self.after.is_empty() } - /// Performs a word-diff of the hunk. + /// Performs a word-diff on this hunk. /// /// This requires passing the original [`input`](InternedInput) in order to look up /// the tokens of the current hunk, which typically are lines. diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 3bcdbfe..7a98727 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -205,12 +205,14 @@ i mod latin_word_diff { use crate::ALL_ALGORITHMS; + use imara_diff::sources::words; - use imara_diff::{Diff, InternedInput}; + use imara_diff::{Diff, InternedInput, Token}; use std::mem::swap; + use std::ops::Range; #[test] - fn pure() { + fn pure_insertion_or_removal() { let before = r#"fn foo() -> Bar{ let mut foo = 2.0; foo *= 100 / 2; @@ -223,50 +225,60 @@ mod latin_word_diff { let mut input = InternedInput::new(before, after); for algorithm in ALL_ALGORITHMS { let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); + let mut out = Diff::default(); let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); assert!(first.is_pure_insertion()); assert_eq!(first.before, 0..0); assert_eq!( first.after, 0..words(" println(\"hello world\")\n").count() as u32 ); - assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + assert_eq!( + visualise(&diff_input, &first.after, &diff_input.after), + " |println|(|\"|hello| |world|\"|)|\n" + ); swap(&mut input.before, &mut input.after); let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); - let mut hunks = diff.hunks(); + hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); assert!(first.is_pure_removal()); assert_eq!( first.before, 0..words(" println(\"hello world\")\n").count() as u32 ); assert_eq!(first.after, 0..0); - assert_eq!(h.next(), None); assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + assert_eq!( + visualise(&diff_input, &first.before, &diff_input.before), + " |println|(|\"|hello| |world|\"|)|\n" + ); swap(&mut input.before, &mut input.after); } } #[test] - fn modify() { + fn modification() { let before = r#"fn foo() -> Bar { let mut foo = 2.0; foo *= 100 / 2; @@ -278,22 +290,32 @@ mod latin_word_diff { let mut input = InternedInput::new(before, after); for algorithm in ALL_ALGORITHMS { let mut diff_input = InternedInput::default(); - let mut d = Diff::default(); + let mut out = Diff::default(); let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); let mut hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); assert!(first.is_pure_insertion()); let off = words(" let mut foo = ").count() as u32; assert_eq!(first.before, off..off); let ins = words("3.0 * ").count() as u32; assert_eq!(first.after, off..ins + off); - let second = h.next().expect("missing second inner hunk"); + assert_eq!( + visualise(&diff_input, &first.before, &diff_input.before), + "" + ); + assert_eq!( + visualise(&diff_input, &first.after, &diff_input.after), + "3|.|0| |*| " + ); + + let second = hunks.next().expect("missing second inner hunk"); let off = words( r#" let mut foo = 2.0; foo "#, @@ -301,7 +323,15 @@ mod latin_word_diff { .count() as u32; assert_eq!(second.before, off..1 + off); assert_eq!(second.after, ins + off..1 + ins + off); - assert_eq!(h.next(), None); + assert_eq!( + visualise(&diff_input, &second.before, &diff_input.before), + "*" + ); + assert_eq!( + visualise(&diff_input, &second.after, &diff_input.after), + "+" + ); + assert_eq!(hunks.next(), None); assert_eq!(hunks.next(), None); swap(&mut input.before, &mut input.after); @@ -309,17 +339,19 @@ mod latin_word_diff { let mut diff = Diff::compute(algorithm, &input); diff.postprocess_lines(&input); - let mut hunks = diff.hunks(); + hunks = diff.hunks(); let hunk = hunks.next().expect("missing first hunk"); - hunk.latin_word_diff(&input, &mut diff_input, &mut d); - let mut h = d.hunks(); - let first = h.next().expect("missing first inner hunk"); + + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); assert!(first.is_pure_removal()); let off = words(" let mut foo = ").count() as u32; let rem = words("3.0 * ").count() as u32; assert_eq!(first.before, off..rem + off); assert_eq!(first.after, off..off); - let second = h.next().expect("missing second inner hunk"); + let second = hunks.next().expect("missing second inner hunk"); let off = words( r#" let mut foo = 2.0; foo "#, @@ -327,12 +359,27 @@ mod latin_word_diff { .count() as u32; assert_eq!(second.before, rem + off..1 + rem + off); assert_eq!(second.after, off..1 + off); - assert_eq!(h.next(), None); + assert_eq!(hunks.next(), None); assert_eq!(hunks.next(), None); swap(&mut input.before, &mut input.after); } } + + fn visualise( + diff_input: &InternedInput<&str>, + token_ids: &Range, + tokens: &[Token], + ) -> String { + token_ids + .clone() + .map(|id| { + let id = id as usize; + diff_input.interner[tokens[id]] + }) + .collect::>() + .join("|") + } } pub fn project_root() -> PathBuf {