diff --git a/Cargo.lock b/Cargo.lock index 59140cd..1829ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "cov-mark" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1d92727879fb4f24cec33a35e3bff74035541326cbc12ad44ba8886d1927b0" +checksum = "90863d8442510cddf7f46618c4f92413774635771a3e80830c8b30d183420b14" [[package]] name = "dissimilar" @@ -32,9 +32,9 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "once_cell" diff --git a/src/lib.rs b/src/lib.rs index ce8d306..6a6eea6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,7 +145,10 @@ use std::ops::Range; use std::slice; -use crate::util::{strip_common_postfix, strip_common_prefix}; +use crate::{ + sources::words, + util::{strip_common_postfix, strip_common_prefix}, +}; pub use crate::slider_heuristic::{ IndentHeuristic, IndentLevel, NoSliderHeuristic, SliderHeuristic, @@ -160,13 +163,12 @@ mod myers; mod postprocess; mod slider_heuristic; pub mod sources; +#[cfg(test)] +mod tests; #[cfg(feature = "unified_diff")] mod unified_diff; mod util; -#[cfg(test)] -mod tests; - /// `imara-diff` supports multiple different algorithms /// for computing an edit sequence. /// These algorithms have different performance and all produce different output. @@ -225,11 +227,6 @@ pub enum Algorithm { MyersMinimal, } -impl Algorithm { - #[cfg(test)] - const ALL: [Self; 2] = [Algorithm::Histogram, Algorithm::Myers]; -} - /// Represents the difference between two sequences of tokens. /// /// A `Diff` stores which tokens were removed from the first sequence and which tokens were added to the second sequence. @@ -416,6 +413,81 @@ impl Hunk { pub fn is_pure_removal(&self) -> bool { self.after.is_empty() } + + /// Performs a word-diff on this hunk. + /// + /// This requires passing the original [`input`](InternedInput) in order to look up + /// the tokens of the current hunk, which typically are lines. + /// Each token is split into words using the built-in [`words`] tokenizer. + /// The resulting word tokens are stored in a second [`diff_input`](InternedInput), + /// and a [`diff`](Diff) is computed on them, with basic post-processing applied. + /// + /// For performance reasons, this second [`diff_input`](InternedInput) as well as + /// the computed [`diff`](Diff) need to be passed as parameters so that they can be + /// re-used when iterating over hunks. Note that word tokens are always + /// added but never removed from the interner. Consider clearing it if you expect + /// your input to have a large vocabulary. + /// + /// # Examples + /// + /// ``` + /// # use imara_diff::{InternedInput, Diff, Algorithm}; + /// // Compute diff normally + /// let before = "before text"; + /// let after = "after text"; + /// let mut lines = InternedInput::new(before, after); + /// let mut diff = Diff::compute(Algorithm::Histogram, &lines); + /// diff.postprocess_lines(&lines); + /// + /// // Compute word-diff per hunk, reusing allocations across iterations + /// let mut hunk_diff_input = InternedInput::default(); + /// let mut hunk_diff = Diff::default(); + /// for hunk in diff.hunks() { + /// hunk.latin_word_diff(&lines, &mut hunk_diff_input, &mut hunk_diff); + /// let added = hunk_diff.count_additions(); + /// let removed = hunk_diff.count_removals(); + /// println!("word-diff of this hunk has {added} additions and {removed} removals"); + /// // optionally, clear the interner: + /// hunk_diff_input.clear(); + /// } + /// ``` + pub fn latin_word_diff<'a>( + &self, + input: &InternedInput<&'a str>, + word_tokens: &mut InternedInput<&'a str>, + diff: &mut Diff, + ) { + let Hunk { before, after } = self.clone(); + word_tokens.update_before( + before + .map(|index| input.before[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); + word_tokens.update_after( + after + .map(|index| input.after[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); + diff.removed.clear(); + diff.removed.resize(word_tokens.before.len(), false); + diff.added.clear(); + diff.added.resize(word_tokens.after.len(), false); + if self.is_pure_removal() { + diff.removed.fill(true); + } else if self.is_pure_insertion() { + diff.added.fill(true); + } else { + diff.compute_with( + Algorithm::Myers, + &word_tokens.before, + &word_tokens.after, + word_tokens.interner.num_tokens(), + ); + diff.postprocess_no_heuristic(word_tokens); + } + } } /// Yields all [`Hunk`]s in a file in monotonically increasing order. diff --git a/src/sources.rs b/src/sources.rs index 0b668b6..e8ddcfc 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -17,6 +17,14 @@ pub fn lines(data: &str) -> Lines<'_> { Lines(ByteLines(data.as_bytes())) } +/// Returns a [`TokenSource`] that uses the words in `data` as Tokens. A word is +/// a sequence of alphanumeric characters as determined by +/// `char::is_alphanumeric`, or a sequence of just the space character ' '. Any +/// other characters are their own word. +pub fn words(data: &str) -> Words<'_> { + Words(data) +} + /// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline /// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing /// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is @@ -84,6 +92,53 @@ impl<'a> TokenSource for Lines<'a> { } } +/// A [`TokenSource`] that returns the words of a string as tokens. See +/// [`words`] for details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Words<'a>(&'a str); + +impl<'a> Iterator for Words<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + if self.0.is_empty() { + return None; + } + + let initial = self.0.chars().next().unwrap(); + let word_len = if initial == ' ' { + self.0 + .char_indices() + .find(|(_, c)| *c != ' ') + .map_or(self.0.len(), |(index, _)| index) + } else if initial.is_alphanumeric() { + self.0 + .char_indices() + .find(|(_, c)| !c.is_alphanumeric() && *c != '_') + .map_or(self.0.len(), |(index, _)| index) + } else { + initial.len_utf8() + }; + + let (word, rem) = self.0.split_at(word_len); + self.0 = rem; + Some(word) + } +} +impl<'a> TokenSource for Words<'a> { + type Token = &'a str; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + (self.0.len() / 3) as u32 + } +} + /// A [`TokenSource`] that returns the lines of a byte slice as tokens. See [`byte_lines`] /// for details. #[derive(Clone, Copy, PartialEq, Eq)] diff --git a/src/tests.rs b/src/tests.rs index 721282b..a0e9355 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,154 +1,6 @@ -use std::fs::read_to_string; -use std::mem::swap; -use std::path::PathBuf; +use crate::{Algorithm, BasicLineDiffPrinter, Diff, InternedInput, UnifiedDiffConfig}; +use expect_test::expect; -use expect_test::{expect, expect_file}; -// use git::bstr::BStr; -// use git_repository as git; - -use crate::intern::InternedInput; -use crate::unified_diff::BasicLineDiffPrinter; -use crate::{Algorithm, Diff, UnifiedDiffConfig}; - -#[test] -fn postprocess() { - let before = r#" - /* - * Stay on the safe side. if read_directory() has run once on - * "dir", some sticky flag may have been left. Clear them all. - */ - clear_sticky(dir); - - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - * subset of positive ones, which has no impacts on - * foo - * bar - * test - */ - foo - "#; - let after = r#" - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - * subset of positive ones, which has no impacts on - * foo - * bar - * test - */ - foo - "#; - - let input = InternedInput::new(before, after); - for algorithm in [Algorithm::Histogram, Algorithm::Myers] { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - let diff = diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(); - println!("{diff:?}"); - expect![[r#" - @@ -1,10 +1,4 @@ - - - /* - - * Stay on the safe side. if read_directory() has run once on - - * "dir", some sticky flag may have been left. Clear them all. - - */ - - clear_sticky(dir); - - - /* - * exclude patterns are treated like positive ones in - * create_simplify. Usually exclude patterns should be a - "#]] - .assert_eq(&diff); - } -} - -#[test] -fn replace() { - let before = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println!("hello world") -} -"#; - - let after = r#"const TEST: i32 = 0; -fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println!("hello world"); - println!("hello foo {TEST}"); -} - -"#; - let input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,5 +1,8 @@ - +const TEST: i32 = 0; - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - - println!("hello world") - + println!("hello world"); - + println!("hello foo {TEST}"); - } - + - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -#[test] -fn myers_is_odd() { - let before = "a\nb\nx\ny\nx\n"; - let after = "b\na\nx\ny\n"; - - cov_mark::check!(ODD_SPLIT); - // if the check for odd doesn't work then - // we still find the correct result but the number of search - // iterations increases - cov_mark::check_count!(SPLIT_SEARCH_ITER, 9); - let input = InternedInput::new(before, after); - let diff = Diff::compute(Algorithm::Myers, &input); - expect![[r#" - @@ -1,5 +1,4 @@ - -a - b - +a - x - y - -x - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); -} #[test] fn myers_is_even() { let before = "a\nb\nx\nx\ny\n"; @@ -184,304 +36,33 @@ fn myers_is_even() { } #[test] -fn identical_files() { - let file = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; -}"#; +fn myers_is_odd() { + let before = "a\nb\nx\ny\nx\n"; + let after = "b\na\nx\ny\n"; - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let input = InternedInput::new(file, file); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - assert_eq!( - diff.unified_diff( + cov_mark::check!(ODD_SPLIT); + // if the check for odd doesn't work then + // we still find the correct result but the number of search + // iterations increases + cov_mark::check_count!(SPLIT_SEARCH_ITER, 9); + let input = InternedInput::new(before, after); + let diff = Diff::compute(Algorithm::Myers, &input); + expect![[r#" + @@ -1,5 +1,4 @@ + -a + b + +a + x + y + -x + "#]] + .assert_eq( + &diff + .unified_diff( &BasicLineDiffPrinter(&input.interner), UnifiedDiffConfig::default(), &input, ) .to_string(), - "" - ); - } -} - -#[test] -fn simple_insert() { - let before = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; -}"#; - - let after = r#"fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - println("hello world") -}"#; - - let mut input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,4 +1,5 @@ - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - + println("hello world") - } - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - - swap(&mut input.before, &mut input.after); - - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -1,5 +1,4 @@ - fn foo() -> Bar{ - let mut foo = 2.0; - foo *= 100 / 2; - - println("hello world") - } - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - swap(&mut input.before, &mut input.after); - } -} - -#[test] -fn unified_diff_context_lines_near_input_start_and_end() { - let before = r#"a -b -c -d -e -f -g -h -i -"#; - - let after = r#"a -b -c -d -edit -f -g -h -i -"#; - - let input = InternedInput::new(before, after); - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect![[r#" - @@ -2,7 +2,7 @@ - b - c - d - -e - +edit - f - g - h - "#]] - .assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -pub fn project_root() -> PathBuf { - let dir = env!("CARGO_MANIFEST_DIR"); - let mut res = PathBuf::from(dir); - while !res.join("README.md").exists() { - res = res - .parent() - .expect("reached fs root without finding project root") - .to_owned() - } - res -} - -#[test] -#[cfg(not(miri))] -fn hand_checked_unidiffs() { - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let test_dir = project_root().join("tests"); - let file = "helix_syntax.rs"; - let path_before = test_dir.join(format!("{file}.before")); - let path_after = test_dir.join(format!("{file}.after")); - let path_diff = test_dir.join(format!("{file}.{algorithm:?}.diff")); - let before = read_to_string(path_before).unwrap(); - let after = read_to_string(path_after).unwrap(); - let input = InternedInput::new(&*before, &*after); - let mut diff = Diff::compute(algorithm, &input); - diff.postprocess_lines(&input); - expect_file![path_diff].assert_eq( - &diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(), - ); - } -} - -#[test] -#[cfg(not(miri))] -fn complex_diffs() { - for algorithm in Algorithm::ALL { - println!("{algorithm:?}"); - let test_dir = project_root().join("tests"); - for (file1, file2) in [ - ("test1.json", "test2.json"), - ("helix_syntax.rs.Histogram.diff", "helix_syntax.rs.after"), - ] { - let path_before = test_dir.join(file1); - let path_diff = test_dir.join(file2); - let before = read_to_string(path_before).unwrap(); - let after = read_to_string(path_diff).unwrap(); - let input = InternedInput::new(&*before, &*after); - let mut diff = Diff::compute(algorithm, &input); - println!("start postprocess {file1}"); - diff.postprocess_lines(&input); - println!("-{} +{}", diff.count_removals(), diff.count_additions()) - } - } + ); } - -// fn git_diff( -// algo: Algorithm, -// repo: &git::Repository, -// rev1: &git::bstr::BStr, -// rev2: &git::bstr::BStr, -// ) -> String { -// let commit1 = repo -// .rev_parse_single(rev1) -// .unwrap() -// .object() -// .unwrap() -// .peel_to_kind(git::object::Kind::Commit) -// .unwrap() -// .into_commit(); -// let commit2 = repo -// .rev_parse_single(rev2) -// .unwrap() -// .object() -// .unwrap() -// .peel_to_kind(git::object::Kind::Commit) -// .unwrap() -// .into_commit(); -// let mut res = String::new(); -// commit1 -// .tree() -// .unwrap() -// .changes() -// .track_path() -// .for_each_to_obtain_tree( -// &commit2.tree().unwrap(), -// |change| -> Result<_, fmt::Error> { -// match change.event { -// git::object::tree::diff::change::Event::Addition { id, .. } => { -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// writeln!(&mut res, "@@")?; -// for line in blob.data.as_slice().tokenize() { -// write!(&mut res, "+{}", BStr::new(line))?; -// } -// } -// git::object::tree::diff::change::Event::Deletion { id, .. } => { -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// writeln!(&mut res, "@@")?; -// for line in blob.data.as_slice().tokenize() { -// write!(&mut res, "-{}", BStr::new(line))?; -// } -// if !res.ends_with('\n') { -// writeln!(&mut res)?; -// } -// } -// git::object::tree::diff::change::Event::Modification { -// previous_id, -// id, -// .. -// } => { -// let prev_blob = previous_id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// let blob = id -// .object() -// .unwrap() -// .peel_to_kind(git::objs::Kind::Blob) -// .unwrap(); -// let mut input = InternedInput::default(); -// input.reserve( -// prev_blob.data.as_slice().estimate_tokens(), -// blob.data.as_slice().estimate_tokens(), -// ); -// input.update_before(prev_blob.data.as_slice().tokenize().map(BStr::new)); -// input.update_after(blob.data.as_slice().tokenize().map(BStr::new)); -// let mut diff = Diff::compute(algo, &input); -// diff.postprocess(&input); -// write!( -// &mut res, -// "{}", -// diff.unified_diff( -// &SimpleLineDiff(&input.interner), -// UnifiedDiffConfig::default(), -// &input, -// ) -// )?; -// } -// } -// if !res.ends_with('\n') { -// writeln!(&mut res)?; -// } - -// Ok(git::object::tree::diff::Action::Continue) -// }, -// ) -// .unwrap(); - -// res -// } diff --git a/tests/integration/main.rs b/tests/integration/main.rs new file mode 100644 index 0000000..7a98727 --- /dev/null +++ b/tests/integration/main.rs @@ -0,0 +1,504 @@ +use std::fs::read_to_string; +use std::mem::swap; +use std::path::PathBuf; + +use expect_test::{expect, expect_file}; +// use git::bstr::BStr; +// use git_repository as git; + +use imara_diff::sources::words; +use imara_diff::BasicLineDiffPrinter; +use imara_diff::InternedInput; +use imara_diff::{Algorithm, Diff, UnifiedDiffConfig}; + +const ALL_ALGORITHMS: [Algorithm; 2] = [Algorithm::Histogram, Algorithm::Myers]; + +#[test] +fn words_tokenizer() { + let text = "Hello, imara!\n (foo-bar_baz)"; + let tokens = words(text).collect::>(); + assert_eq!( + tokens, + vec!["Hello", ",", " ", "imara", "!", "\n", " ", "(", "foo", "-", "bar_baz", ")"] + ); +} + +#[test] +fn replace() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println!("hello world") +} +"#; + + let after = r#"const TEST: i32 = 0; +fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println!("hello world"); + println!("hello foo {TEST}"); +} + +"#; + let input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,5 +1,8 @@ + +const TEST: i32 = 0; + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + - println!("hello world") + + println!("hello world"); + + println!("hello foo {TEST}"); + } + + + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +#[test] +fn identical_files() { + let file = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + + for algorithm in ALL_ALGORITHMS { + let input = InternedInput::new(file, file); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + assert_eq!( + diff.unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + "" + ); + } +} + +#[test] +fn simple_insert() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + + let after = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println("hello world") +}"#; + + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,4 +1,5 @@ + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + + println("hello world") + } + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -1,5 +1,4 @@ + fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + - println("hello world") + } + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + swap(&mut input.before, &mut input.after); + } +} + +#[test] +fn unified_diff_context_lines_near_input_start_and_end() { + let before = r#"a +b +c +d +e +f +g +h +i +"#; + + let after = r#"a +b +c +d +edit +f +g +h +i +"#; + + let input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect![[r#" + @@ -2,7 +2,7 @@ + b + c + d + -e + +edit + f + g + h + "#]] + .assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +mod latin_word_diff { + use crate::ALL_ALGORITHMS; + + use imara_diff::sources::words; + use imara_diff::{Diff, InternedInput, Token}; + use std::mem::swap; + use std::ops::Range; + + #[test] + fn pure_insertion_or_removal() { + let before = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar{ + let mut foo = 2.0; + foo *= 100 / 2; + println("hello world") +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff_input = InternedInput::default(); + let mut out = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + assert_eq!(first.before, 0..0); + assert_eq!( + first.after, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + assert_eq!( + visualise(&diff_input, &first.after, &diff_input.after), + " |println|(|\"|hello| |world|\"|)|\n" + ); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + assert_eq!( + first.before, + 0..words(" println(\"hello world\")\n").count() as u32 + ); + assert_eq!(first.after, 0..0); + assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + assert_eq!( + visualise(&diff_input, &first.before, &diff_input.before), + " |println|(|\"|hello| |world|\"|)|\n" + ); + + swap(&mut input.before, &mut input.after); + } + } + + #[test] + fn modification() { + let before = r#"fn foo() -> Bar { + let mut foo = 2.0; + foo *= 100 / 2; +}"#; + let after = r#"fn foo() -> Bar { + let mut foo = 3.0 * 2.0; + foo += 100 / 2; +}"#; + let mut input = InternedInput::new(before, after); + for algorithm in ALL_ALGORITHMS { + let mut diff_input = InternedInput::default(); + let mut out = Diff::default(); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + let mut hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); + assert!(first.is_pure_insertion()); + let off = words(" let mut foo = ").count() as u32; + assert_eq!(first.before, off..off); + let ins = words("3.0 * ").count() as u32; + assert_eq!(first.after, off..ins + off); + assert_eq!( + visualise(&diff_input, &first.before, &diff_input.before), + "" + ); + assert_eq!( + visualise(&diff_input, &first.after, &diff_input.after), + "3|.|0| |*| " + ); + + let second = hunks.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; + assert_eq!(second.before, off..1 + off); + assert_eq!(second.after, ins + off..1 + ins + off); + assert_eq!( + visualise(&diff_input, &second.before, &diff_input.before), + "*" + ); + assert_eq!( + visualise(&diff_input, &second.after, &diff_input.after), + "+" + ); + assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + + hunks = diff.hunks(); + let hunk = hunks.next().expect("missing first hunk"); + + hunk.latin_word_diff(&input, &mut diff_input, &mut out); + hunks = out.hunks(); + + let first = hunks.next().expect("missing first inner hunk"); + assert!(first.is_pure_removal()); + let off = words(" let mut foo = ").count() as u32; + let rem = words("3.0 * ").count() as u32; + assert_eq!(first.before, off..rem + off); + assert_eq!(first.after, off..off); + let second = hunks.next().expect("missing second inner hunk"); + let off = words( + r#" let mut foo = 2.0; + foo "#, + ) + .count() as u32; + assert_eq!(second.before, rem + off..1 + rem + off); + assert_eq!(second.after, off..1 + off); + assert_eq!(hunks.next(), None); + assert_eq!(hunks.next(), None); + + swap(&mut input.before, &mut input.after); + } + } + + fn visualise( + diff_input: &InternedInput<&str>, + token_ids: &Range, + tokens: &[Token], + ) -> String { + token_ids + .clone() + .map(|id| { + let id = id as usize; + diff_input.interner[tokens[id]] + }) + .collect::>() + .join("|") + } +} + +pub fn project_root() -> PathBuf { + let dir = env!("CARGO_MANIFEST_DIR"); + let mut res = PathBuf::from(dir); + while !res.join("README.md").exists() { + res = res + .parent() + .expect("reached fs root without finding project root") + .to_owned() + } + res +} + +#[test] +#[cfg(not(miri))] +fn hand_checked_unidiffs() { + for algorithm in ALL_ALGORITHMS { + println!("{algorithm:?}"); + let test_dir = project_root().join("tests"); + let file = "helix_syntax.rs"; + let path_before = test_dir.join(format!("{file}.before")); + let path_after = test_dir.join(format!("{file}.after")); + let path_diff = test_dir.join(format!("{file}.{algorithm:?}.diff")); + let before = read_to_string(path_before).unwrap(); + let after = read_to_string(path_after).unwrap(); + let input = InternedInput::new(&*before, &*after); + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + expect_file![path_diff].assert_eq( + &diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(), + ); + } +} + +#[test] +#[cfg(not(miri))] +#[ignore = "visually assert diffs, maybe removable if hand-checked unidiffs are the same"] +fn complex_diffs() { + for algorithm in ALL_ALGORITHMS { + let test_dir = project_root().join("tests"); + for (file1, file2) in [ + ("test1.json", "test2.json"), + ("helix_syntax.rs.Histogram.diff", "helix_syntax.rs.after"), + ] { + let path_before = test_dir.join(file1); + let path_diff = test_dir.join(file2); + let before = read_to_string(path_before).unwrap(); + let after = read_to_string(path_diff).unwrap(); + let input = InternedInput::new(&*before, &*after); + let mut diff = Diff::compute(algorithm, &input); + println!("start postprocess {file1}"); + diff.postprocess_lines(&input); + println!("-{} +{}", diff.count_removals(), diff.count_additions()) + } + } +} + +#[test] +fn postprocess() { + let before = r#" + /* + * Stay on the safe side. if read_directory() has run once on + * "dir", some sticky flag may have been left. Clear them all. + */ + clear_sticky(dir); + + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * foo + * bar + * test + */ + foo + "#; + let after = r#" + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * foo + * bar + * test + */ + foo + "#; + + let input = InternedInput::new(before, after); + for algorithm in [Algorithm::Histogram, Algorithm::Myers] { + let mut diff = Diff::compute(algorithm, &input); + diff.postprocess_lines(&input); + let diff = diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(); + expect![[r#" + @@ -1,10 +1,4 @@ + + - /* + - * Stay on the safe side. if read_directory() has run once on + - * "dir", some sticky flag may have been left. Clear them all. + - */ + - clear_sticky(dir); + - + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + "#]] + .assert_eq(&diff); + } +}