Add #![deny(missing_docs)] and comprehensive documentation

src/histogram.rs

@@ -6,13 +6,27 @@ use crate::myers;
 mod lcs;
 mod list_pool;
 
+/// Maximum number of occurrences tracked for a single token.
+/// Tokens appearing more frequently fall back to Myers algorithm.
 const MAX_CHAIN_LEN: u32 = 63;
 
+/// State for computing histogram-based diffs.
 struct Histogram {
+    /// Tracks where each token appears in the "before" sequence.
     token_occurrences: Vec<ListHandle>,
+    /// Memory pool for efficiently storing occurrence lists.
     pool: ListPool,
 }
 
+/// Computes a diff using the histogram algorithm.
+///
+/// # Parameters
+///
+/// * `before` - The token sequence from the first file, before changes.
+/// * `after` - The token sequence from the second file, after changes.
+/// * `removed` - Output array marking removed tokens
+/// * `added` - Output array marking added tokens
+/// * `num_tokens` - The total number of distinct tokens
 pub fn diff(
     before: &[Token],
     after: &[Token],

src/histogram/lcs.rs

@@ -1,6 +1,9 @@
 use crate::histogram::{Histogram, MAX_CHAIN_LEN};
 use crate::intern::Token;
 
+/// Finds the longest common subsequence (LCS) using a histogram-based approach.
+///
+/// Returns `None` if the sequences are highly repetitive and should fall back to Myers.
 pub(super) fn find_lcs(
     before: &[Token],
     after: &[Token],
@@ -19,16 +22,24 @@ pub(super) fn find_lcs(
     }
 }
 
+/// Represents a longest common subsequence found by the histogram algorithm.
 #[derive(Default, Debug)]
 pub struct Lcs {
+    /// Starting position in the "before" sequence.
     pub before_start: u32,
+    /// Starting position in the "after" sequence.
     pub after_start: u32,
+    /// Length of the common subsequence.
     pub len: u32,
 }
 
+/// State for searching for the longest common subsequence.
 pub struct LcsSearch {
+    /// The best LCS found so far.
     lcs: Lcs,
+    /// The minimum occurrence count of tokens in the best LCS.
     min_occurrences: u32,
+    /// Whether any common subsequence was found.
     found_cs: bool,
 }
 

src/intern.rs

@@ -29,14 +29,26 @@ impl From<Token> for u32 {
     }
 }
 
+/// A trait for types that can be split into tokens for diffing.
+///
+/// Implementing this trait allows a type to be used with [`InternedInput`] to create
+/// interned token sequences for computing diffs. For example, `&str` implements this trait
+/// by default to split text into lines.
 pub trait TokenSource {
+    /// The type of token this source produces. Must be hashable and comparable for equality.
     type Token: Hash + Eq;
+    /// An iterator that yields tokens from this source.
     type Tokenizer: Iterator<Item = Self::Token>;
+    /// Creates an iterator that yields all tokens from this source.
     fn tokenize(&self) -> Self::Tokenizer;
+    /// Provides an estimate of the number of tokens this source will produce.
+    ///
+    /// This is used to pre-allocate memory for better performance. The estimate
+    /// does not need to be exact.
     fn estimate_tokens(&self) -> u32;
 }
 
-/// Two lists of interned [tokens](crate::intern::Token) that a [`Diff`](crate::Diff) can be computed from.
+/// Two lists of interned [tokens](Token) that a [`Diff`](crate::Diff) can be computed from.
 ///
 /// A token represents the smallest possible unit of change during a diff.
 /// For text this is usually a line, a word or a single character.
@@ -47,12 +59,21 @@ pub trait TokenSource {
 /// While you can intern tokens yourself it is strongly recommended to use [`InternedInput`] module.
 #[derive(Default)]
 pub struct InternedInput<T> {
+    /// The list of interned tokens from the first sequence (before changes).
     pub before: Vec<Token>,
+    /// The list of interned tokens from the second sequence (after changes).
     pub after: Vec<Token>,
+    /// The interner that stores the actual token data and maps tokens to their interned IDs.
     pub interner: Interner<T>,
 }
 
 impl<T> InternedInput<T> {
+    /// Clears all token sequences and the interner.
+    ///
+    /// This removes all tokens from both the before and after sequences, as well as
+    /// clearing the interner's storage.
+    ///
+    /// Note that this will not free the allocated memory.
     pub fn clear(&mut self) {
         self.before.clear();
         self.after.clear();
@@ -61,6 +82,16 @@ impl<T> InternedInput<T> {
 }
 
 impl<T: Eq + Hash> InternedInput<T> {
+    /// Creates a new `InternedInput` by tokenizing and interning two token sources.
+    ///
+    /// # Parameters
+    ///
+    /// * `before` - The token source for the first sequence
+    /// * `after` - The token source for the second sequence
+    ///
+    /// # Returns
+    ///
+    /// An `InternedInput` containing interned token sequences ready for diffing
     pub fn new<I: TokenSource<Token = T>>(before: I, after: I) -> Self {
         let token_estimate_before = before.estimate_tokens() as usize;
         let token_estimate_after = after.estimate_tokens() as usize;
@@ -74,8 +105,13 @@ impl<T: Eq + Hash> InternedInput<T> {
         res
     }
 
-    /// Create an Interner with an intial capacity calculated by calling
-    /// [`estimate_tokens`](crate::intern::TokenSource::estimate_tokens) methods of `before` and `after`
+    /// Reserve capacity so that `before` and `after` would not need
+    /// to allocate if their [`estimate_tokens`](TokenSource::estimate_tokens)
+    /// would represent an exact match of their actual tokens.
+    ///
+    /// Useful for minimisation of allocation before calls to
+    /// [`update_before`](InternedInput::update_before) and
+    /// [`update_after`](InternedInput::update_after).
     pub fn reserve_for_token_source<S: TokenSource<Token = T> + ?Sized>(
         &mut self,
         before: &S,
@@ -84,6 +120,12 @@ impl<T: Eq + Hash> InternedInput<T> {
         self.reserve(before.estimate_tokens(), after.estimate_tokens())
     }
 
+    /// Reserves capacity for the specified number of tokens in each sequence.
+    ///
+    /// # Parameters
+    ///
+    /// * `capacity_before` - The number of tokens to reserve for the "before" sequence
+    /// * `capacity_after` - The number of tokens to reserve for the "after" sequence
     pub fn reserve(&mut self, capacity_before: u32, capacity_after: u32) {
         self.before.reserve(capacity_before as usize);
         self.after.reserve(capacity_after as usize);
@@ -94,7 +136,7 @@ impl<T: Eq + Hash> InternedInput<T> {
     /// replaces `self.before` with the interned Tokens yielded by `input`
     /// Note that this does not erase any tokens from the interner and might therefore be considered
     /// a memory leak. If this function is called often over a long_running process
-    /// consider clearing the interner with [`clear`](crate::intern::Interner::clear).
+    /// consider clearing the interner with [`clear`](Interner::clear).
     pub fn update_before(&mut self, input: impl Iterator<Item = T>) {
         self.before.clear();
         self.before
@@ -104,8 +146,8 @@ impl<T: Eq + Hash> InternedInput<T> {
     /// replaces `self.before` with the interned Tokens yielded by `input`
     /// Note that this does not erase any tokens from the interner and might therefore be considered
     /// a memory leak. If this function is called often over a long_running process
-    /// consider clearing the interner with [`clear`](crate::intern::Interner::clear) or
-    /// [`erase_tokens_after`](crate::intern::Interner::erase_tokens_after).
+    /// consider clearing the interner with [`clear`](Interner::clear) or
+    /// [`erase_tokens_after`](Interner::erase_tokens_after).
     pub fn update_after(&mut self, input: impl Iterator<Item = T>) {
         self.after.clear();
         self.after
@@ -123,7 +165,7 @@ pub struct Interner<T> {
 
 impl<T> Interner<T> {
     /// Create an Interner with an initial capacity calculated by summing the results of calling
-    /// [`estimate_tokens`](crate::intern::TokenSource::estimate_tokens) methods of `before` and `after`.
+    /// [`estimate_tokens`](TokenSource::estimate_tokens) methods of `before` and `after`.
     pub fn new_for_token_source<S: TokenSource<Token = T>>(before: &S, after: &S) -> Self {
         Self::new(before.estimate_tokens() as usize + after.estimate_tokens() as usize)
     }
@@ -150,20 +192,25 @@ impl<T> Interner<T> {
 }
 
 impl<T: Hash + Eq> Interner<T> {
-    /// Create an Interner with an intial capacity calculated by calling
-    /// [`estimate_tokens`](crate::intern::TokenSource::estimate_tokens) methods of `before` and `after`
+    /// Create an Interner with an initial capacity calculated by calling
+    /// [`estimate_tokens`](TokenSource::estimate_tokens) methods of `before` and `after`
     pub fn reserve_for_token_source<S: TokenSource<Token = T>>(&mut self, before: &S, after: &S) {
         self.reserve(before.estimate_tokens() as usize + after.estimate_tokens() as usize)
     }
 
+    /// Reserves capacity for at least the specified number of additional tokens.
+    ///
+    /// # Parameters
+    ///
+    /// * `capacity` - The number of additional tokens to reserve space for
     pub fn reserve(&mut self, capacity: usize) {
         self.table.reserve(capacity, |&token| {
             self.hasher.hash_one(&self.tokens[token.0 as usize])
         });
         self.tokens.reserve(capacity);
     }
 
-    /// Intern `token` and return a the interned integer.
+    /// Intern `token` and return the interned integer.
     pub fn intern(&mut self, token: T) -> Token {
         let hash = self.hasher.hash_one(&token);
         match self.table.entry(