refactor: update documentation for clarity and accuracy in process and matcher modules

Author: Foster Guo

matcher_rs/src/process/mod.rs

@@ -19,8 +19,9 @@
 //!
 //! - [`step`] — [`TransformStep`](step::TransformStep) enum and the global
 //!   `OnceLock` cache that lazily compiles each single-bit step once.
-//! - [`transform`] — Low-level engines (charwise page-table, Aho-Corasick
-//!   normalizer, SIMD delete).
+//! - [`transform`] — Low-level engines (page-table lookups for
+//!   VariantNorm/Normalize/Romanize/EmojiNorm, bitset-based Delete, SIMD skip
+//!   helpers).
 pub(crate) mod process_type;
 pub(crate) mod step;
 pub(crate) mod transform;

matcher_rs/src/process/process_type.rs

@@ -97,7 +97,7 @@ bitflags! {
         /// Applies the Normalize replacement tables (e.g. full-width forms, digit-like
         /// variants).
         ///
-        /// Uses an Aho-Corasick automaton compiled from `process_map/NORM.txt` and
+        /// Uses a page-table lookup compiled from `process_map/NORM.txt` and
         /// `process_map/NUM-NORM.txt`.
         const Normalize = 0b00001000;
 
@@ -125,7 +125,7 @@ bitflags! {
         /// Also strips emoji modifiers (ZWJ, VS16, skin tones) by mapping them to empty string.
         ///
         /// Does NOT compose usefully with [`Delete`](Self::Delete) — Delete removes emoji
-        /// before EmojiNorm can see them. Use one or the other.
+        /// before EmojiNorm can see them. Use `EmojiNorm | Normalize` for emoji→word matching.
         const EmojiNorm = 0b01000000;
     }
 }

matcher_rs/src/process/step.rs

@@ -1,7 +1,7 @@
 //! Compiled single-step transforms for the text-processing pipeline.
 //!
-//! Each [`TransformStep`] variant wraps a low-level matcher (VariantNorm,
-//! Delete, Normalize, Romanize) and provides a uniform
+//! Each [`TransformStep`] variant wraps a low-level matcher (None, VariantNorm,
+//! Delete, Normalize, Romanize, RomanizeChar, EmojiNorm) and provides a uniform
 //! [`apply`](TransformStep::apply) interface. Returns `Option<String>` — `None`
 //! when the input is unaffected.
 //!
@@ -37,7 +37,7 @@ pub(crate) enum TransformStep {
     VariantNorm(VariantNormMatcher),
     /// Codepoint deletion using a bitset, with optional SIMD acceleration.
     Delete(DeleteMatcher),
-    /// Multi-character normalization replacements via Aho-Corasick.
+    /// Multi-character normalization replacements via page-table lookup.
     Normalize(NormalizeMatcher),
     /// CJK romanization with inter-syllable spaces preserved.
     Romanize(RomanizeMatcher),
@@ -47,13 +47,14 @@ pub(crate) enum TransformStep {
     EmojiNorm(RomanizeMatcher),
 }
 
-/// Streaming byte iterator wrapping one of the four fusible
-/// [`FilterIterator`] specializations.
+/// Streaming byte iterator wrapping one of the four [`FilterIterator`]
+/// specializations (Delete, Normalize, VariantNorm, Romanize).
 ///
 /// Returned by [`TransformStep::filter_bytes`] for steps that support the
-/// fused transform-scan path (Delete, Normalize, VariantNorm, Romanize,
-/// RomanizeChar). `EmojiNorm` and `None` return `Option::None` from
-/// `filter_bytes`.
+/// fused transform-scan path. Five steps support filtering (Delete,
+/// Normalize, VariantNorm, Romanize, RomanizeChar) — RomanizeChar reuses
+/// the `Romanize` filter variant. `EmojiNorm` and `None` return
+/// `Option::None` from `filter_bytes`.
 pub(crate) enum TransformFilter<'a> {
     Delete(FilterIterator<'a, DeleteFilter<'a>>),
     Normalize(FilterIterator<'a, NormalizeFilter<'a>>),

matcher_rs/src/simple_matcher/mod.rs

@@ -144,8 +144,9 @@ pub struct SimpleResult<'a> {
 /// **Pass 1 — Transform and Scan**: The input text is transformed through the
 /// configured [`ProcessType`](crate::ProcessType) pipelines, producing the
 /// distinct text variants needed for this matcher. Each variant is scanned by
-/// the bytewise or charwise engine, selected by SIMD density scan (≤0.67
-/// non-ASCII → bytewise, >0.67 → charwise). Hits update per-rule state;
+/// the bytewise or charwise engine, selected by SIMD density scan (density
+/// ≥ 0.55 → bytewise, < 0.55 → charwise; higher density = more ASCII).
+/// Hits update per-rule state;
 /// simple rules stay on a bitmask fast path, while more complex rules fall
 /// back to a per-rule counter matrix.
 ///

matcher_rs/src/simple_matcher/pattern.rs

@@ -125,8 +125,8 @@ pub(super) fn decode_direct(raw: u32) -> (u8, u8, PatternKind, usize, usize) {
 /// - No operator, or `&` → [`And`](Self::And)
 /// - `~` → [`Not`](Self::Not)
 ///
-/// Single-segment rules without NOT use `SatisfactionMethod::SingleAnd` for the
-/// simplified satisfaction path. The DIRECT bit-packing in `process_match`
+/// Single-segment rules without NOT use [`SatisfactionMethod::Immediate`] for
+/// the simplified satisfaction path. The DIRECT bit-packing in `process_match`
 /// handles these inline without consulting `PatternKind`.
 ///
 /// `repr(u8)` keeps this type small for dense storage in [`PatternEntry`].
@@ -137,7 +137,7 @@ pub(super) enum PatternKind {
     ///
     /// All AND segments in a rule must be satisfied (across any text variant)
     /// before the rule can fire. Single-segment rules also use this variant
-    /// (with `SatisfactionMethod::SingleAnd`).
+    /// (with [`SatisfactionMethod::Immediate`]).
     And = 0,
     /// Negative segment that vetoes the rule when observed.
     ///

matcher_rs/src/simple_matcher/scan.rs

@@ -446,7 +446,7 @@ macro_rules! dispatch {
 /// Both engines are always built from the full pattern set. The charwise
 /// engine gives ~1.6–1.9× throughput over bytewise on CJK-heavy text (3 UTF-8
 /// bytes → 1 charwise transition). Engine selection is density-based at
-/// runtime: bytewise for ≤ [`CHARWISE_DENSITY_THRESHOLD`], charwise above.
+/// runtime: bytewise for ≥ [`CHARWISE_DENSITY_THRESHOLD`], charwise below.
 #[derive(Clone)]
 pub(super) struct ScanPlan {
     engines: Engines,