From 835fe7213ce7e2e0300d336e317b56c69e49166c Mon Sep 17 00:00:00 2001 From: Richard Quinn Date: Mon, 13 Jul 2026 23:34:33 +0200 Subject: [PATCH] test: exhaustive Tier-3 gates for fold/slug/whitespace/dot/preset invariants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #523: where an invariant is *local* (decidable per code point or per small window) but guarded only by a random `\PC*` proptest, random generation merely samples the failure space — exactly how the confusables idempotency bug hid for so long. These invariants are bounded and enumerable, so add deterministic exhaustive Tier-3 gates (all `#[ignore]`, run by the `--lib -- --ignored` step wired in #523; ~11s combined in release). No production code changes; all pass — the confusables fix (#523) plus the already-exhaustive case-fold / transliterate layers leave the higher-level transforms clean. - case_fold (`exhaustive_fold_case_invariants`): every code point. fold_case is composition-free and per-code-point, so single-code-point enumeration is a *complete proof* for all strings — idempotency, no residual ASCII uppercase, no drop, ASCII-in⇒ASCII-out. - slugify (`exhaustive_slug_codomain`): every code point. The slug charset is a per-code-point property, so this completely proves `output_is_ascii` and the charset-membership half of `output_charset` (separator *position* stays with the proptests). - whitespace (`exhaustive_collapse_whitespace`): every code point in a run context + every pattern over {SPACE, NBSP, x} to length 7 — the collapse/trim state machine (uniform whitespace handling). - filename (`exhaustive_collapse_dot_sequences`): every dot pattern over {., a} to length 12 + every non-dot code point preserved — the dot-collapse machine. - presets (`exhaustive_preset_idempotency`): every code point (the #498 class) + every BMP base × combining diacritical U+0300–036F (the #523 class), for canonicalize / sort_key / search_key / catalog_key / ml_normalize. Signed-off-by: Richard Quinn Assisted-by: Claude Code:claude-opus-4-8 --- src/case_fold.rs | 41 +++++++++++++++++++++++++++++++++ src/filename.rs | 46 +++++++++++++++++++++++++++++++++++++ src/presets.rs | 58 +++++++++++++++++++++++++++++++++++++++++++++++ src/slugify.rs | 31 +++++++++++++++++++++++++ src/whitespace.rs | 50 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 226 insertions(+) diff --git a/src/case_fold.rs b/src/case_fold.rs index 3c41879d..df9f787a 100644 --- a/src/case_fold.rs +++ b/src/case_fold.rs @@ -386,6 +386,47 @@ mod tests { } } + /// Tier-3 exhaustive gate for the case-fold invariants over every code point. + /// + /// `fold_case` is a *per-code-point* transform (no composition, no cross-char + /// state), so a string's fold is exactly the concatenation of its chars' folds. + /// That makes single-code-point enumeration a **complete proof** for all inputs — + /// not the sampling the `\PC*` proptests below do. Cheap (~1.1M single chars), and + /// it catches any table entry whose folded form is not itself fully folded + /// (idempotency), reintroduces an ASCII uppercase, or maps to empty. `#[ignore]` + /// (Tier 3); run via the `--lib -- --ignored` step. + #[test] + #[ignore = "exhaustive: every code point through fold_case; run in Tier 3 / pre-release"] + fn exhaustive_fold_case_invariants() { + for cp in 0u32..=0x0010_FFFF { + let Some(ch) = char::from_u32(cp) else { + continue; // surrogates + }; + let s = ch.to_string(); + let once = fold_case_impl(&s); + // idempotent + assert_eq!( + once, + fold_case_impl(&once), + "fold_case not idempotent on U+{cp:04X}" + ); + // never drops the char + assert!(!once.is_empty(), "fold_case emptied U+{cp:04X}"); + // no residual ASCII uppercase + assert!( + !once.chars().any(|c| c.is_ascii_uppercase()), + "fold_case left ASCII uppercase for U+{cp:04X}: {once:?}" + ); + // ASCII in ⇒ ASCII out + if ch.is_ascii() { + assert!( + once.is_ascii(), + "fold_case of ASCII U+{cp:04X} is non-ASCII" + ); + } + } + } + // ── Property-based tests ───────────────────────────────────────── mod proptest_properties { diff --git a/src/filename.rs b/src/filename.rs index 2c77e6b4..a9b38573 100644 --- a/src/filename.rs +++ b/src/filename.rs @@ -492,6 +492,52 @@ mod tests { // Must be valid UTF-8 (implicit — Rust String guarantees this) } + /// Tier-3 exhaustive gate for `collapse_dot_sequences`. + /// + /// The collapse state machine turns purely on `.`-vs-not, so two exhaustive sweeps + /// prove it completely where the `\PC*` proptests sample: (1) every pattern over + /// {`.`, `a`} up to length 12, proving no `".."`, idempotency, and single-dot + /// preservation over every dot arrangement; and (2) every non-`.` code point is + /// preserved verbatim (per-char property). `#[ignore]` (Tier 3); run via + /// `--lib -- --ignored`. + #[test] + #[ignore = "exhaustive: collapse_dot_sequences over every dot pattern + code point; Tier 3"] + fn exhaustive_collapse_dot_sequences() { + // (1) every dot arrangement up to length 12. + let alphabet = ['.', 'a']; + let mut stack = vec![String::new()]; + while let Some(s) = stack.pop() { + let once = collapse_dot_sequences(&s); + assert!(!once.contains(".."), "double dots from {s:?} → {once:?}"); + assert_eq!( + once, + collapse_dot_sequences(&once), + "not idempotent on {s:?}" + ); + if !s.contains("..") { + assert_eq!(once, s, "single-dot input altered: {s:?}"); + } + if s.len() < 12 { + for &a in &alphabet { + let mut n = s.clone(); + n.push(a); + stack.push(n); + } + } + } + // (2) every non-dot code point passes through unchanged. + for cp in 0u32..=0x0010_FFFF { + let Some(c) = char::from_u32(cp) else { + continue; + }; + if c == '.' { + continue; + } + let s = c.to_string(); + assert_eq!(collapse_dot_sequences(&s), s, "dropped non-dot U+{cp:04X}"); + } + } + mod proptest_properties { use super::*; use proptest::prelude::*; diff --git a/src/presets.rs b/src/presets.rs index 023dec83..a79b7b7c 100644 --- a/src/presets.rs +++ b/src/presets.rs @@ -2460,6 +2460,64 @@ mod tests { }) } + /// Tier-3 exhaustive gate for preset idempotency (#416/#467/#498/#523 class). + /// + /// The key presets are fixed points: `canonicalize(canonicalize(x)) == + /// canonicalize(x)`, and likewise for `sort_key`/`search_key`/`catalog_key`/ + /// `ml_normalize`. The `adversarial()` proptests sample `any::()`; this + /// enumerates the two domains where non-idempotency actually lives. (1) Every + /// single code point — the #498 class (a base exposed by NFKD/strip that only + /// resolves on a second pass). (2) Every BMP base × every combining diacritical + /// (U+0300–036F) — the #523 class (a fold/transliterate output that composes with + /// a following mark, or a composition that exposes a new fold); BMP covers every + /// composable Latin/Greek/Cyrillic base, and the astral planes are covered by (1). + /// `#[ignore]` (Tier 3): ~1.1M scalars across 5 presets plus ~7.1M BMP base×mark + /// pairs across 2 presets, each checked twice — on the order of 40M preset calls, + /// a few seconds in release. + #[test] + #[ignore = "exhaustive: preset idempotency over code points + base×mark; Tier 3"] + fn exhaustive_preset_idempotency() { + // Generic (monomorphized) so the tens-of-millions of calls in the inner loops + // pay no vtable dispatch — Tier-3 runtime stays predictable. + fn idem String>(label: &str, f: F, s: &str) { + let once = f(s); + assert_eq!(once, f(&once), "{label} not idempotent on {s:?}"); + } + let cat = |s: &str| catalog_key(s, None, false).unwrap().into_owned(); + let ml = |s: &str| ml_normalize(s, None, "cldr").unwrap().into_owned(); + + // (1) every single code point, across the key presets. + for cp in 0u32..=0x0010_FFFF { + let Some(c) = char::from_u32(cp) else { + continue; + }; + let s = c.to_string(); + idem( + "canonicalize", + |x| canonicalize(x).unwrap().into_owned(), + &s, + ); + idem("sort_key", |x| sort_key(x, None).unwrap().into_owned(), &s); + idem( + "search_key", + |x| search_key(x, None).unwrap().into_owned(), + &s, + ); + idem("catalog_key", cat, &s); + idem("ml_normalize", ml, &s); + } + + // (2) every BMP base × every combining diacritical — the compose/fold class. + let marks: Vec = (0x0300u32..=0x036F).filter_map(char::from_u32).collect(); + for base in (0u32..=0xFFFF).filter_map(char::from_u32) { + for &m in &marks { + let s: String = [base, m].iter().collect(); + idem("catalog_key", cat, &s); + idem("ml_normalize", ml, &s); + } + } + } + proptest! { #![proptest_config(ProptestConfig::with_cases(1000))] diff --git a/src/slugify.rs b/src/slugify.rs index 94bee11a..f76846c6 100644 --- a/src/slugify.rs +++ b/src/slugify.rs @@ -1295,6 +1295,37 @@ mod tests { assert!(err.contains("[unclosed"), "pattern not echoed: {err}"); } + /// Tier-3 exhaustive gate for the slug *codomain* over every code point. + /// + /// A slug's character set is a per-code-point property: every output char comes from + /// transliterating one input char to ASCII and slug-filtering it, so if every single + /// code point slugs to `[a-z0-9-]` (ASCII), so does every string. That makes this a + /// **complete proof** of `slugify_output_is_ascii` and the charset-membership half of + /// `slugify_output_charset`, where the `\PC*` proptests only sample. (The separator- + /// *position* rules — no leading/trailing/`--` — are cross-char and stay with the + /// proptests.) `#[ignore]` (Tier 3); run via `--lib -- --ignored`. + #[test] + #[ignore = "exhaustive: every code point through slugify; run in Tier 3 / pre-release"] + fn exhaustive_slug_codomain() { + let config = default_config(); + for cp in 0u32..=0x0010_FFFF { + let Some(ch) = char::from_u32(cp) else { + continue; // surrogates + }; + let result = slugify_impl(&ch.to_string(), &config); + assert!( + result.is_ascii(), + "non-ASCII slug for U+{cp:04X}: {result:?}" + ); + for c in result.chars() { + assert!( + c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-', + "slug of U+{cp:04X} has out-of-charset {c:?}: {result:?}" + ); + } + } + } + mod proptest_properties { use super::*; use proptest::prelude::*; diff --git a/src/whitespace.rs b/src/whitespace.rs index e3403adf..98ca5ba5 100644 --- a/src/whitespace.rs +++ b/src/whitespace.rs @@ -257,6 +257,56 @@ mod tests { } } + /// Tier-3 exhaustive gate for the whitespace-collapse invariants (#433). + /// + /// `collapse_whitespace` treats every whitespace code point *uniformly* (any run of + /// fold-whitespace / blank-render collapses to one space, the ends are trimmed), so + /// its state machine turns on the whitespace-vs-not *pattern*. Two exhaustive sweeps + /// together pin it where the `\PC*` proptests only sample: (1) every code point in a + /// run context (`x␟c␟c␟y c z`), proving each is classified so runs collapse + /// idempotently with no `" "` and trimmed ends; and (2) every pattern over {two + /// distinct whitespace chars, a non-ws letter} up to length 7, proving the + /// collapse/trim state machine on mixed runs and boundaries. `#[ignore]` (Tier 3); + /// run via `--lib -- --ignored`. + #[test] + #[ignore = "exhaustive: whitespace-collapse over every code point + patterns; Tier 3"] + fn exhaustive_collapse_whitespace() { + let check = |s: &str| { + let once = collapse_whitespace(s); + assert_eq!(once, collapse_whitespace(&once), "not idempotent on {s:?}"); + assert!(!once.contains(" "), "double space from {s:?} → {once:?}"); + if !once.is_empty() { + assert_ne!(once.as_bytes()[0], b' ', "leading space from {s:?}"); + assert_ne!( + *once.as_bytes().last().unwrap(), + b' ', + "trailing space {s:?}" + ); + } + }; + // (1) every code point in a collapsing run context. + for cp in 0u32..=0x0010_FFFF { + let Some(c) = char::from_u32(cp) else { + continue; + }; + check(&format!("x{c}{c}y{c}z")); + } + // (2) every pattern over {SPACE, NBSP, 'x'} up to length 7 (uniform ws handling + // means two distinct ws chars suffice to exercise mixed runs). + let alphabet = [' ', '\u{00A0}', 'x']; + let mut stack = vec![String::new()]; + while let Some(s) = stack.pop() { + check(&s); + if s.chars().count() < 7 { + for &a in &alphabet { + let mut n = s.clone(); + n.push(a); + stack.push(n); + } + } + } + } + mod proptest_properties { use super::*; use proptest::prelude::*;