diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a91cd869..a40ff963 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,10 +32,12 @@ jobs: echo " lock_api -> 0.4.12" echo " quote -> 1.0.40" echo " unicode-ident -> 1.0.13" + echo " zeroize -> 1.8.1" cargo update -p parking_lot --precise 0.12.3 cargo update -p lock_api --precise 0.4.12 cargo update -p quote --precise 1.0.40 cargo update -p unicode-ident --precise 1.0.13 + cargo update -p zeroize --precise 1.8.1 elif [ "${{ matrix.rust }}" = "1.85" ]; then echo "Pinning packages for Rust 1.85:" echo " diesel -> 2.2.12" diff --git a/base/benches/benchmarks.rs b/base/benches/benchmarks.rs index 6023a33c..e0b05dca 100644 --- a/base/benches/benchmarks.rs +++ b/base/benches/benchmarks.rs @@ -14,9 +14,11 @@ macro_rules! uop_case { ($t:ty, $bits:literal, $method:ident, $rng:ident, $group:ident) => { let bits = $bits; let a: $t = $rng.gen_range(0..1 << $bits); - $group.bench_with_input(BenchmarkId::from_parameter(bits), &bits, |bencher, _| { - bencher.iter(|| black_box(a).$method()) - }); + $group.bench_with_input( + BenchmarkId::from_parameter(format!("{}b", bits)), + &bits, + |bencher, _| bencher.iter(|| black_box(a).$method()), + ); }; } @@ -25,9 +27,11 @@ macro_rules! binop_case { let bits = $bits; let a: $t = $rng.gen_range(0..1 << $bits); let b: $t = $rng.gen_range(0..1 << $bits); - $group.bench_with_input(BenchmarkId::from_parameter(bits), &bits, |bencher, _| { - bencher.iter(|| black_box(a).$method(black_box(b))) - }); + $group.bench_with_input( + BenchmarkId::from_parameter(format!("{}b", bits)), + &bits, + |bencher, _| bencher.iter(|| black_box(a).$method(black_box(b))), + ); }; } diff --git a/float/benches/io.rs b/float/benches/io.rs index dbeacf3f..3159ba04 100644 --- a/float/benches/io.rs +++ b/float/benches/io.rs @@ -33,13 +33,17 @@ fn dbig_to_string(criterion: &mut Criterion) { let precision = 10usize.pow(log_prec); let a = random_dbig(precision, &mut rng); let mut out = String::new(); - group.bench_with_input(BenchmarkId::from_parameter(precision), &a, |bencher, ta| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{}", ta).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_prec)), + &a, + |bencher, ta| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{}", ta).unwrap(); + out.len() + }) + }, + ); } group.finish(); @@ -54,9 +58,11 @@ fn dbig_from_str(criterion: &mut Criterion) { let precision = 10usize.pow(log_prec); let a = random_dbig(precision, &mut rng); let s = a.to_string(); - group.bench_with_input(BenchmarkId::from_parameter(precision), &s, |bencher, ts| { - bencher.iter(|| ts.parse::()) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_prec)), + &s, + |bencher, ts| bencher.iter(|| ts.parse::()), + ); } group.finish(); @@ -71,13 +77,17 @@ fn dbig_scientific_fmt(criterion: &mut Criterion) { let precision = 10usize.pow(log_prec); let a = random_dbig(precision, &mut rng); let mut out = String::new(); - group.bench_with_input(BenchmarkId::from_parameter(precision), &a, |bencher, ta| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{:e}", ta).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_prec)), + &a, + |bencher, ta| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{:e}", ta).unwrap(); + out.len() + }) + }, + ); } group.finish(); diff --git a/float/benches/primitive.rs b/float/benches/primitive.rs index 4a8d5afc..c1048220 100644 --- a/float/benches/primitive.rs +++ b/float/benches/primitive.rs @@ -50,7 +50,7 @@ macro_rules! add_binop_benchmark { let a = random_fbig(precision, &mut rng); let b = random_fbig(precision, &mut rng); group.bench_with_input( - BenchmarkId::from_parameter(precision), + BenchmarkId::from_parameter(format!("1e{}", log_prec)), &(a, b), |bencher, (ta, tb)| bencher.iter(|| ta.$method(tb)), ); @@ -70,7 +70,7 @@ macro_rules! add_binop_benchmark { let a = random_dbig(precision, &mut rng); let b = random_dbig(precision, &mut rng); group.bench_with_input( - BenchmarkId::from_parameter(precision), + BenchmarkId::from_parameter(format!("1e{}", log_prec)), &(a, b), |bencher, (ta, tb)| bencher.iter(|| ta.$method(tb)), ); diff --git a/integer/CHANGELOG.md b/integer/CHANGELOG.md index 9d4e8d94..71fb36ab 100644 --- a/integer/CHANGELOG.md +++ b/integer/CHANGELOG.md @@ -6,7 +6,8 @@ - `UBig::from_u64` and `IBig::from_i64`, const on 32-bit and 64-bit targets. ### Improve -- Addition and subtraction carry/borrow propagation now uses `Word` (u64/u32) instead of `bool` throughout the architecture-specific `add_with_carry` and `sub_with_borrow` functions, eliminating `bool`↔Word conversions in the inner loops. Combined with comparison-based overflow detection on the generic code path, this reduces the per-word instruction count from ~9 to ~7 on ARM64. ~36% faster addition and ~17% faster subtraction at 10000-bit operand sizes, bringing dashu roughly even with malachite at this scale. Added `#[inline]` annotations on the hot loop functions to ensure cross-function optimization. +- Basecase (schoolbook) multiplication now uses an dword mult inner kernel (two multiplier words per sweep over the accumulator, mirroring GMP's `mpn_addmul_2` and `mpn_submul_2`), roughly halving accumulator memory traffic. +- Addition and subtraction carry/borrow propagation now uses `Word` (u64/u32) instead of `bool` throughout the architecture-specific `add_with_carry` and `sub_with_borrow` functions, eliminating `bool`↔Word conversions in the inner loops. ### Improve - Logarithm for very large values uses power-sequence decomposition, replacing iterative single-step multiplication. diff --git a/integer/benches/io.rs b/integer/benches/io.rs index 2e254b5a..5cbe885d 100644 --- a/integer/benches/io.rs +++ b/integer/benches/io.rs @@ -26,13 +26,17 @@ fn ubig_to_hex(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_ubig(bits, &mut rng); let mut out = String::with_capacity(bits / 4 + 1); - group.bench_with_input(BenchmarkId::from_parameter(bits), &a, |bencher, ta| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{:x}", &ta).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &a, + |bencher, ta| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{:x}", &ta).unwrap(); + out.len() + }) + }, + ); } group.finish(); @@ -47,13 +51,17 @@ fn ubig_to_dec(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_ubig(bits, &mut rng); let mut out = String::with_capacity(bits / 3 + 1); - group.bench_with_input(BenchmarkId::from_parameter(bits), &a, |bencher, ta| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{}", &ta).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &a, + |bencher, ta| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{}", &ta).unwrap(); + out.len() + }) + }, + ); } group.finish(); @@ -68,9 +76,11 @@ fn ubig_from_hex(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_ubig(bits, &mut rng); let s = a.in_radix(16).to_string(); - group.bench_with_input(BenchmarkId::from_parameter(bits), &s, |bencher, ts| { - bencher.iter(|| UBig::from_str_radix(ts, 16)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &s, + |bencher, ts| bencher.iter(|| UBig::from_str_radix(ts, 16)), + ); } group.finish(); @@ -85,9 +95,11 @@ fn ubig_from_dec(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_ubig(bits, &mut rng); let s = a.in_radix(10).to_string(); - group.bench_with_input(BenchmarkId::from_parameter(bits), &s, |bencher, ts| { - bencher.iter(|| UBig::from_str_radix(ts, 10)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &s, + |bencher, ts| bencher.iter(|| UBig::from_str_radix(ts, 10)), + ); } group.finish(); diff --git a/integer/benches/primitive.rs b/integer/benches/primitive.rs index f8302bcd..2958fc54 100644 --- a/integer/benches/primitive.rs +++ b/integer/benches/primitive.rs @@ -35,7 +35,7 @@ macro_rules! add_binop_benchmark { let a = random_ubig(bits, &mut rng); let b = random_ubig(bits, &mut rng) + &a; // make b > a so that sub won't underflow group.bench_with_input( - BenchmarkId::from_parameter(bits), + BenchmarkId::from_parameter(format!("1e{}", log_bits)), &(a, b), |bencher, (ta, tb)| bencher.iter(|| tb.$method(ta)), ); @@ -48,7 +48,7 @@ macro_rules! add_binop_benchmark { add_binop_benchmark!(ubig_add, add, 6); add_binop_benchmark!(ubig_sub, sub, 6); -add_binop_benchmark!(ubig_mul, mul, 6); +add_binop_benchmark!(ubig_mul, mul, 7); add_binop_benchmark!(ubig_div, div, 6); add_binop_benchmark!(ubig_gcd, gcd, 6); add_binop_benchmark!(ubig_gcd_ext, gcd_ext, 5); @@ -59,9 +59,11 @@ fn ubig_pow(criterion: &mut Criterion) { for log_power in 1..=6 { let p = 10usize.pow(log_power); - group.bench_with_input(BenchmarkId::from_parameter(p), &p, |bencher, p| { - bencher.iter(|| UBig::from(3u8).pow(*p)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_power)), + &p, + |bencher, p| bencher.iter(|| UBig::from(3u8).pow(*p)), + ); } group.finish(); @@ -78,9 +80,11 @@ fn ubig_modulo_mul(criterion: &mut Criterion) { let ring = ConstDivisor::new(m); let a = ring.reduce(random_ubig(bits, &mut rng)); let b = ring.reduce(random_ubig(bits, &mut rng)); - group.bench_with_input(BenchmarkId::from_parameter(bits), &(a, b), |bencher, (ta, tb)| { - bencher.iter(|| ta * tb) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &(a, b), + |bencher, (ta, tb)| bencher.iter(|| ta * tb), + ); } group.finish(); @@ -100,9 +104,11 @@ fn ubig_modulo_pow(criterion: &mut Criterion) { let ring = ConstDivisor::new(m); let a = ring.reduce(random_ubig(2048, &mut rng)); let b = random_ubig(bits, &mut rng); - group.bench_with_input(BenchmarkId::from_parameter(bits), &(a, b), |bencher, (ta, tb)| { - bencher.iter(|| ta.pow(tb)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &(a, b), + |bencher, (ta, tb)| bencher.iter(|| ta.pow(tb)), + ); } group.finish(); @@ -115,9 +121,11 @@ fn ubig_pow_large_base(criterion: &mut Criterion) { let base = UBig::from(12345u32); for log_exp in 1..=6usize { let exp = 10usize.pow(log_exp as u32); - group.bench_with_input(BenchmarkId::from_parameter(exp), &exp, |bencher, exp| { - bencher.iter(|| base.pow(*exp)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_exp)), + &exp, + |bencher, exp| bencher.iter(|| base.pow(*exp)), + ); } group.finish(); @@ -132,9 +140,11 @@ fn ubig_ilog_large(criterion: &mut Criterion) { for log_bits in 1..=6usize { let bits = 10usize.pow(log_bits as u32); let n = random_ubig(bits, &mut rng); - group.bench_with_input(BenchmarkId::from_parameter(bits), &n, |bencher, tn| { - bencher.iter(|| tn.ilog(&base)) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &n, + |bencher, tn| bencher.iter(|| tn.ilog(&base)), + ); } group.finish(); diff --git a/integer/src/mul/simple.rs b/integer/src/mul/simple.rs index 86a02311..8dfa9f96 100644 --- a/integer/src/mul/simple.rs +++ b/integer/src/mul/simple.rs @@ -1,12 +1,12 @@ //! Simple multiplication algorithm. use crate::{ - arch::{ - self, - word::{SignedWord, Word}, - }, + add, + arch::word::{SignedWord, Word}, + math, memory::Memory, mul::{self, helpers}, + primitive::double_word, Sign::{self, *}, }; @@ -80,6 +80,34 @@ fn add_signed_mul_chunk( } } +/// `words[..n] += rhs[..n] * (mult0 + mult1 * B)`, where `n == rhs.len()`. +/// +/// It consumes two multiplier words per sweep over `rhs`, so the accumulator +/// word `words[k]` is loaded and stored once per two multiplier words instead +/// of once per word. This halves the memory traffic on `words` and exposes two +/// independent multiply chains, mirroring GMP's `mpn_addmul_2`. +/// +/// Only `words[..n]` is modified. The two extra high words of the product +/// (the carries out of columns `n` and `n + 1`) are returned as +/// `(carry_lo, carry_hi)`, to be accumulated by the caller at `words[n]` +/// and `words[n + 1]`. +#[inline] +fn add_mul_dword_same_len_in_place( + words: &mut [Word], + rhs: &[Word], + mult0: Word, + mult1: Word, +) -> (Word, Word) { + debug_assert!(words.len() == rhs.len()); + let mut carry_lo: Word = 0; + let mut carry_hi: Word = 0; + for (x, &y) in words.iter_mut().zip(rhs.iter()) { + (*x, carry_lo) = math::mul_add_2carry(y, mult0, *x, carry_lo); + (carry_lo, carry_hi) = math::mul_add_2carry(y, mult1, carry_lo, carry_hi); + } + (carry_lo, carry_hi) +} + /// c += a * b /// Simple method: O(a.len() * b.len()). /// @@ -87,14 +115,59 @@ fn add_signed_mul_chunk( fn add_mul_chunk(c: &mut [Word], a: &[Word], b: &[Word]) -> bool { debug_assert!(a.len() >= b.len() && c.len() == a.len() + b.len()); debug_assert!(a.len() < 2 * CHUNK_LEN); - let mut carry: Word = 0; - for (i, m) in b.iter().enumerate() { - let carry_word = mul::add_mul_word_same_len_in_place(&mut c[i..i + a.len()], *m, a); - let (carry_word, carry_next) = arch::add::add_with_carry(c[i + a.len()], carry_word, carry); - c[i + a.len()] = carry_word; - carry = carry_next; + let n = a.len(); + let mut overflow = false; + let mut i = 0; + + // Consume the multiplier two words at a time via the add_mul_dword_same_len_in_place kernel. + let mut pairs = b.chunks_exact(2); + for pair in &mut pairs { + let (carry_lo, carry_hi) = + add_mul_dword_same_len_in_place(&mut c[i..i + n], a, pair[0], pair[1]); + overflow |= add::add_dword_in_place(&mut c[i + n..], double_word(carry_lo, carry_hi)); + i += 2; + } + + // Handle the leftover odd multiplier word with a plain add_mul_word_same_len_in_place. + if let &[m] = pairs.remainder() { + let carry_word = mul::add_mul_word_same_len_in_place(&mut c[i..i + n], m, a); + overflow |= add::add_word_in_place(&mut c[i + n..], carry_word); + } + + overflow +} + +/// `words[..n] -= rhs[..n] * (mult0 + mult1 * B)`, where `n == rhs.len()`. +/// +/// This function is the analogue of [`add_mul_dword_same_len_in_place`]: the product limbs +/// of `rhs * (mult0 + mult1 * B)` are formed with the same two-multiplier-per-sweep +/// carry recurrence (here with a zero accumulator), and subtracted from `words` in +/// the same pass, so `words[k]` is touched once per two multiplier words. +/// +/// Only `words[..n]` is modified. `borrow` from the low subtraction is folded into +/// `carry_lo`, so the caller only needs [`add::sub_dword_in_place`] on the returned +/// `(carry_lo, carry_hi)`. +#[inline] +fn sub_mul_dword_same_len_in_place( + words: &mut [Word], + rhs: &[Word], + mult0: Word, + mult1: Word, +) -> (Word, Word) { + debug_assert!(words.len() == rhs.len()); + let mut carry_lo: Word = 0; + let mut carry_hi: Word = 0; + let mut borrow: Word = 0; + for (x, &y) in words.iter_mut().zip(rhs.iter()) { + let (prod_limb, carry_a) = math::mul_add_carry(y, mult0, carry_lo); + (carry_lo, carry_hi) = math::mul_add_2carry(y, mult1, carry_a, carry_hi); + let (t, b1) = (*x).overflowing_sub(prod_limb); + let (t, b2) = t.overflowing_sub(borrow); + *x = t; + borrow = Word::from(b1 | b2); } - carry != 0 + let (lo, overflow) = carry_lo.overflowing_add(borrow); + (lo, carry_hi.wrapping_add(overflow as Word)) } /// c -= a * b @@ -104,13 +177,24 @@ fn add_mul_chunk(c: &mut [Word], a: &[Word], b: &[Word]) -> bool { fn sub_mul_chunk(c: &mut [Word], a: &[Word], b: &[Word]) -> bool { debug_assert!(a.len() >= b.len() && c.len() == a.len() + b.len()); debug_assert!(a.len() < 2 * CHUNK_LEN); - let mut borrow: Word = 0; - for (i, m) in b.iter().enumerate() { - let borrow_word = mul::sub_mul_word_same_len_in_place(&mut c[i..i + a.len()], *m, a); - let (borrow_word, borrow_next) = - arch::add::sub_with_borrow(c[i + a.len()], borrow_word, borrow); - c[i + a.len()] = borrow_word; - borrow = borrow_next; + let n = a.len(); + let mut borrow_out = false; + let mut i = 0; + + // Consume the multiplier two words at a time via the submul_2 kernel. + let mut pairs = b.chunks_exact(2); + for pair in &mut pairs { + let (carry_lo, carry_hi) = + sub_mul_dword_same_len_in_place(&mut c[i..i + n], a, pair[0], pair[1]); + borrow_out |= add::sub_dword_in_place(&mut c[i + n..], double_word(carry_lo, carry_hi)); + i += 2; } - borrow != 0 + + // Handle the leftover odd multiplier word with a plain submul_1. + if let &[m] = pairs.remainder() { + let borrow_word = mul::sub_mul_word_same_len_in_place(&mut c[i..i + n], m, a); + borrow_out |= add::sub_word_in_place(&mut c[i + n..], borrow_word); + } + + borrow_out } diff --git a/rational/benches/io.rs b/rational/benches/io.rs index 67117e5e..6c5d7cfe 100644 --- a/rational/benches/io.rs +++ b/rational/benches/io.rs @@ -32,13 +32,17 @@ fn rbig_to_string(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_rbig(bits, &mut rng); let mut out = String::new(); - group.bench_with_input(BenchmarkId::from_parameter(bits), &a, |bencher, ta| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{}", ta).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &a, + |bencher, ta| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{}", ta).unwrap(); + out.len() + }) + }, + ); } group.finish(); @@ -53,9 +57,11 @@ fn rbig_from_str(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_rbig(bits, &mut rng); let s = a.to_string(); - group.bench_with_input(BenchmarkId::from_parameter(bits), &s, |bencher, ts| { - bencher.iter(|| ts.parse::()) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_bits)), + &s, + |bencher, ts| bencher.iter(|| ts.parse::()), + ); } group.finish(); @@ -71,7 +77,7 @@ fn rbig_in_radix_fmt(criterion: &mut Criterion) { let bits = 10usize.pow(log_bits); let a = random_rbig(bits, &mut rng); let mut out = String::new(); - let param = format!("radix={},bits={}", radix, bits); + let param = format!("radix={},1e{}", radix, log_bits); group.bench_with_input( BenchmarkId::from_parameter(param), &(a, radix), @@ -96,15 +102,20 @@ fn rbig_in_expanded_fmt(criterion: &mut Criterion) { // Use moderate-size rationals and vary output precision. let a = random_rbig(1000, &mut rng); - for &prec in &[10, 100, 1000, 10000] { + for log_prec in 1..=4 { + let prec = 10usize.pow(log_prec); let mut out = String::new(); - group.bench_with_input(BenchmarkId::from_parameter(prec), &prec, |bencher, prec| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{:.prec$}", a.in_expanded(10), prec = prec).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_prec)), + &prec, + |bencher, prec| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{:.prec$}", a.in_expanded(10), prec = prec).unwrap(); + out.len() + }) + }, + ); } group.finish(); @@ -116,15 +127,20 @@ fn rbig_in_expanded_scientific(criterion: &mut Criterion) { group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic)); let a = random_rbig(1000, &mut rng); - for &prec in &[10, 100, 1000, 10000] { + for log_prec in 1..=4 { + let prec = 10usize.pow(log_prec); let mut out = String::new(); - group.bench_with_input(BenchmarkId::from_parameter(prec), &prec, |bencher, prec| { - bencher.iter(|| { - out.clear(); - write!(&mut out, "{:.prec$e}", a.in_expanded(10), prec = prec).unwrap(); - out.len() - }) - }); + group.bench_with_input( + BenchmarkId::from_parameter(format!("1e{}", log_prec)), + &prec, + |bencher, prec| { + bencher.iter(|| { + out.clear(); + write!(&mut out, "{:.prec$e}", a.in_expanded(10), prec = prec).unwrap(); + out.len() + }) + }, + ); } group.finish(); diff --git a/rational/benches/primitive.rs b/rational/benches/primitive.rs index fa7a6113..9e149859 100644 --- a/rational/benches/primitive.rs +++ b/rational/benches/primitive.rs @@ -35,7 +35,7 @@ macro_rules! add_binop_benchmark { let a = random_rbig(bits, &mut rng); let b = random_rbig(bits, &mut rng) + &a; // make b > a so that sub won't underflow group.bench_with_input( - BenchmarkId::from_parameter(bits), + BenchmarkId::from_parameter(format!("1e{}", log_bits)), &(a, b), |bencher, (ta, tb)| bencher.iter(|| tb.$method(ta)), );