rscrypto: feat(crc32): implement multi-way PCLMUL/VPCLMUL (2/4/7-way

ILP)

  Add multi-stream PCLMUL kernels for CRC32-IEEE and CRC32C following
  the CRC64 canonical pattern. Uses 128-byte blocks with multiple
  independent folding streams to maximize instruction-level parallelism.

Author: loadingalias

crates/checksum/src/common/clmul.rs

@@ -342,6 +342,9 @@ pub(crate) struct Crc32ClmulConstants {
   pub fold_128b: (u64, u64),
   /// 16B folding coefficient (K_127, K_191) for lane reduction.
   pub fold_16b: (u64, u64),
+  /// Tail fold coefficients (distance 112..16 bytes): (K_{d-1}, K_{d+63}).
+  /// Used to reduce 8 lanes to 1 lane in multi-way kernels.
+  pub tail_fold_16b: [(u64, u64); 7],
   /// 128→96 bit reduction constant (K_95).
   pub k_96: u64,
   /// 96→64 bit reduction constant (K_63).
@@ -492,6 +495,17 @@ impl Crc32ClmulConstants {
       fold_128b: (fold_k_32(normal, 1023), fold_k_32(normal, 1087)),
       // 16B lane: fold by 16 bytes = 128 bits
       fold_16b: (fold_k_32(normal, 127), fold_k_32(normal, 191)),
+      // Tail fold coefficients for reducing 8 lanes to 1 lane.
+      // Each coefficient shifts lane i by (7-i)*16 bytes to align with lane 7.
+      tail_fold_16b: [
+        (fold_k_32(normal, 895), fold_k_32(normal, 959)), // 112 bytes (lane 0)
+        (fold_k_32(normal, 767), fold_k_32(normal, 831)), // 96 bytes (lane 1)
+        (fold_k_32(normal, 639), fold_k_32(normal, 703)), // 80 bytes (lane 2)
+        (fold_k_32(normal, 511), fold_k_32(normal, 575)), // 64 bytes (lane 3)
+        (fold_k_32(normal, 383), fold_k_32(normal, 447)), // 48 bytes (lane 4)
+        (fold_k_32(normal, 255), fold_k_32(normal, 319)), // 32 bytes (lane 5)
+        (fold_k_32(normal, 127), fold_k_32(normal, 191)), // 16 bytes (lane 6)
+      ],
       // 128→96 bits: K_95
       k_96: fold_k_32(normal, 95),
       // 96→64 bits: K_63

crates/checksum/src/crc32/x86_64.rs

@@ -905,16 +905,24 @@ unsafe fn fold_block_128_32(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) {
 #[inline]
 #[target_feature(enable = "sse2", enable = "pclmulqdq")]
 unsafe fn fold_tail_32(x: [Simd; 8], consts: &Crc32ClmulConstants) -> __m128i {
-  let c = Simd::new(consts.fold_16b.0, consts.fold_16b.1);
+  // Tail reduction (8×16B → 1×16B), using per-lane coefficients.
+  // Each lane needs to be shifted by a different amount to align with lane 7.
+  let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); // 112 bytes
+  let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); // 96 bytes
+  let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); // 80 bytes
+  let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); // 64 bytes
+  let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); // 48 bytes
+  let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); // 32 bytes
+  let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); // 16 bytes
 
   let mut acc = x[7];
-  acc ^= x[0].fold_16(c);
-  acc ^= x[1].fold_16(c);
-  acc ^= x[2].fold_16(c);
-  acc ^= x[3].fold_16(c);
-  acc ^= x[4].fold_16(c);
-  acc ^= x[5].fold_16(c);
-  acc ^= x[6].fold_16(c);
+  acc ^= x[0].fold_16(c0);
+  acc ^= x[1].fold_16(c1);
+  acc ^= x[2].fold_16(c2);
+  acc ^= x[3].fold_16(c3);
+  acc ^= x[4].fold_16(c4);
+  acc ^= x[5].fold_16(c5);
+  acc ^= x[6].fold_16(c6);
 
   acc.0
 }