diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58c210691..2a243f829 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,12 +3,12 @@ env: # version like 1.70. Note that we only specify MAJOR.MINOR and not PATCH so that bugfixes still # come automatically. If the version specified here is no longer the latest stable version, # then please feel free to submit a PR that adjusts it along with the potential clippy fixes. - RUST_STABLE_VER: "1.88" # In quotes because otherwise (e.g.) 1.70 would be interpreted as 1.7 + RUST_STABLE_VER: "1.89" # In quotes because otherwise (e.g.) 1.70 would be interpreted as 1.7 # The purpose of checking with the minimum supported Rust toolchain is to detect its staleness. # If the compilation fails, then the version specified here needs to be bumped up to reality. # Be sure to also update the rust-version property in the workspace Cargo.toml file, # plus all the README.md files of the affected packages. - RUST_MIN_VER: "1.88" + RUST_MIN_VER: "1.89" # List of packages that will be checked with the minimum supported Rust version. # This should be limited to packages that are intended for publishing. RUST_MIN_VER_PKGS: "-p fearless_simd" @@ -275,8 +275,7 @@ jobs: - name: run tests on CPU with AVX-512 # Github Actions doesn't give us AVX-512 so this is the only way to exercise AVX-512 codepaths on CI. # -icl stands for Ice Lake. Technically Skylake added AVX-512 first, but it's mostly useless there due to - # downclocking. When we do eventually add explicit AVX-512 support, we'll likely target the Ice Lake feature - # level. + # downclocking, so our explicit AVX-512 level targets Ice Lake. run: ${SDE_PKG}-lin/sde64 -icl -- cargo test $CARGO_TEST_ARGS test-aarch64-qemu: diff --git a/CHANGELOG.md b/CHANGELOG.md index c37e864db..cb0aab5b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,15 @@ You can find its changes [documented below](#050-2026-06-18). ## [Unreleased] -This release has an [MSRV][] of 1.88. +This release has an [MSRV][] of 1.89. + +### Added + +- Added Ice Lake-class AVX-512 support with a generated `Avx512` level and 512-bit native-width vector types. + +### Changed + +- The MSRV is now Rust 1.89. ## [0.5.0][] (2026-06-18) diff --git a/Cargo.toml b/Cargo.toml index 0158a30a3..398c2c514 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ license = "Apache-2.0 OR MIT" repository = "https://github.com/linebender/fearless_simd" # Keep in sync with RUST_MIN_VER in .github/workflows/ci.yml, with the relevant README.md files # and with the MSRV in the `Unreleased` section of CHANGELOG.md. -rust-version = "1.88" +rust-version = "1.89" [workspace.lints] @@ -44,10 +44,11 @@ clippy.collection_is_never_read = "warn" clippy.default_trait_access = "warn" clippy.dbg_macro = "warn" clippy.debug_assert_with_mut_call = "warn" +clippy.disallowed_methods = "deny" clippy.doc_markdown = "warn" clippy.fn_to_numeric_cast_any = "warn" clippy.infinite_loop = "warn" -clippy.large_stack_arrays = "warn" +clippy.large_stack_arrays = "allow" # appears to be buggy as of 1.93, fixed in 1.95. TODO: re-enable clippy.mismatching_type_param_order = "warn" clippy.missing_assert_message = "warn" clippy.missing_fields_in_debug = "warn" diff --git a/README.md b/README.md index 94c789849..05667e4fb 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ It benefited from conversations with Luca Versari, though he is not responsible ## Minimum supported Rust Version (MSRV) -This version of Fearless SIMD has been verified to compile with **Rust 1.88** and later. +This version of Fearless SIMD has been verified to compile with **Rust 1.89** and later. Future versions of Fearless SIMD might increase the Rust version requirement. It will not be treated as a breaking change and as such can even happen with small patch releases. diff --git a/check_targets.sh b/check_targets.sh index 90b09fb7f..98e61c22c 100644 --- a/check_targets.sh +++ b/check_targets.sh @@ -15,6 +15,8 @@ cargo check -p fearless_simd --target aarch64-linux-android --features force_su cargo check -p fearless_simd --target aarch64-linux-android # x86_64, at all supported static SIMD levels. +RUSTFLAGS=-Ctarget-cpu=icelake-server cargo check -p fearless_simd --target x86_64-unknown-linux-gnu +RUSTFLAGS=-Ctarget-cpu=icelake-server cargo check -p fearless_simd --target x86_64-unknown-linux-gnu --features force_support_fallback RUSTFLAGS=-Ctarget-feature=+avx2,+fma cargo check -p fearless_simd --target x86_64-unknown-linux-gnu RUSTFLAGS=-Ctarget-feature=+avx2,+fma cargo check -p fearless_simd --target x86_64-unknown-linux-gnu --features force_support_fallback RUSTFLAGS=-Ctarget-feature=+sse4.2 cargo check -p fearless_simd --target x86_64-unknown-linux-gnu diff --git a/fearless_simd/README.md b/fearless_simd/README.md index 17bb1c3aa..ffba6b0fd 100644 --- a/fearless_simd/README.md +++ b/fearless_simd/README.md @@ -146,7 +146,7 @@ case. There's also Q&A on [Zulip](https://xi.zulipchat.com/#narrow/channel/51423 ## Instruction set support -- x86/x86-64: [v2](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (SSE4.2), [v3](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (AVX2) +- x86/x86-64: [v2](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (SSE4.2), [v3](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (AVX2), [Ice Lake](https://en.wikipedia.org/wiki/AVX-512#CPUs_with_AVX-512) (AVX-512, avoiding early slow implementations) - Aarch64: Baseline [NEON](https://en.wikipedia.org/wiki/Arm_architecture_family#Advanced_SIMD_(Neon)) - WebAssembly: [128-bit packed SIMD](https://github.com/WebAssembly/spec/blob/main/proposals/simd/SIMD.md), [relaxed SIMD](https://github.com/WebAssembly/relaxed-simd/blob/main/proposals/relaxed-simd/Overview.md) @@ -204,7 +204,7 @@ At least one of `std` and `libm` is required; `std` overrides `libm`. ## Minimum supported Rust Version (MSRV) -This version of Fearless SIMD has been verified to compile with **Rust 1.88** and later. +This version of Fearless SIMD has been verified to compile with **Rust 1.89** and later. Future versions of Fearless SIMD might increase the Rust version requirement. It will not be treated as a breaking change and as such can even happen with small patch releases. diff --git a/fearless_simd/src/generated.rs b/fearless_simd/src/generated.rs index 381ffadf8..e5d417ece 100644 --- a/fearless_simd/src/generated.rs +++ b/fearless_simd/src/generated.rs @@ -41,6 +41,8 @@ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod avx2; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx512; mod fallback; #[cfg(target_arch = "aarch64")] mod neon; @@ -54,6 +56,8 @@ mod wasm; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub use avx2::*; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub use avx512::*; pub use fallback::*; #[cfg(target_arch = "aarch64")] pub use neon::*; diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 7018c4e42..d89d9f0f2 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -6,9 +6,9 @@ use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; #[cfg(target_arch = "x86")] use core::arch::x86::*; @@ -45,6 +45,8 @@ impl ArchTypes for Avx2 { type u32x4 = crate::support::Aligned128<__m128i>; type mask32x4 = crate::support::Aligned128<__m128i>; type f64x2 = crate::support::Aligned128<__m128d>; + type i64x2 = crate::support::Aligned128<__m128i>; + type u64x2 = crate::support::Aligned128<__m128i>; type mask64x2 = crate::support::Aligned128<__m128i>; type f32x8 = crate::support::Aligned256<__m256>; type i8x32 = crate::support::Aligned256<__m256i>; @@ -57,6 +59,8 @@ impl ArchTypes for Avx2 { type u32x8 = crate::support::Aligned256<__m256i>; type mask32x8 = crate::support::Aligned256<__m256i>; type f64x4 = crate::support::Aligned256<__m256d>; + type i64x4 = crate::support::Aligned256<__m256i>; + type u64x4 = crate::support::Aligned256<__m256i>; type mask64x4 = crate::support::Aligned256<__m256i>; type f32x16 = crate::support::Aligned512<[__m256; 2usize]>; type i8x64 = crate::support::Aligned512<[__m256i; 2usize]>; @@ -69,6 +73,8 @@ impl ArchTypes for Avx2 { type u32x16 = crate::support::Aligned512<[__m256i; 2usize]>; type mask32x16 = crate::support::Aligned512<[__m256i; 2usize]>; type f64x8 = crate::support::Aligned512<[__m256d; 2usize]>; + type i64x8 = crate::support::Aligned512<[__m256i; 2usize]>; + type u64x8 = crate::support::Aligned512<[__m256i; 2usize]>; type mask64x8 = crate::support::Aligned512<[__m256i; 2usize]>; } impl Simd for Avx2 { @@ -80,6 +86,8 @@ impl Simd for Avx2 { type i16s = i16x16; type u32s = u32x8; type i32s = i32x8; + type u64s = u64x4; + type i64s = i64x4; type mask8s = mask8x32; type mask16s = mask16x16; type mask32s = mask32x8; @@ -785,7 +793,27 @@ impl Simd for Avx2 { } #[inline(always)] fn shlv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i8x16(self, a: i8x16, shift: u32) -> i8x16 { @@ -805,7 +833,27 @@ impl Simd for Avx2 { } #[inline(always)] fn shrv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { @@ -1153,7 +1201,27 @@ impl Simd for Avx2 { } #[inline(always)] fn shlv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u8x16(self, a: u8x16, shift: u32) -> u8x16 { @@ -1173,7 +1241,27 @@ impl Simd for Avx2 { } #[inline(always)] fn shrv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { @@ -1403,6 +1491,17 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask8x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x16(lanes); + } + #[inline(always)] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { crate::kernel!( #[inline(always)] @@ -1670,7 +1769,19 @@ impl Simd for Avx2 { } #[inline(always)] fn shlv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i16x8(self, a: i16x8, shift: u32) -> i16x8 { @@ -1684,7 +1795,19 @@ impl Simd for Avx2 { } #[inline(always)] fn shrv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { @@ -2019,7 +2142,19 @@ impl Simd for Avx2 { } #[inline(always)] fn shlv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u16x8(self, a: u16x8, shift: u32) -> u16x8 { @@ -2033,7 +2168,19 @@ impl Simd for Avx2 { } #[inline(always)] fn shrv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { @@ -2261,6 +2408,17 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask16x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x8(lanes); + } + #[inline(always)] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3155,6 +3313,17 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask32x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x4(lanes); + } + #[inline(always)] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { crate::kernel!( #[inline(always)] @@ -3678,1322 +3847,1692 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn splat_mask64x2(self, val: bool) -> mask64x2 { + fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: bool) -> mask64x2 { - let val: i64 = if val { !0 } else { 0 }; + fn kernel(token: Avx2, val: i64) -> i64x2 { _mm_set1_epi64x(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { - mask64x2 { + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0) } #[inline(always)] - fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [i64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [i64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i64x2(b).val.0, + self.cvt_to_bytes_i64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask64x2 { - { - let bit_lanes = _mm_set1_epi64x(bits.cast_signed()); - let bit_mask = _mm_set_epi64x(2, 1); - _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask) - } - .simd_into(token) + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, bits) + kernel(self, a, b) } #[inline(always)] - fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2) -> u64 { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64 + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { _mm_and_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { _mm_or_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { _mm_xor_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask64x2(self, a: mask64x2) -> mask64x2 { - self.xor_mask64x2(a, self.splat_mask64x2(true)) + fn not_i64x2(self, a: i64x2) -> i64x2 { + a ^ !0 } #[inline(always)] - fn select_mask64x2( - self, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { - _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2, shift: u32) -> i64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, shift) } #[inline(always)] - fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { - _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_sllv_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask64x2(self, a: mask64x2) -> bool { + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let result: [i64; 2usize] = [ + core::ops::Shr::shr(a[0usize], shift), + core::ops::Shr::shr(a[1usize], shift), + ]; + result.simd_into(self) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0 + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> mask64x2 { + _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_true_mask64x2(self, a: mask64x2) -> bool { + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11 + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn any_false_mask64x2(self, a: mask64x2) -> bool { + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11 + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_false_mask64x2(self, a: mask64x2) -> bool { + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0 + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x4 { - _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn splat_f32x8(self, val: f32) -> f32x8 { + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: f32) -> f32x8 { - _mm256_set1_ps(val).simd_into(token) + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { - f32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { - f32x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + kernel(self, a, b) } #[inline(always)] - fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { - crate::transmute::checked_transmute_copy::<__m256, [f32; 8usize]>(&a.val.0) + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { - crate::transmute::checked_cast_ref::<__m256, [f32; 8usize]>(&a.val.0) + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { - crate::transmute::checked_cast_mut::<__m256, [f32; 8usize]>(&mut a.val.0) + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) } #[inline(always)] - fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { - f32x8 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b)) } #[inline(always)] - fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask64x2, + b: i64x2, + c: i64x2, + ) -> i64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) } #[inline(always)] - fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - if SHIFT >= 8usize { - return b; - } - let result = cross_block_alignr_256x1( - self, - self.cvt_to_bytes_f32x8(b).val.0, - self.cvt_to_bytes_f32x8(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_f32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn slide_within_blocks_f32x8( - self, - a: f32x8, - b: f32x8, - ) -> f32x8 { - if SHIFT >= 4usize { - return b; - } - let result = dyn_alignr_256( - self, - self.cvt_to_bytes_f32x8(b).val.0, - self.cvt_to_bytes_f32x8(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_f32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn abs_f32x8(self, a: f32x8) -> f32x8 { + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2, b: i64x2) -> i64x4 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn neg_f32x8(self, a: f32x8) -> f32x8 { + fn neg_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token) + fn kernel(token: Avx2, a: i64x2) -> i64x2 { + _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn sqrt_f32x8(self, a: f32x8) -> f32x8 { + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_sqrt_ps(a.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2) -> u8x16 { + __m128i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_rcp_ps(a.into()).simd_into(token) + fn kernel(token: Avx2, a: i64x2) -> u32x4 { + __m128i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_add_ps(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, val: u64) -> u64x2 { + _mm_set1_epi64x(val.cast_signed()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, val) } #[inline(always)] - fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_sub_ps(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [u64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [u64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_u64x2(b).val.0, + self.cvt_to_bytes_u64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u64x2( + self, + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) + } + #[inline(always)] + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_mul_ps(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_div_ps(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let mask = _mm256_set1_ps(-0.0); - _mm256_or_ps( - _mm256_and_ps(mask, b.into()), - _mm256_andnot_ps(mask, a.into()), - ) - .simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { - _mm256_castps_si256(_mm256_cmp_ps::<0i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { - _mm256_castps_si256(_mm256_cmp_ps::<17i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + fn not_u64x2(self, a: u64x2) -> u64x2 { + a ^ !0 + } + #[inline(always)] + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { - _mm256_castps_si256(_mm256_cmp_ps::<18i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u64x2, shift: u32) -> u64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, shift) } #[inline(always)] - fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { - _mm256_castps_si256(_mm256_cmp_ps::<29i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_sllv_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { - _mm256_castps_si256(_mm256_cmp_ps::<30i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u64x2, shift: u32) -> u64x2 { + _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, shift) } #[inline(always)] - fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let lo = _mm256_unpacklo_ps(a.into(), b.into()); - let hi = _mm256_unpackhi_ps(a.into(), b.into()); - _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_srlv_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let lo = _mm256_unpacklo_ps(a.into(), b.into()); - let hi = _mm256_unpackhi_ps(a.into(), b.into()); - _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> mask64x2 { + _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let t1 = - _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - let t2 = - _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let t1 = - _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - let t2 = - _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let lo = _mm256_unpacklo_ps(a.into(), b.into()); - let hi = _mm256_unpackhi_ps(a.into(), b.into()); - ( - _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token), - _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token), - ) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let t1 = - _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - let t2 = - _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); - ( - _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token), - _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token), - ) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_max_ps(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - _mm256_min_ps(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let intermediate = _mm256_max_ps(a.into(), b.into()); - let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into()); - _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { - let intermediate = _mm256_min_ps(a.into(), b.into()); - let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into()); - _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) + } + #[inline(always)] + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b)) + } + #[inline(always)] + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - _mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token) + fn kernel( + token: Avx2, + a: mask64x2, + b: u64x2, + c: u64x2, + ) -> u64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) + } + #[inline(always)] + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) + } + #[inline(always)] + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - _mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token) + fn kernel(token: Avx2, a: u64x2, b: u64x2) -> u64x4 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, b) } #[inline(always)] - fn floor_f32x8(self, a: f32x8) -> f32x8 { + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, a: u64x2) -> u8x16 { + __m128i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn ceil_f32x8(self, a: f32x8) -> f32x8 { + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, a: u64x2) -> u32x4 { + __m128i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { + fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, val: bool) -> mask64x2 { + let val: i64 = if val { !0 } else { 0 }; + _mm_set1_epi64x(val).simd_into(token) } ); - kernel(self, a) + kernel(self, val) } #[inline(always)] - fn fract_f32x8(self, a: f32x8) -> f32x8 { - a - self.trunc_f32x8(a) + fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { + mask64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn trunc_f32x8(self, a: f32x8) -> f32x8 { + fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f32x8 { - _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, bits: u64) -> mask64x2 { + { + let bit_lanes = _mm_set1_epi64x(bits.cast_signed()); + let bit_mask = _mm_set_epi64x(2, 1); + _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask) + } + .simd_into(token) } ); - kernel(self, a) + kernel(self, bits) } #[inline(always)] - fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: mask32x8, - b: f32x8, - c: f32x8, - ) -> f32x8 { - _mm256_blendv_ps(c.into(), b.into(), _mm256_castsi256_ps(a.into())).simd_into(token) + fn kernel(token: Avx2, a: mask64x2) -> u64 { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64 } ); - kernel(self, a, b, c) + kernel(self, a) } #[inline(always)] - fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { - f32x16 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let mut lanes = self.as_array_mask64x2(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x2(lanes); } #[inline(always)] - fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> (f32x4, f32x4) { - ( - _mm256_extractf128_ps::<0>(a.into()).simd_into(token), - _mm256_extractf128_ps::<1>(a.into()).simd_into(token), - ) + fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> f64x4 { - _mm256_castps_pd(a.into()).simd_into(token) + fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> i32x8 { - _mm256_castps_si256(a.into()).simd_into(token) + fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + self.xor_mask64x2(a, self.splat_mask64x2(true)) + } + #[inline(always)] + fn select_mask64x2( + self, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> u8x32 { - _mm256_castps_si256(a.into()).simd_into(token) + fn kernel( + token: Avx2, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b, c) } #[inline(always)] - fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> u32x8 { - _mm256_castps_si256(a.into()).simd_into(token) + fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + fn any_true_mask64x2(self, a: mask64x2) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> u32x8 { - let mut converted = _mm256_cvttps_epi32(a.into()); - let in_range = _mm256_cmp_ps::<17i32>(a.into(), _mm256_set1_ps(2147483648.0)); - let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; - if !all_in_range { - let excess = _mm256_sub_ps(a.into(), _mm256_set1_ps(2147483648.0)); - let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess)); - converted = _mm256_add_epi32(converted, excess_converted); - } - converted.simd_into(token) + fn kernel(token: Avx2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0 } ); kernel(self, a) } #[inline(always)] - fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + fn all_true_mask64x2(self, a: mask64x2) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> u32x8 { - let a = _mm256_max_ps(a.into(), _mm256_setzero_ps()); - let mut converted = _mm256_cvttps_epi32(a); - let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0)); - let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; - if !all_in_range { - let exceeds_unsigned_range = _mm256_castps_si256(_mm256_cmp_ps::<17i32>( - _mm256_set1_ps(4294967040.0), - a, - )); - let excess = _mm256_sub_ps(a, _mm256_set1_ps(2147483648.0)); - let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess)); - converted = _mm256_add_epi32(converted, excess_converted); - converted = _mm256_blendv_epi8( - converted, - _mm256_set1_epi32(u32::MAX.cast_signed()), - exceeds_unsigned_range, - ); - } - converted.simd_into(token) + fn kernel(token: Avx2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11 } ); kernel(self, a) } #[inline(always)] - fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + fn any_false_mask64x2(self, a: mask64x2) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> i32x8 { - _mm256_cvttps_epi32(a.into()).simd_into(token) + fn kernel(token: Avx2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11 } ); kernel(self, a) } #[inline(always)] - fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + fn all_false_mask64x2(self, a: mask64x2) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f32x8) -> i32x8 { - let a = a.into(); - let mut converted = _mm256_cvttps_epi32(a); - let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0)); - let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; - if !all_in_range { - converted = _mm256_blendv_epi8( - _mm256_set1_epi32(i32::MAX), - converted, - _mm256_castps_si256(in_range), - ); - let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a)); - converted = _mm256_and_si256(converted, is_not_nan); - } - converted.simd_into(token) + fn kernel(token: Avx2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0 } ); kernel(self, a) } #[inline(always)] - fn splat_i8x32(self, val: i8) -> i8x32 { + fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: i8) -> i8x32 { - _mm256_set1_epi8(val).simd_into(token) + fn kernel(token: Avx2, a: mask64x2, b: mask64x2) -> mask64x4 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn splat_f32x8(self, val: f32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: f32) -> f32x8 { + _mm256_set1_ps(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0) + fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256, [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { - crate::transmute::checked_cast_ref::<__m256i, [i8; 32usize]>(&a.val.0) + fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { + crate::transmute::checked_cast_ref::<__m256, [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { - crate::transmute::checked_cast_mut::<__m256i, [i8; 32usize]>(&mut a.val.0) + fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { + crate::transmute::checked_cast_mut::<__m256, [f32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { - i8x32 { + fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - if SHIFT >= 32usize { + fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_i8x32(b).val.0, - self.cvt_to_bytes_i8x32(a).val.0, - SHIFT, + self.cvt_to_bytes_f32x8(b).val.0, + self.cvt_to_bytes_f32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i8x32(u8x32 { + self.cvt_from_bytes_f32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x32( + fn slide_within_blocks_f32x8( self, - a: i8x32, - b: i8x32, - ) -> i8x32 { - if SHIFT >= 16usize { + a: f32x8, + b: f32x8, + ) -> f32x8 { + if SHIFT >= 4usize { return b; } let result = dyn_alignr_256( self, - self.cvt_to_bytes_i8x32(b).val.0, - self.cvt_to_bytes_i8x32(a).val.0, - SHIFT, + self.cvt_to_bytes_f32x8(b).val.0, + self.cvt_to_bytes_f32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i8x32(u8x32 { + self.cvt_from_bytes_f32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_add_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn neg_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn sqrt_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - let dst_even = _mm256_mullo_epi16(a.into(), b.into()); - let dst_odd = _mm256_mullo_epi16( - _mm256_srli_epi16::<8>(a.into()), - _mm256_srli_epi16::<8>(b.into()), - ); - _mm256_or_si256( - _mm256_slli_epi16(dst_odd, 8), - _mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)), - ) - .simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_sqrt_ps(a.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_rcp_ps(a.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_add_ps(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_sub_ps(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_i8x32(self, a: i8x32) -> i8x32 { - a ^ !0 - } - #[inline(always)] - fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, shift: u32) -> i8x32 { - let val = a.into(); - let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); - let lo_16 = - _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); - let hi_16 = - _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); - let lo_shifted = _mm256_sll_epi16(lo_16, shift_count); - let hi_shifted = _mm256_sll_epi16(hi_16, shift_count); - _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_mul_ps(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, shift) - } - #[inline(always)] - fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + kernel(self, a, b) } #[inline(always)] - fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, shift: u32) -> i8x32 { - let val = a.into(); - let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); - let lo_16 = - _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); - let hi_16 = - _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); - let lo_shifted = _mm256_sra_epi16(lo_16, shift_count); - let hi_shifted = _mm256_sra_epi16(hi_16, shift_count); - _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_div_ps(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, shift) + kernel(self, a, b) } #[inline(always)] - fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let mask = _mm256_set1_ps(-0.0); + _mm256_or_ps( + _mm256_and_ps(mask, b.into()), + _mm256_andnot_ps(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { - _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { + _mm256_castps_si256(_mm256_cmp_ps::<0i32>(a.into(), b.into())).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { - _mm256_cmpgt_epi8(b.into(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { + _mm256_castps_si256(_mm256_cmp_ps::<17i32>(a.into(), b.into())).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { - _mm256_cmpeq_epi8(_mm256_min_epi8(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { + _mm256_castps_si256(_mm256_cmp_ps::<18i32>(a.into(), b.into())).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { - _mm256_cmpeq_epi8(_mm256_max_epi8(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { + _mm256_castps_si256(_mm256_cmp_ps::<29i32>(a.into(), b.into())).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { - _mm256_cmpgt_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> mask32x8 { + _mm256_castps_si256(_mm256_cmp_ps::<30i32>(a.into(), b.into())).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - let lo = _mm256_unpacklo_epi8(a.into(), b.into()); - let hi = _mm256_unpackhi_epi8(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let lo = _mm256_unpacklo_ps(a.into(), b.into()); + let hi = _mm256_unpackhi_ps(a.into(), b.into()); + _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - let lo = _mm256_unpacklo_epi8(a.into(), b.into()); - let hi = _mm256_unpackhi_epi8(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let lo = _mm256_unpacklo_ps(a.into(), b.into()); + let hi = _mm256_unpackhi_ps(a.into(), b.into()); + _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - a.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - b.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let t1 = + _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); + let t2 = + _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); + _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - a.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - b.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let t1 = + _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); + let t2 = + _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); + _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let lo = _mm256_unpacklo_epi8(a.into(), b.into()); - let hi = _mm256_unpackhi_epi8(a.into(), b.into()); + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let lo = _mm256_unpacklo_ps(a.into(), b.into()); + let hi = _mm256_unpackhi_ps(a.into(), b.into()); ( - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), + _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token), ) } ); kernel(self, a, b) } #[inline(always)] - fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - a.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - b.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let t1 = + _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); + let t2 = + _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); ( - _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), - _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), + _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token), ) } ); kernel(self, a, b) } #[inline(always)] - fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { + fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: mask8x32, - b: i8x32, - c: i8x32, - ) -> i8x32 { - _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_max_ps(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, b) } #[inline(always)] - fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_min_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + _mm256_min_ps(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { - _mm256_max_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let intermediate = _mm256_max_ps(a.into(), b.into()); + let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into()); + _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { - i8x64 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } + fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, b: f32x8) -> f32x8 { + let intermediate = _mm256_min_ps(a.into(), b.into()); + let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into()); + _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { + fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32) -> (i8x16, i8x16) { - ( - _mm256_extracti128_si256::<0>(a.into()).simd_into(token), - _mm256_extracti128_si256::<1>(a.into()).simd_into(token), - ) + fn kernel(token: Avx2, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + _mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b, c) } #[inline(always)] - fn neg_i8x32(self, a: i8x32) -> i8x32 { + fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32) -> i8x32 { - _mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + _mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b, c) } #[inline(always)] - fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { + fn floor_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32) -> u8x32 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { + fn ceil_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i8x32) -> u32x8 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn splat_u8x32(self, val: u8) -> u8x32 { + fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: u8) -> u8x32 { - _mm256_set1_epi8(val.cast_signed()).simd_into(token) + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + kernel(self, a) } #[inline(always)] - fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { - crate::transmute::checked_transmute_copy::<__m256i, [u8; 32usize]>(&a.val.0) + fn fract_f32x8(self, a: f32x8) -> f32x8 { + a - self.trunc_f32x8(a) } #[inline(always)] - fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { - crate::transmute::checked_cast_ref::<__m256i, [u8; 32usize]>(&a.val.0) + fn trunc_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { - crate::transmute::checked_cast_mut::<__m256i, [u8; 32usize]>(&mut a.val.0) + fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask32x8, + b: f32x8, + c: f32x8, + ) -> f32x8 { + _mm256_blendv_ps(c.into(), b.into(), _mm256_castsi256_ps(a.into())).simd_into(token) + } + ); + kernel(self, a, b, c) } #[inline(always)] - fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { + f32x16 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> (f32x4, f32x4) { + ( + _mm256_extractf128_ps::<0>(a.into()).simd_into(token), + _mm256_extractf128_ps::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> f64x4 { + _mm256_castps_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> i32x8 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> u8x32 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> u32x8 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> u32x8 { + let mut converted = _mm256_cvttps_epi32(a.into()); + let in_range = _mm256_cmp_ps::<17i32>(a.into(), _mm256_set1_ps(2147483648.0)); + let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; + if !all_in_range { + let excess = _mm256_sub_ps(a.into(), _mm256_set1_ps(2147483648.0)); + let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess)); + converted = _mm256_add_epi32(converted, excess_converted); + } + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> u32x8 { + let a = _mm256_max_ps(a.into(), _mm256_setzero_ps()); + let mut converted = _mm256_cvttps_epi32(a); + let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0)); + let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; + if !all_in_range { + let exceeds_unsigned_range = _mm256_castps_si256(_mm256_cmp_ps::<17i32>( + _mm256_set1_ps(4294967040.0), + a, + )); + let excess = _mm256_sub_ps(a, _mm256_set1_ps(2147483648.0)); + let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess)); + converted = _mm256_add_epi32(converted, excess_converted); + converted = _mm256_blendv_epi8( + converted, + _mm256_set1_epi32(u32::MAX.cast_signed()), + exceeds_unsigned_range, + ); + } + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> i32x8 { + _mm256_cvttps_epi32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8) -> i32x8 { + let a = a.into(); + let mut converted = _mm256_cvttps_epi32(a); + let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0)); + let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; + if !all_in_range { + converted = _mm256_blendv_epi8( + _mm256_set1_epi32(i32::MAX), + converted, + _mm256_castps_si256(in_range), + ); + let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a)); + converted = _mm256_and_si256(converted, is_not_nan); + } + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i8x32(self, val: i8) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: i8) -> i8x32 { + _mm256_set1_epi8(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { + crate::transmute::checked_cast_ref::<__m256i, [i8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { + crate::transmute::checked_cast_mut::<__m256i, [i8; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { - u8x32 { + fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { if SHIFT >= 32usize { return b; } let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_u8x32(b).val.0, - self.cvt_to_bytes_u8x32(a).val.0, + self.cvt_to_bytes_i8x32(b).val.0, + self.cvt_to_bytes_i8x32(a).val.0, SHIFT, ); - self.cvt_from_bytes_u8x32(u8x32 { + self.cvt_from_bytes_i8x32(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x32( + fn slide_within_blocks_i8x32( self, - a: u8x32, - b: u8x32, - ) -> u8x32 { + a: i8x32, + b: i8x32, + ) -> i8x32 { if SHIFT >= 16usize { return b; } let result = dyn_alignr_256( self, - self.cvt_to_bytes_u8x32(b).val.0, - self.cvt_to_bytes_u8x32(a).val.0, + self.cvt_to_bytes_i8x32(b).val.0, + self.cvt_to_bytes_i8x32(a).val.0, SHIFT, ); - self.cvt_from_bytes_u8x32(u8x32 { + self.cvt_from_bytes_i8x32(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { _mm256_add_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { _mm256_sub_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { let dst_even = _mm256_mullo_epi16(a.into(), b.into()); let dst_odd = _mm256_mullo_epi16( _mm256_srli_epi16::<8>(a.into()), @@ -5009,140 +5548,210 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_u8x32(self, a: u8x32) -> u8x32 { + fn not_i8x32(self, a: i8x32) -> i8x32 { a ^ !0 } #[inline(always)] - fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, shift: u32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, shift: u32) -> i8x32 { let val = a.into(); let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); - let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); - let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); + let lo_16 = + _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let hi_16 = + _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); let lo_shifted = _mm256_sll_epi16(lo_16, shift_count); let hi_shifted = _mm256_sll_epi16(hi_16, shift_count); - _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let a: [i8; 32usize] = a.into(); + let b: [i8; 32usize] = b.into(); + let result: [i8; 32usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + core::ops::Shl::shl(a[16usize], b[16usize]), + core::ops::Shl::shl(a[17usize], b[17usize]), + core::ops::Shl::shl(a[18usize], b[18usize]), + core::ops::Shl::shl(a[19usize], b[19usize]), + core::ops::Shl::shl(a[20usize], b[20usize]), + core::ops::Shl::shl(a[21usize], b[21usize]), + core::ops::Shl::shl(a[22usize], b[22usize]), + core::ops::Shl::shl(a[23usize], b[23usize]), + core::ops::Shl::shl(a[24usize], b[24usize]), + core::ops::Shl::shl(a[25usize], b[25usize]), + core::ops::Shl::shl(a[26usize], b[26usize]), + core::ops::Shl::shl(a[27usize], b[27usize]), + core::ops::Shl::shl(a[28usize], b[28usize]), + core::ops::Shl::shl(a[29usize], b[29usize]), + core::ops::Shl::shl(a[30usize], b[30usize]), + core::ops::Shl::shl(a[31usize], b[31usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, shift: u32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, shift: u32) -> i8x32 { let val = a.into(); let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); - let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); - let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); - let lo_shifted = _mm256_srl_epi16(lo_16, shift_count); - let hi_shifted = _mm256_srl_epi16(hi_16, shift_count); - _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + let lo_16 = + _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let hi_16 = + _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let lo_shifted = _mm256_sra_epi16(lo_16, shift_count); + let hi_shifted = _mm256_sra_epi16(hi_16, shift_count); + _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let a: [i8; 32usize] = a.into(); + let b: [i8; 32usize] = b.into(); + let result: [i8; 32usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + core::ops::Shr::shr(a[16usize], b[16usize]), + core::ops::Shr::shr(a[17usize], b[17usize]), + core::ops::Shr::shr(a[18usize], b[18usize]), + core::ops::Shr::shr(a[19usize], b[19usize]), + core::ops::Shr::shr(a[20usize], b[20usize]), + core::ops::Shr::shr(a[21usize], b[21usize]), + core::ops::Shr::shr(a[22usize], b[22usize]), + core::ops::Shr::shr(a[23usize], b[23usize]), + core::ops::Shr::shr(a[24usize], b[24usize]), + core::ops::Shr::shr(a[25usize], b[25usize]), + core::ops::Shr::shr(a[26usize], b[26usize]), + core::ops::Shr::shr(a[27usize], b[27usize]), + core::ops::Shr::shr(a[28usize], b[28usize]), + core::ops::Shr::shr(a[29usize], b[29usize]), + core::ops::Shr::shr(a[30usize], b[30usize]), + core::ops::Shr::shr(a[31usize], b[31usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { - let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi8(b_signed, a_signed).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { + _mm256_cmpgt_epi8(b.into(), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { - _mm256_cmpeq_epi8(_mm256_min_epu8(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { + _mm256_cmpeq_epi8(_mm256_min_epi8(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { - _mm256_cmpeq_epi8(_mm256_max_epu8(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { + _mm256_cmpeq_epi8(_mm256_max_epi8(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { - let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi8(a_signed, b_signed).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> mask8x32 { + _mm256_cmpgt_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { let lo = _mm256_unpacklo_epi8(a.into(), b.into()); let hi = _mm256_unpackhi_epi8(a.into(), b.into()); _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) @@ -5151,10 +5760,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { let lo = _mm256_unpacklo_epi8(a.into(), b.into()); let hi = _mm256_unpackhi_epi8(a.into(), b.into()); _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) @@ -5163,10 +5772,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -5187,10 +5796,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -5211,10 +5820,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> (i8x32, i8x32) { let lo = _mm256_unpacklo_epi8(a.into(), b.into()); let hi = _mm256_unpackhi_epi8(a.into(), b.into()); ( @@ -5226,10 +5835,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> (i8x32, i8x32) { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -5253,52 +5862,52 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, a: mask8x32, - b: u8x32, - c: u8x32, - ) -> u8x32 { + b: i8x32, + c: i8x32, + ) -> i8x32 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_min_epu8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { + _mm256_min_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_max_epu8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i8x32, b: i8x32) -> i8x32 { + _mm256_max_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { - u8x64 { + fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { + i8x64 { val: crate::support::Aligned512([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32) -> (u8x16, u8x16) { + fn kernel(token: Avx2, a: i8x32) -> (i8x16, i8x16) { ( _mm256_extracti128_si256::<0>(a.into()).simd_into(token), _mm256_extracti128_si256::<1>(a.into()).simd_into(token), @@ -5308,482 +5917,407 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn widen_u8x32(self, a: u8x32) -> u16x32 { + fn neg_i8x32(self, a: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32) -> u16x32 { - let (a0, a1) = token.split_u8x32(a); - let high = _mm256_cvtepu8_epi16(a0.into()).simd_into(token); - let low = _mm256_cvtepu8_epi16(a1.into()).simd_into(token); - token.combine_u16x16(high, low) + fn kernel(token: Avx2, a: i8x32) -> i8x32 { + _mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { + fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32) -> u32x8 { + fn kernel(token: Avx2, a: i8x32) -> u8x32 { __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn splat_mask8x32(self, val: bool) -> mask8x32 { + fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: bool) -> mask8x32 { - let val: i8 = if val { !0 } else { 0 }; - _mm256_set1_epi8(val).simd_into(token) + fn kernel(token: Avx2, a: i8x32) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u8x32(self, val: u8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: u8) -> u8x32 { + _mm256_set1_epi8(val.cast_signed()).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { - mask8x32 { + fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0) + fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask8x32 { - { - let bit_bytes = _mm256_broadcastsi128_si256(_mm_cvtsi32_si128(bits as i32)); - let bit_bytes = _mm256_shuffle_epi8( - bit_bytes, - _mm256_setr_epi8( - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, - ), - ); - let bit_mask = _mm256_setr_epi8( - 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, - 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, - ); - _mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask) - } - .simd_into(token) - } + fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { + crate::transmute::checked_cast_ref::<__m256i, [u8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { + crate::transmute::checked_cast_mut::<__m256i, [u8; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_alignr_256x1( + self, + self.cvt_to_bytes_u8x32(b).val.0, + self.cvt_to_bytes_u8x32(a).val.0, + SHIFT, ); - kernel(self, bits) + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> u64 { - _mm256_movemask_epi8(a.into()) as u32 as u64 - } + fn slide_within_blocks_u8x32( + self, + a: u8x32, + b: u8x32, + ) -> u8x32 { + if SHIFT >= 16usize { + return b; + } + let result = dyn_alignr_256( + self, + self.cvt_to_bytes_u8x32(b).val.0, + self.cvt_to_bytes_u8x32(a).val.0, + SHIFT, ); - kernel(self, a) + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_add_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_sub_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + let dst_even = _mm256_mullo_epi16(a.into(), b.into()); + let dst_odd = _mm256_mullo_epi16( + _mm256_srli_epi16::<8>(a.into()), + _mm256_srli_epi16::<8>(b.into()), + ); + _mm256_or_si256( + _mm256_slli_epi16(dst_odd, 8), + _mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)), + ) + .simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask8x32(self, a: mask8x32) -> mask8x32 { - self.xor_mask8x32(a, self.splat_mask8x32(true)) - } - #[inline(always)] - fn select_mask8x32( - self, - a: mask8x32, - b: mask8x32, - c: mask8x32, - ) -> mask8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx2, - a: mask8x32, - b: mask8x32, - c: mask8x32, - ) -> mask8x32 { - _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) - } - ); - kernel(self, a, b, c) - } - #[inline(always)] - fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { - _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask8x32(self, a: mask8x32) -> bool { + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> bool { - _mm256_movemask_epi8(a.into()) as u32 != 0 + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_true_mask8x32(self, a: mask8x32) -> bool { + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> bool { - _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn any_false_mask8x32(self, a: mask8x32) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> bool { - _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff - } - ); - kernel(self, a) + fn not_u8x32(self, a: u8x32) -> u8x32 { + a ^ !0 } #[inline(always)] - fn all_false_mask8x32(self, a: mask8x32) -> bool { + fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> bool { - _mm256_movemask_epi8(a.into()) as u32 == 0 + fn kernel(token: Avx2, a: u8x32, shift: u32) -> u8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); + let lo_shifted = _mm256_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm256_sll_epi16(hi_16, shift_count); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) } ); - kernel(self, a) + kernel(self, a, shift) } #[inline(always)] - fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { - mask8x64 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } + fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let a: [u8; 32usize] = a.into(); + let b: [u8; 32usize] = b.into(); + let result: [u8; 32usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + core::ops::Shl::shl(a[16usize], b[16usize]), + core::ops::Shl::shl(a[17usize], b[17usize]), + core::ops::Shl::shl(a[18usize], b[18usize]), + core::ops::Shl::shl(a[19usize], b[19usize]), + core::ops::Shl::shl(a[20usize], b[20usize]), + core::ops::Shl::shl(a[21usize], b[21usize]), + core::ops::Shl::shl(a[22usize], b[22usize]), + core::ops::Shl::shl(a[23usize], b[23usize]), + core::ops::Shl::shl(a[24usize], b[24usize]), + core::ops::Shl::shl(a[25usize], b[25usize]), + core::ops::Shl::shl(a[26usize], b[26usize]), + core::ops::Shl::shl(a[27usize], b[27usize]), + core::ops::Shl::shl(a[28usize], b[28usize]), + core::ops::Shl::shl(a[29usize], b[29usize]), + core::ops::Shl::shl(a[30usize], b[30usize]), + core::ops::Shl::shl(a[31usize], b[31usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { + fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask8x32) -> (mask8x16, mask8x16) { - ( - _mm256_extracti128_si256::<0>(a.into()).simd_into(token), - _mm256_extracti128_si256::<1>(a.into()).simd_into(token), - ) + fn kernel(token: Avx2, a: u8x32, shift: u32) -> u8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); + let lo_shifted = _mm256_srl_epi16(lo_16, shift_count); + let hi_shifted = _mm256_srl_epi16(hi_16, shift_count); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) } ); - kernel(self, a) + kernel(self, a, shift) } #[inline(always)] - fn splat_i16x16(self, val: i16) -> i16x16 { + fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let a: [u8; 32usize] = a.into(); + let b: [u8; 32usize] = b.into(); + let result: [u8; 32usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + core::ops::Shr::shr(a[16usize], b[16usize]), + core::ops::Shr::shr(a[17usize], b[17usize]), + core::ops::Shr::shr(a[18usize], b[18usize]), + core::ops::Shr::shr(a[19usize], b[19usize]), + core::ops::Shr::shr(a[20usize], b[20usize]), + core::ops::Shr::shr(a[21usize], b[21usize]), + core::ops::Shr::shr(a[22usize], b[22usize]), + core::ops::Shr::shr(a[23usize], b[23usize]), + core::ops::Shr::shr(a[24usize], b[24usize]), + core::ops::Shr::shr(a[25usize], b[25usize]), + core::ops::Shr::shr(a[26usize], b[26usize]), + core::ops::Shr::shr(a[27usize], b[27usize]), + core::ops::Shr::shr(a[28usize], b[28usize]), + core::ops::Shr::shr(a[29usize], b[29usize]), + core::ops::Shr::shr(a[30usize], b[30usize]), + core::ops::Shr::shr(a[31usize], b[31usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: i16) -> i16x16 { - _mm256_set1_epi16(val).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token) } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { - crate::transmute::checked_cast_ref::<__m256i, [i16; 16usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { - crate::transmute::checked_cast_mut::<__m256i, [i16; 16usize]>(&mut a.val.0) - } - #[inline(always)] - fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); - } - #[inline(always)] - fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - if SHIFT >= 16usize { - return b; - } - let result = cross_block_alignr_256x1( - self, - self.cvt_to_bytes_i16x16(b).val.0, - self.cvt_to_bytes_i16x16(a).val.0, - SHIFT * 2usize, - ); - self.cvt_from_bytes_i16x16(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) - } - #[inline(always)] - fn slide_within_blocks_i16x16( - self, - a: i16x16, - b: i16x16, - ) -> i16x16 { - if SHIFT >= 8usize { - return b; - } - let result = dyn_alignr_256( - self, - self.cvt_to_bytes_i16x16(b).val.0, - self.cvt_to_bytes_i16x16(a).val.0, - SHIFT * 2usize, - ); - self.cvt_from_bytes_i16x16(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + kernel(self, a, b) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_add_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi8(b_signed, a_signed).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_sub_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + _mm256_cmpeq_epi8(_mm256_min_epu8(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_mullo_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + _mm256_cmpeq_epi8(_mm256_max_epu8(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> mask8x32 { + let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi8(a_signed, b_signed).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + let lo = _mm256_unpacklo_epi8(a.into(), b.into()); + let hi = _mm256_unpackhi_epi8(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + let lo = _mm256_unpacklo_epi8(a.into(), b.into()); + let hi = _mm256_unpackhi_epi8(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_i16x16(self, a: i16x16) -> i16x16 { - a ^ !0 - } - #[inline(always)] - fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, shift: u32) -> i16x16 { - _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) - } - ); - kernel(self, a, shift) - } - #[inline(always)] - fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) - } - #[inline(always)] - fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, shift: u32) -> i16x16 { - _mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) - } - ); - kernel(self, a, shift) - } - #[inline(always)] - fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) - } - #[inline(always)] - fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { - _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { - _mm256_cmpgt_epi16(b.into(), a.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { - _mm256_cmpeq_epi16(_mm256_min_epi16(a.into(), b.into()), a.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { - _mm256_cmpeq_epi16(_mm256_max_epi16(a.into(), b.into()), a.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { - _mm256_cmpgt_epi16(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - let lo = _mm256_unpacklo_epi16(a.into(), b.into()); - let hi = _mm256_unpackhi_epi16(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - let lo = _mm256_unpacklo_epi16(a.into(), b.into()); - let hi = _mm256_unpackhi_epi16(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( b.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) @@ -5792,22 +6326,22 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( b.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) @@ -5816,16 +6350,12 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: i16x16, - b: i16x16, - ) -> (i16x16, i16x16) { - let lo = _mm256_unpacklo_epi16(a.into(), b.into()); - let hi = _mm256_unpackhi_epi16(a.into(), b.into()); + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let lo = _mm256_unpacklo_epi8(a.into(), b.into()); + let hi = _mm256_unpackhi_epi8(a.into(), b.into()); ( _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), @@ -5835,26 +6365,22 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: i16x16, - b: i16x16, - ) -> (i16x16, i16x16) { + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( b.into(), _mm256_setr_epi8( - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, - 13, 2, 3, 6, 7, 10, 11, 14, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, ), )); ( @@ -5866,52 +6392,52 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, - a: mask16x16, - b: i16x16, - c: i16x16, - ) -> i16x16 { + a: mask8x32, + b: u8x32, + c: u8x32, + ) -> u8x32 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_min_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_min_epu8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { - _mm256_max_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_max_epu8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { - i16x32 { + fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { + u8x64 { val: crate::support::Aligned512([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { + fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16) -> (i16x8, i16x8) { + fn kernel(token: Avx2, a: u8x32) -> (u8x16, u8x16) { ( _mm256_extracti128_si256::<0>(a.into()).simd_into(token), _mm256_extracti128_si256::<1>(a.into()).simd_into(token), @@ -5921,278 +6447,497 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn neg_i16x16(self, a: i16x16) -> i16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x16) -> i16x16 { - _mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token) - } - ); - kernel(self, a) - } - #[inline(always)] - fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { + fn widen_u8x32(self, a: u8x32) -> u16x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16) -> u8x32 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: u8x32) -> u16x32 { + let (a0, a1) = token.split_u8x32(a); + let high = _mm256_cvtepu8_epi16(a0.into()).simd_into(token); + let low = _mm256_cvtepu8_epi16(a1.into()).simd_into(token); + token.combine_u16x16(high, low) } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { + fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i16x16) -> u32x8 { + fn kernel(token: Avx2, a: u8x32) -> u32x8 { __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn splat_u16x16(self, val: u16) -> u16x16 { + fn splat_mask8x32(self, val: bool) -> mask8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: u16) -> u16x16 { - _mm256_set1_epi16(val.cast_signed()).simd_into(token) + fn kernel(token: Avx2, val: bool) -> mask8x32 { + let val: i8 = if val { !0 } else { 0 }; + _mm256_set1_epi8(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { - u16x16 { + fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { + mask8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { - u16x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { - crate::transmute::checked_transmute_copy::<__m256i, [u16; 16usize]>(&a.val.0) + fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, bits: u64) -> mask8x32 { + { + let bit_bytes = _mm256_broadcastsi128_si256(_mm_cvtsi32_si128(bits as i32)); + let bit_bytes = _mm256_shuffle_epi8( + bit_bytes, + _mm256_setr_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, + ), + ); + let bit_mask = _mm256_setr_epi8( + 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, + 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, + ); + _mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask) + } + .simd_into(token) + } + ); + kernel(self, bits) } #[inline(always)] - fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { - crate::transmute::checked_cast_ref::<__m256i, [u16; 16usize]>(&a.val.0) + fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> u64 { + _mm256_movemask_epi8(a.into()) as u32 as u64 + } + ); + kernel(self, a) } #[inline(always)] - fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { - crate::transmute::checked_cast_mut::<__m256i, [u16; 16usize]>(&mut a.val.0) + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask8x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x32(lanes); } #[inline(always)] - fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_mask8x32(self, a: mask8x32) -> mask8x32 { + self.xor_mask8x32(a, self.splat_mask8x32(true)) + } + #[inline(always)] + fn select_mask8x32( + self, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32, b: mask8x32) -> mask8x32 { + _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn any_true_mask8x32(self, a: mask8x32) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> bool { + _mm256_movemask_epi8(a.into()) as u32 != 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_true_mask8x32(self, a: mask8x32) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> bool { + _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff + } + ); + kernel(self, a) + } + #[inline(always)] + fn any_false_mask8x32(self, a: mask8x32) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> bool { + _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_false_mask8x32(self, a: mask8x32) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> bool { + _mm256_movemask_epi8(a.into()) as u32 == 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { + mask8x64 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask8x32) -> (mask8x16, mask8x16) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i16x16(self, val: i16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: i16) -> i16x16 { + _mm256_set1_epi16(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { + crate::transmute::checked_cast_ref::<__m256i, [i16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { + crate::transmute::checked_cast_mut::<__m256i, [i16; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { - u16x16 { + fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { if SHIFT >= 16usize { return b; } let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_u16x16(b).val.0, - self.cvt_to_bytes_u16x16(a).val.0, + self.cvt_to_bytes_i16x16(b).val.0, + self.cvt_to_bytes_i16x16(a).val.0, SHIFT * 2usize, ); - self.cvt_from_bytes_u16x16(u8x32 { + self.cvt_from_bytes_i16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u16x16( + fn slide_within_blocks_i16x16( self, - a: u16x16, - b: u16x16, - ) -> u16x16 { + a: i16x16, + b: i16x16, + ) -> i16x16 { if SHIFT >= 8usize { return b; } let result = dyn_alignr_256( self, - self.cvt_to_bytes_u16x16(b).val.0, - self.cvt_to_bytes_u16x16(a).val.0, + self.cvt_to_bytes_i16x16(b).val.0, + self.cvt_to_bytes_i16x16(a).val.0, SHIFT * 2usize, ); - self.cvt_from_bytes_u16x16(u8x32 { + self.cvt_from_bytes_i16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_add_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_sub_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_mullo_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_u16x16(self, a: u16x16) -> u16x16 { + fn not_i16x16(self, a: i16x16) -> i16x16 { a ^ !0 } #[inline(always)] - fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, shift: u32) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, shift: u32) -> i16x16 { _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let a: [i16; 16usize] = a.into(); + let b: [i16; 16usize] = b.into(); + let result: [i16; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, shift: u32) -> u16x16 { - _mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + fn kernel(token: Avx2, a: i16x16, shift: u32) -> i16x16 { + _mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let a: [i16; 16usize] = a.into(); + let b: [i16; 16usize] = b.into(); + let result: [i16; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { - let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi16(b_signed, a_signed).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { + _mm256_cmpgt_epi16(b.into(), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { - _mm256_cmpeq_epi16(_mm256_min_epu16(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { + _mm256_cmpeq_epi16(_mm256_min_epi16(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { - _mm256_cmpeq_epi16(_mm256_max_epu16(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { + _mm256_cmpeq_epi16(_mm256_max_epi16(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { - let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi16(a_signed, b_signed).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> mask16x16 { + _mm256_cmpgt_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { let lo = _mm256_unpacklo_epi16(a.into(), b.into()); let hi = _mm256_unpackhi_epi16(a.into(), b.into()); _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) @@ -6201,10 +6946,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { let lo = _mm256_unpacklo_epi16(a.into(), b.into()); let hi = _mm256_unpackhi_epi16(a.into(), b.into()); _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) @@ -6213,10 +6958,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -6237,10 +6982,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -6261,14 +7006,14 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, - a: u16x16, - b: u16x16, - ) -> (u16x16, u16x16) { + a: i16x16, + b: i16x16, + ) -> (i16x16, i16x16) { let lo = _mm256_unpacklo_epi16(a.into(), b.into()); let hi = _mm256_unpackhi_epi16(a.into(), b.into()); ( @@ -6280,14 +7025,14 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, - a: u16x16, - b: u16x16, - ) -> (u16x16, u16x16) { + a: i16x16, + b: i16x16, + ) -> (i16x16, i16x16) { let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( a.into(), _mm256_setr_epi8( @@ -6311,52 +7056,52 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, a: mask16x16, - b: u16x16, - c: u16x16, - ) -> u16x16 { + b: i16x16, + c: i16x16, + ) -> i16x16 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { - _mm256_min_epu16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { + _mm256_min_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { - _mm256_max_epu16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i16x16, b: i16x16) -> i16x16 { + _mm256_max_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { + fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { + i16x32 { val: crate::support::Aligned512([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16) -> (u16x8, u16x8) { + fn kernel(token: Avx2, a: i16x16) -> (i16x8, i16x8) { ( _mm256_extracti128_si256::<0>(a.into()).simd_into(token), _mm256_extracti128_si256::<1>(a.into()).simd_into(token), @@ -6366,910 +7111,965 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn narrow_u16x16(self, a: u16x16) -> u8x16 { + fn neg_i16x16(self, a: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16) -> u8x16 { - let mask = _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, - 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, - ); - let shuffled = _mm256_shuffle_epi8(a.into(), mask); - let packed = _mm256_permute4x64_epi64::<0b11_01_10_00>(shuffled); - _mm256_castsi256_si128(packed).simd_into(token) + fn kernel(token: Avx2, a: i16x16) -> i16x16 { + _mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { + fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16) -> u8x32 { + fn kernel(token: Avx2, a: i16x16) -> u8x32 { __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { + fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u16x16) -> u32x8 { + fn kernel(token: Avx2, a: i16x16) -> u32x8 { __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn splat_mask16x16(self, val: bool) -> mask16x16 { + fn splat_u16x16(self, val: u16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: bool) -> mask16x16 { - let val: i16 = if val { !0 } else { 0 }; - _mm256_set1_epi16(val).simd_into(token) + fn kernel(token: Avx2, val: u16) -> u16x16 { + _mm256_set1_epi16(val.cast_signed()).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { - mask16x16 { + fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0) + fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask16x16 { - { - let bit_lanes = _mm256_set1_epi16(bits as i16); - let bit_mask = _mm256_setr_epi16( - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, - -32768, - ); - _mm256_cmpeq_epi16(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) - } - .simd_into(token) - } + fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { + crate::transmute::checked_cast_ref::<__m256i, [u16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { + crate::transmute::checked_cast_mut::<__m256i, [u16; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_alignr_256x1( + self, + self.cvt_to_bytes_u16x16(b).val.0, + self.cvt_to_bytes_u16x16(a).val.0, + SHIFT * 2usize, ); - kernel(self, bits) + self.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { + fn slide_within_blocks_u16x16( + self, + a: u16x16, + b: u16x16, + ) -> u16x16 { + if SHIFT >= 8usize { + return b; + } + let result = dyn_alignr_256( + self, + self.cvt_to_bytes_u16x16(b).val.0, + self.cvt_to_bytes_u16x16(a).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> u64 { - { - let halves: [__m128i; 2usize] = - crate::transmute::checked_transmute_copy(&a.val.0); - let packed = _mm_packs_epi16(halves[0], halves[1]); - _mm_movemask_epi8(packed) as u32 as u64 - } + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_add_epi16(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_sub_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_mullo_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask16x16(self, a: mask16x16) -> mask16x16 { - self.xor_mask16x16(a, self.splat_mask16x16(true)) - } - #[inline(always)] - fn select_mask16x16( - self, - a: mask16x16, - b: mask16x16, - c: mask16x16, - ) -> mask16x16 { + fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: mask16x16, - b: mask16x16, - c: mask16x16, - ) -> mask16x16 { - _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, b) } #[inline(always)] - fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { - _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask16x16(self, a: mask16x16) -> bool { + fn not_u16x16(self, a: u16x16) -> u16x16 { + a ^ !0 + } + #[inline(always)] + fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> bool { - _mm256_movemask_epi8(a.into()) as u32 != 0 + fn kernel(token: Avx2, a: u16x16, shift: u32) -> u16x16 { + _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a) + kernel(self, a, shift) } #[inline(always)] - fn all_true_mask16x16(self, a: mask16x16) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> bool { - _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff - } - ); - kernel(self, a) + fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let a: [u16; 16usize] = a.into(); + let b: [u16; 16usize] = b.into(); + let result: [u16; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn any_false_mask16x16(self, a: mask16x16) -> bool { + fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> bool { - _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff + fn kernel(token: Avx2, a: u16x16, shift: u32) -> u16x16 { + _mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a) + kernel(self, a, shift) } #[inline(always)] - fn all_false_mask16x16(self, a: mask16x16) -> bool { + fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let a: [u16; 16usize] = a.into(); + let b: [u16; 16usize] = b.into(); + let result: [u16; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> bool { - _mm256_movemask_epi8(a.into()) as u32 == 0 + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { - mask16x32 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } + kernel(self, a, b) } #[inline(always)] - fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask16x16) -> (mask16x8, mask16x8) { - ( - _mm256_extracti128_si256::<0>(a.into()).simd_into(token), - _mm256_extracti128_si256::<1>(a.into()).simd_into(token), - ) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi16(b_signed, a_signed).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn splat_i32x8(self, val: i32) -> i32x8 { + fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: i32) -> i32x8 { - _mm256_set1_epi32(val).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + _mm256_cmpeq_epi16(_mm256_min_epu16(a.into(), b.into()), a.into()).simd_into(token) } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { - i32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { - i32x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { - crate::transmute::checked_cast_ref::<__m256i, [i32; 8usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { - crate::transmute::checked_cast_mut::<__m256i, [i32; 8usize]>(&mut a.val.0) - } - #[inline(always)] - fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); - } - #[inline(always)] - fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { - i32x8 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - if SHIFT >= 8usize { - return b; - } - let result = cross_block_alignr_256x1( - self, - self.cvt_to_bytes_i32x8(b).val.0, - self.cvt_to_bytes_i32x8(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_i32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) - } - #[inline(always)] - fn slide_within_blocks_i32x8( - self, - a: i32x8, - b: i32x8, - ) -> i32x8 { - if SHIFT >= 4usize { - return b; - } - let result = dyn_alignr_256( - self, - self.cvt_to_bytes_i32x8(b).val.0, - self.cvt_to_bytes_i32x8(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_i32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + kernel(self, a, b) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_add_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + _mm256_cmpeq_epi16(_mm256_max_epu16(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_sub_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> mask16x16 { + let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi16(a_signed, b_signed).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_mullo_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + let lo = _mm256_unpacklo_epi16(a.into(), b.into()); + let hi = _mm256_unpackhi_epi16(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + let lo = _mm256_unpacklo_epi16(a.into(), b.into()); + let hi = _mm256_unpackhi_epi16(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + a.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + b.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + a.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + b.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_i32x8(self, a: i32x8) -> i32x8 { - a ^ !0 - } - #[inline(always)] - fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, shift: u32) -> i32x8 { - _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + fn kernel( + token: Avx2, + a: u16x16, + b: u16x16, + ) -> (u16x16, u16x16) { + let lo = _mm256_unpacklo_epi16(a.into(), b.into()); + let hi = _mm256_unpackhi_epi16(a.into(), b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), + ) } ); - kernel(self, a, shift) + kernel(self, a, b) } #[inline(always)] - fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_sllv_epi32(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx2, + a: u16x16, + b: u16x16, + ) -> (u16x16, u16x16) { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + a.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + b.into(), + _mm256_setr_epi8( + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, + 13, 2, 3, 6, 7, 10, 11, 14, 15, + ), + )); + ( + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), + ) } ); kernel(self, a, b) } #[inline(always)] - fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, shift: u32) -> i32x8 { - _mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + fn kernel( + token: Avx2, + a: mask16x16, + b: u16x16, + c: u16x16, + ) -> u16x16 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); - kernel(self, a, shift) + kernel(self, a, b, c) } #[inline(always)] - fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_srav_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_min_epu16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { - _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16, b: u16x16) -> u16x16 { + _mm256_max_epu16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { - _mm256_cmpgt_epi32(b.into(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16) -> (u16x8, u16x8) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + fn narrow_u16x16(self, a: u16x16) -> u8x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { - _mm256_cmpeq_epi32(_mm256_min_epi32(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16) -> u8x16 { + let mask = _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, + 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + ); + let shuffled = _mm256_shuffle_epi8(a.into(), mask); + let packed = _mm256_permute4x64_epi64::<0b11_01_10_00>(shuffled); + _mm256_castsi256_si128(packed).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { - _mm256_cmpeq_epi32(_mm256_max_epi32(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16) -> u8x32 { + __m256i::from(a).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { - _mm256_cmpgt_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u16x16) -> u32x8 { + __m256i::from(a).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn splat_mask16x16(self, val: bool) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - let lo = _mm256_unpacklo_epi32(a.into(), b.into()); - let hi = _mm256_unpackhi_epi32(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) + fn kernel(token: Avx2, val: bool) -> mask16x16 { + let val: i16 = if val { !0 } else { 0 }; + _mm256_set1_epi16(val).simd_into(token) } ); - kernel(self, a, b) + kernel(self, val) } #[inline(always)] - fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { + mask16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - let lo = _mm256_unpacklo_epi32(a.into(), b.into()); - let hi = _mm256_unpackhi_epi32(a.into(), b.into()); - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) + fn kernel(token: Avx2, bits: u64) -> mask16x16 { + { + let bit_lanes = _mm256_set1_epi16(bits as i16); + let bit_mask = _mm256_setr_epi16( + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, + -32768, + ); + _mm256_cmpeq_epi16(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) + } + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, bits) } #[inline(always)] - fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - let t1 = _mm256_permutevar8x32_epi32( - a.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - let t2 = _mm256_permutevar8x32_epi32( - b.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: mask16x16) -> u64 { + { + let halves: [__m128i; 2usize] = + crate::transmute::checked_transmute_copy(&a.val.0); + let packed = _mm_packs_epi16(halves[0], halves[1]); + _mm_movemask_epi8(packed) as u32 as u64 + } } ); - kernel(self, a, b) + kernel(self, a) } #[inline(always)] - fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask16x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x16(lanes); + } + #[inline(always)] + fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - let t1 = _mm256_permutevar8x32_epi32( - a.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - let t2 = _mm256_permutevar8x32_epi32( - b.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let lo = _mm256_unpacklo_epi32(a.into(), b.into()); - let hi = _mm256_unpackhi_epi32(a.into(), b.into()); - ( - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), - ) + fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let t1 = _mm256_permutevar8x32_epi32( - a.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - let t2 = _mm256_permutevar8x32_epi32( - b.into(), - _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), - ); - ( - _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), - _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), - ) + fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { + fn not_mask16x16(self, a: mask16x16) -> mask16x16 { + self.xor_mask16x16(a, self.splat_mask16x16(true)) + } + #[inline(always)] + fn select_mask16x16( + self, + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, - a: mask32x8, - b: i32x8, - c: i32x8, - ) -> i32x8 { + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_min_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: mask16x16, b: mask16x16) -> mask16x16 { + _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn any_true_mask16x16(self, a: mask16x16) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { - _mm256_max_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: mask16x16) -> bool { + _mm256_movemask_epi8(a.into()) as u32 != 0 } ); - kernel(self, a, b) - } - #[inline(always)] - fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { - i32x16 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } + kernel(self, a) } #[inline(always)] - fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { + fn all_true_mask16x16(self, a: mask16x16) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8) -> (i32x4, i32x4) { - ( - _mm256_extracti128_si256::<0>(a.into()).simd_into(token), - _mm256_extracti128_si256::<1>(a.into()).simd_into(token), - ) + fn kernel(token: Avx2, a: mask16x16) -> bool { + _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff } ); kernel(self, a) } #[inline(always)] - fn neg_i32x8(self, a: i32x8) -> i32x8 { + fn any_false_mask16x16(self, a: mask16x16) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8) -> i32x8 { - _mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token) + fn kernel(token: Avx2, a: mask16x16) -> bool { + _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { + fn all_false_mask16x16(self, a: mask16x16) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8) -> u8x32 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: mask16x16) -> bool { + _mm256_movemask_epi8(a.into()) as u32 == 0 } ); kernel(self, a) } #[inline(always)] - fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { + fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { + mask16x32 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: i32x8) -> u32x8 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: mask16x16) -> (mask16x8, mask16x8) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) } ); kernel(self, a) } #[inline(always)] - fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i32x8) -> f32x8 { - _mm256_cvtepi32_ps(a.into()).simd_into(token) - } - ); - kernel(self, a) - } - #[inline(always)] - fn splat_u32x8(self, val: u32) -> u32x8 { + fn splat_i32x8(self, val: i32) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: u32) -> u32x8 { - _mm256_set1_epi32(val.cast_signed()).simd_into(token) + fn kernel(token: Avx2, val: i32) -> i32x8 { + _mm256_set1_epi32(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { - crate::transmute::checked_transmute_copy::<__m256i, [u32; 8usize]>(&a.val.0) + fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { - crate::transmute::checked_cast_ref::<__m256i, [u32; 8usize]>(&a.val.0) + fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { + crate::transmute::checked_cast_ref::<__m256i, [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { - crate::transmute::checked_cast_mut::<__m256i, [u32; 8usize]>(&mut a.val.0) + fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { + crate::transmute::checked_cast_mut::<__m256i, [i32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { + fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { - u32x8 { + fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { + fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { if SHIFT >= 8usize { return b; } let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_u32x8(b).val.0, - self.cvt_to_bytes_u32x8(a).val.0, + self.cvt_to_bytes_i32x8(b).val.0, + self.cvt_to_bytes_i32x8(a).val.0, SHIFT * 4usize, ); - self.cvt_from_bytes_u32x8(u8x32 { + self.cvt_from_bytes_i32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u32x8( + fn slide_within_blocks_i32x8( self, - a: u32x8, - b: u32x8, - ) -> u32x8 { + a: i32x8, + b: i32x8, + ) -> i32x8 { if SHIFT >= 4usize { return b; } let result = dyn_alignr_256( self, - self.cvt_to_bytes_u32x8(b).val.0, - self.cvt_to_bytes_u32x8(a).val.0, + self.cvt_to_bytes_i32x8(b).val.0, + self.cvt_to_bytes_i32x8(a).val.0, SHIFT * 4usize, ); - self.cvt_from_bytes_u32x8(u8x32 { + self.cvt_from_bytes_i32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_add_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_sub_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_mullo_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_u32x8(self, a: u32x8) -> u32x8 { + fn not_i32x8(self, a: i32x8) -> i32x8 { a ^ !0 } #[inline(always)] - fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, shift: u32) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, shift: u32) -> i32x8 { _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { _mm256_sllv_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, shift: u32) -> u32x8 { - _mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + fn kernel(token: Avx2, a: i32x8, shift: u32) -> i32x8 { + _mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); kernel(self, a, shift) } #[inline(always)] - fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { - _mm256_srlv_epi32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { + _mm256_srav_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { - let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi32(b_signed, a_signed).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { + _mm256_cmpgt_epi32(b.into(), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { - _mm256_cmpeq_epi32(_mm256_min_epu32(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { + _mm256_cmpeq_epi32(_mm256_min_epi32(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { - _mm256_cmpeq_epi32(_mm256_max_epu32(a.into(), b.into()), a.into()).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { + _mm256_cmpeq_epi32(_mm256_max_epi32(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { - let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed()); - let a_signed = _mm256_xor_si256(a.into(), sign_bit); - let b_signed = _mm256_xor_si256(b.into(), sign_bit); - _mm256_cmpgt_epi32(a_signed, b_signed).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> mask32x8 { + _mm256_cmpgt_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { let lo = _mm256_unpacklo_epi32(a.into(), b.into()); let hi = _mm256_unpackhi_epi32(a.into(), b.into()); _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) @@ -7278,10 +8078,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { let lo = _mm256_unpacklo_epi32(a.into(), b.into()); let hi = _mm256_unpackhi_epi32(a.into(), b.into()); _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) @@ -7290,10 +8090,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { let t1 = _mm256_permutevar8x32_epi32( a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), @@ -7308,10 +8108,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { let t1 = _mm256_permutevar8x32_epi32( a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), @@ -7326,10 +8126,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> (i32x8, i32x8) { let lo = _mm256_unpacklo_epi32(a.into(), b.into()); let hi = _mm256_unpackhi_epi32(a.into(), b.into()); ( @@ -7341,10 +8141,10 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> (i32x8, i32x8) { let t1 = _mm256_permutevar8x32_epi32( a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), @@ -7362,52 +8162,52 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { + fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, a: mask32x8, - b: u32x8, - c: u32x8, - ) -> u32x8 { + b: i32x8, + c: i32x8, + ) -> i32x8 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { - _mm256_min_epu32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { + _mm256_min_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { - _mm256_max_epu32(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: i32x8, b: i32x8) -> i32x8 { + _mm256_max_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { - u32x16 { + fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { + i32x16 { val: crate::support::Aligned512([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { + fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8) -> (u32x4, u32x4) { + fn kernel(token: Avx2, a: i32x8) -> (i32x4, i32x4) { ( _mm256_extracti128_si256::<0>(a.into()).simd_into(token), _mm256_extracti128_si256::<1>(a.into()).simd_into(token), @@ -7417,702 +8217,508 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { + fn neg_i32x8(self, a: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8) -> u8x32 { - __m256i::from(a).simd_into(token) + fn kernel(token: Avx2, a: i32x8) -> i32x8 { + _mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { + fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u32x8) -> f32x8 { - let a = a.into(); - let lo = _mm256_blend_epi16::<0xAA>(a, _mm256_set1_epi32(0x4B000000)); - let hi = _mm256_blend_epi16::<0xAA>( - _mm256_srli_epi32::<16>(a), - _mm256_set1_epi32(0x53000000), - ); - let fhi = _mm256_sub_ps( - _mm256_castsi256_ps(hi), - _mm256_set1_ps(f32::from_bits(0x53000080)), - ); - let result = _mm256_add_ps(_mm256_castsi256_ps(lo), fhi); - result.simd_into(token) + fn kernel(token: Avx2, a: i32x8) -> u8x32 { + __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn splat_mask32x8(self, val: bool) -> mask32x8 { + fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: bool) -> mask32x8 { - let val: i32 = if val { !0 } else { 0 }; - _mm256_set1_epi32(val).simd_into(token) + fn kernel(token: Avx2, a: i32x8) -> u32x8 { + __m256i::from(a).simd_into(token) } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { - mask32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0) + kernel(self, a) } #[inline(always)] - fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { + fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask32x8 { - { - let bit_lanes = _mm256_set1_epi32(bits as i32); - let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128); - _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) - } - .simd_into(token) + fn kernel(token: Avx2, a: i32x8) -> f32x8 { + _mm256_cvtepi32_ps(a.into()).simd_into(token) } ); - kernel(self, bits) + kernel(self, a) } #[inline(always)] - fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { + fn splat_u32x8(self, val: u32) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> u64 { - _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 as u64 - } - ); - kernel(self, a) - } - #[inline(always)] - fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { - _mm256_and_si256(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { - _mm256_or_si256(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { - _mm256_xor_si256(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn not_mask32x8(self, a: mask32x8) -> mask32x8 { - self.xor_mask32x8(a, self.splat_mask32x8(true)) - } - #[inline(always)] - fn select_mask32x8( - self, - a: mask32x8, - b: mask32x8, - c: mask32x8, - ) -> mask32x8 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx2, - a: mask32x8, - b: mask32x8, - c: mask32x8, - ) -> mask32x8 { - _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) - } - ); - kernel(self, a, b, c) - } - #[inline(always)] - fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { - _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn any_true_mask32x8(self, a: mask32x8) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> bool { - _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0 - } - ); - kernel(self, a) - } - #[inline(always)] - fn all_true_mask32x8(self, a: mask32x8) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> bool { - _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0b11111111 - } - ); - kernel(self, a) - } - #[inline(always)] - fn any_false_mask32x8(self, a: mask32x8) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> bool { - _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0b11111111 - } - ); - kernel(self, a) - } - #[inline(always)] - fn all_false_mask32x8(self, a: mask32x8) -> bool { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> bool { - _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0 - } - ); - kernel(self, a) - } - #[inline(always)] - fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { - mask32x16 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } - } - #[inline(always)] - fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: mask32x8) -> (mask32x4, mask32x4) { - ( - _mm256_extracti128_si256::<0>(a.into()).simd_into(token), - _mm256_extracti128_si256::<1>(a.into()).simd_into(token), - ) - } - ); - kernel(self, a) - } - #[inline(always)] - fn splat_f64x4(self, val: f64) -> f64x4 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, val: f64) -> f64x4 { - _mm256_set1_pd(val).simd_into(token) + fn kernel(token: Avx2, val: u32) -> u32x8 { + _mm256_set1_epi32(val.cast_signed()).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { - f64x4 { + fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { - f64x4 { + fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { - crate::transmute::checked_transmute_copy::<__m256d, [f64; 4usize]>(&a.val.0) + fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { - crate::transmute::checked_cast_ref::<__m256d, [f64; 4usize]>(&a.val.0) + fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { + crate::transmute::checked_cast_ref::<__m256i, [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { - crate::transmute::checked_cast_mut::<__m256d, [f64; 4usize]>(&mut a.val.0) + fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { + crate::transmute::checked_cast_mut::<__m256i, [u32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { + fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { - f64x4 { + fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - if SHIFT >= 4usize { + fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_f64x4(b).val.0, - self.cvt_to_bytes_f64x4(a).val.0, - SHIFT * 8usize, + self.cvt_to_bytes_u32x8(b).val.0, + self.cvt_to_bytes_u32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_f64x4(u8x32 { + self.cvt_from_bytes_u32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x4( + fn slide_within_blocks_u32x8( self, - a: f64x4, - b: f64x4, - ) -> f64x4 { - if SHIFT >= 2usize { + a: u32x8, + b: u32x8, + ) -> u32x8 { + if SHIFT >= 4usize { return b; } let result = dyn_alignr_256( self, - self.cvt_to_bytes_f64x4(b).val.0, - self.cvt_to_bytes_f64x4(a).val.0, - SHIFT * 8usize, + self.cvt_to_bytes_u32x8(b).val.0, + self.cvt_to_bytes_u32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_f64x4(u8x32 { + self.cvt_from_bytes_u32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn abs_f64x4(self, a: f64x4) -> f64x4 { + fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_add_epi32(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn neg_f64x4(self, a: f64x4) -> f64x4 { + fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_sub_epi32(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_sqrt_pd(a.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_mullo_epi32(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { - 1.0 / a + kernel(self, a, b) } #[inline(always)] - fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_add_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_sub_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_mul_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn not_u32x8(self, a: u32x8) -> u32x8 { + a ^ !0 + } + #[inline(always)] + fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_div_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, shift: u32) -> u32x8 { + _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, shift) } #[inline(always)] - fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let mask = _mm256_set1_pd(-0.0); - _mm256_or_pd( - _mm256_and_pd(mask, b.into()), - _mm256_andnot_pd(mask, a.into()), - ) - .simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_sllv_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { - _mm256_castpd_si256(_mm256_cmp_pd::<0i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u32x8, shift: u32) -> u32x8 { + _mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, shift) } #[inline(always)] - fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { - _mm256_castpd_si256(_mm256_cmp_pd::<17i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_srlv_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { - _mm256_castpd_si256(_mm256_cmp_pd::<18i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { - _mm256_castpd_si256(_mm256_cmp_pd::<29i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi32(b_signed, a_signed).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { - _mm256_castpd_si256(_mm256_cmp_pd::<30i32>(a.into(), b.into())).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + _mm256_cmpeq_epi32(_mm256_min_epu32(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let lo = _mm256_unpacklo_pd(a.into(), b.into()); - let hi = _mm256_unpackhi_pd(a.into(), b.into()); - _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + _mm256_cmpeq_epi32(_mm256_max_epu32(a.into(), b.into()), a.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let lo = _mm256_unpacklo_pd(a.into(), b.into()); - let hi = _mm256_unpackhi_pd(a.into(), b.into()); - _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> mask32x8 { + let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed()); + let a_signed = _mm256_xor_si256(a.into(), sign_bit); + let b_signed = _mm256_xor_si256(b.into(), sign_bit); + _mm256_cmpgt_epi32(a_signed, b_signed).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); - let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); - _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + let lo = _mm256_unpacklo_epi32(a.into(), b.into()); + let hi = _mm256_unpackhi_epi32(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); - let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); - _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + let lo = _mm256_unpacklo_epi32(a.into(), b.into()); + let hi = _mm256_unpackhi_epi32(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let lo = _mm256_unpacklo_pd(a.into(), b.into()); - let hi = _mm256_unpackhi_pd(a.into(), b.into()); - ( - _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token), - _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token), - ) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + let t1 = _mm256_permutevar8x32_epi32( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + let t2 = _mm256_permutevar8x32_epi32( + b.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); - let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); - ( - _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token), - _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token), - ) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + let t1 = _mm256_permutevar8x32_epi32( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + let t2 = _mm256_permutevar8x32_epi32( + b.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_max_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let lo = _mm256_unpacklo_epi32(a.into(), b.into()); + let hi = _mm256_unpackhi_epi32(a.into(), b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), + ) } ); kernel(self, a, b) } #[inline(always)] - fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - _mm256_min_pd(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let t1 = _mm256_permutevar8x32_epi32( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + let t2 = _mm256_permutevar8x32_epi32( + b.into(), + _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7), + ); + ( + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), + ) } ); kernel(self, a, b) } #[inline(always)] - fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let intermediate = _mm256_max_pd(a.into(), b.into()); - let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into()); - _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token) + fn kernel( + token: Avx2, + a: mask32x8, + b: u32x8, + c: u32x8, + ) -> u32x8 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, b, c) } #[inline(always)] - fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { - let intermediate = _mm256_min_pd(a.into(), b.into()); - let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into()); - _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_min_epu32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - _mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8, b: u32x8) -> u32x8 { + _mm256_max_epu32(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, b) } #[inline(always)] - fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { + u32x16 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - _mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token) + fn kernel(token: Avx2, a: u32x8) -> (u32x4, u32x4) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) } ); - kernel(self, a, b, c) + kernel(self, a) } #[inline(always)] - fn floor_f64x4(self, a: f64x4) -> f64x4 { + fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, a: u32x8) -> u8x32 { + __m256i::from(a).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn ceil_f64x4(self, a: f64x4) -> f64x4 { + fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, a: u32x8) -> f32x8 { + let a = a.into(); + let lo = _mm256_blend_epi16::<0xAA>(a, _mm256_set1_epi32(0x4B000000)); + let hi = _mm256_blend_epi16::<0xAA>( + _mm256_srli_epi32::<16>(a), + _mm256_set1_epi32(0x53000000), + ); + let fhi = _mm256_sub_ps( + _mm256_castsi256_ps(hi), + _mm256_set1_ps(f32::from_bits(0x53000080)), + ); + let result = _mm256_add_ps(_mm256_castsi256_ps(lo), fhi); + result.simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + fn splat_mask32x8(self, val: bool) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) + fn kernel(token: Avx2, val: bool) -> mask32x8 { + let val: i32 = if val { !0 } else { 0 }; + _mm256_set1_epi32(val).simd_into(token) } ); - kernel(self, a) + kernel(self, val) } #[inline(always)] - fn fract_f64x4(self, a: f64x4) -> f64x4 { - a - self.trunc_f64x4(a) + fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { + mask32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn trunc_f64x4(self, a: f64x4) -> f64x4 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f64x4 { - _mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) - .simd_into(token) - } - ); - kernel(self, a) + fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - a: mask64x4, - b: f64x4, - c: f64x4, - ) -> f64x4 { - _mm256_blendv_pd(c.into(), b.into(), _mm256_castsi256_pd(a.into())).simd_into(token) - } - ); - kernel(self, a, b, c) - } - #[inline(always)] - fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { - f64x8 { - val: crate::support::Aligned512([a.val.0, b.val.0]), - simd: self, - } - } - #[inline(always)] - fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> (f64x2, f64x2) { - ( - _mm256_extractf128_pd::<0>(a.into()).simd_into(token), - _mm256_extractf128_pd::<1>(a.into()).simd_into(token), - ) - } - ); - kernel(self, a) - } - #[inline(always)] - fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f64x4) -> f32x8 { - _mm256_castpd_ps(a.into()).simd_into(token) - } - ); - kernel(self, a) - } - #[inline(always)] - fn splat_mask64x4(self, val: bool) -> mask64x4 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, val: bool) -> mask64x4 { - let val: i64 = if val { !0 } else { 0 }; - _mm256_set1_epi64x(val).simd_into(token) - } - ); - kernel(self, val) - } - #[inline(always)] - fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { - mask64x4 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { - crate::transmute::checked_transmute_copy::<__m256i, [i64; 4usize]>(&a.val.0) - } - #[inline(always)] - fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask64x4 { + fn kernel(token: Avx2, bits: u64) -> mask32x8 { { - let bit_lanes = _mm256_set1_epi64x(bits.cast_signed()); - let bit_mask = _mm256_set_epi64x(8, 4, 2, 1); - _mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) + let bit_lanes = _mm256_set1_epi32(bits as i32); + let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128); + _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) } .simd_into(token) } @@ -8120,131 +8726,142 @@ impl Simd for Avx2 { kernel(self, bits) } #[inline(always)] - fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> u64 { - _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 as u64 + fn kernel(token: Avx2, a: mask32x8) -> u64 { + _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 as u64 } ); kernel(self, a) } #[inline(always)] - fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask32x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x8(lanes); + } + #[inline(always)] + fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { _mm256_and_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { _mm256_or_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { _mm256_xor_si256(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask64x4(self, a: mask64x4) -> mask64x4 { - self.xor_mask64x4(a, self.splat_mask64x4(true)) + fn not_mask32x8(self, a: mask32x8) -> mask32x8 { + self.xor_mask32x8(a, self.splat_mask32x8(true)) } #[inline(always)] - fn select_mask64x4( + fn select_mask32x8( self, - a: mask64x4, - b: mask64x4, - c: mask64x4, - ) -> mask64x4 { + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { crate::kernel!( #[inline(always)] fn kernel( token: Avx2, - a: mask64x4, - b: mask64x4, - c: mask64x4, - ) -> mask64x4 { + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { - _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, a: mask32x8, b: mask32x8) -> mask32x8 { + _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask64x4(self, a: mask64x4) -> bool { + fn any_true_mask32x8(self, a: mask32x8) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> bool { - _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0 + fn kernel(token: Avx2, a: mask32x8) -> bool { + _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0 } ); kernel(self, a) } #[inline(always)] - fn all_true_mask64x4(self, a: mask64x4) -> bool { + fn all_true_mask32x8(self, a: mask32x8) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> bool { - _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0b1111 + fn kernel(token: Avx2, a: mask32x8) -> bool { + _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0b11111111 } ); kernel(self, a) } #[inline(always)] - fn any_false_mask64x4(self, a: mask64x4) -> bool { + fn any_false_mask32x8(self, a: mask32x8) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> bool { - _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0b1111 + fn kernel(token: Avx2, a: mask32x8) -> bool { + _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0b11111111 } ); kernel(self, a) } #[inline(always)] - fn all_false_mask64x4(self, a: mask64x4) -> bool { + fn all_false_mask32x8(self, a: mask32x8) -> bool { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> bool { - _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0 + fn kernel(token: Avx2, a: mask32x8) -> bool { + _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0 } ); kernel(self, a) } #[inline(always)] - fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { - mask64x8 { + fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { + mask32x16 { val: crate::support::Aligned512([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: mask64x4) -> (mask64x2, mask64x2) { + fn kernel(token: Avx2, a: mask32x8) -> (mask32x4, mask32x4) { ( _mm256_extracti128_si256::<0>(a.into()).simd_into(token), _mm256_extracti128_si256::<1>(a.into()).simd_into(token), @@ -8254,97 +8871,1770 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn splat_f32x16(self, val: f32) -> f32x16 { - let half = self.splat_f32x8(val); - self.combine_f32x8(half, half) + fn splat_f64x4(self, val: f64) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: f64) -> f64x4 { + _mm256_set1_pd(val).simd_into(token) + } + ); + kernel(self, val) } #[inline(always)] - fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m256; 2usize], [f32; 16usize]>(&a.val.0) + fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256d, [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { - crate::transmute::checked_cast_ref::<[__m256; 2usize], [f32; 16usize]>(&a.val.0) + fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { + crate::transmute::checked_cast_ref::<__m256d, [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { - crate::transmute::checked_cast_mut::<[__m256; 2usize], [f32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { + crate::transmute::checked_cast_mut::<__m256d, [f64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { - f32x16 { + fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { - u8x64 { + fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - if SHIFT >= 16usize { + fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_alignr_256x2( + let result = cross_block_alignr_256x1( self, - self.cvt_to_bytes_f32x16(b).val.0, - self.cvt_to_bytes_f32x16(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_f64x4(b).val.0, + self.cvt_to_bytes_f64x4(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_f32x16(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x16( + fn slide_within_blocks_f64x4( self, - a: f32x16, - b: f32x16, - ) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.slide_within_blocks_f32x8::(a0, b0), - self.slide_within_blocks_f32x8::(a1, b1), - ) - } - #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + a: f64x4, + b: f64x4, + ) -> f64x4 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_256( + self, + self.cvt_to_bytes_f64x4(b).val.0, + self.cvt_to_bytes_f64x4(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + fn abs_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + fn neg_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_sqrt_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + 1.0 / a + } + #[inline(always)] + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_add_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_sub_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_mul_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_div_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let mask = _mm256_set1_pd(-0.0); + _mm256_or_pd( + _mm256_and_pd(mask, b.into()), + _mm256_andnot_pd(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { + _mm256_castpd_si256(_mm256_cmp_pd::<0i32>(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { + _mm256_castpd_si256(_mm256_cmp_pd::<17i32>(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { + _mm256_castpd_si256(_mm256_cmp_pd::<18i32>(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { + _mm256_castpd_si256(_mm256_cmp_pd::<29i32>(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> mask64x4 { + _mm256_castpd_si256(_mm256_cmp_pd::<30i32>(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let lo = _mm256_unpacklo_pd(a.into(), b.into()); + let hi = _mm256_unpackhi_pd(a.into(), b.into()); + _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let lo = _mm256_unpacklo_pd(a.into(), b.into()); + let hi = _mm256_unpackhi_pd(a.into(), b.into()); + _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); + _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); + _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let lo = _mm256_unpacklo_pd(a.into(), b.into()); + let hi = _mm256_unpackhi_pd(a.into(), b.into()); + ( + _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into()); + ( + _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_max_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + _mm256_min_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let intermediate = _mm256_max_pd(a.into(), b.into()); + let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into()); + _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4) -> f64x4 { + let intermediate = _mm256_min_pd(a.into(), b.into()); + let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into()); + _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + _mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + _mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f64x4(self, a: f64x4) -> f64x4 { + a - self.trunc_f64x4(a) + } + #[inline(always)] + fn trunc_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask64x4, + b: f64x4, + c: f64x4, + ) -> f64x4 { + _mm256_blendv_pd(c.into(), b.into(), _mm256_castsi256_pd(a.into())).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { + f64x8 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> (f64x2, f64x2) { + ( + _mm256_extractf128_pd::<0>(a.into()).simd_into(token), + _mm256_extractf128_pd::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> f32x8 { + _mm256_castpd_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i64x4(self, val: i64) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: i64) -> i64x4 { + _mm256_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + crate::transmute::checked_cast_ref::<__m256i, [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + crate::transmute::checked_cast_mut::<__m256i, [i64; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + if SHIFT >= 4usize { + return b; + } + let result = cross_block_alignr_256x1( + self, + self.cvt_to_bytes_i64x4(b).val.0, + self.cvt_to_bytes_i64x4(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x4( + self, + a: i64x4, + b: i64x4, + ) -> i64x4 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_256( + self, + self.cvt_to_bytes_i64x4(b).val.0, + self.cvt_to_bytes_i64x4(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let result: [i64; 4usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + a[2usize].wrapping_mul(b[2usize]), + a[3usize].wrapping_mul(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i64x4(self, a: i64x4) -> i64x4 { + a ^ !0 + } + #[inline(always)] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, shift: u32) -> i64x4 { + _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + _mm256_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let a: [i64; 4usize] = a.into(); + let result: [i64; 4usize] = [ + core::ops::Shr::shr(a[0usize], shift), + core::ops::Shr::shr(a[1usize], shift), + core::ops::Shr::shr(a[2usize], shift), + core::ops::Shr::shr(a[3usize], shift), + ]; + result.simd_into(self) + } + #[inline(always)] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let result: [i64; 4usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> mask64x4 { + _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> mask64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] < b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] < b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> mask64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] <= b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] <= b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> mask64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] >= b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] >= b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> mask64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] > b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] > b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> i64x4 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask64x4, + b: i64x4, + c: i64x4, + ) -> i64x4 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let result: [i64; 4usize] = [ + a[0usize].min(b[0usize]), + a[1usize].min(b[1usize]), + a[2usize].min(b[2usize]), + a[3usize].min(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let a: [i64; 4usize] = a.into(); + let b: [i64; 4usize] = b.into(); + let result: [i64; 4usize] = [ + a[0usize].max(b[0usize]), + a[1usize].max(b[1usize]), + a[2usize].max(b[2usize]), + a[3usize].max(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + i64x8 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4) -> (i64x2, i64x2) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i64x4(self, a: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4) -> i64x4 { + _mm256_sub_epi64(_mm256_setzero_si256(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u64x4(self, val: u64) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: u64) -> u64x4 { + _mm256_set1_epi64x(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + crate::transmute::checked_cast_ref::<__m256i, [u64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + crate::transmute::checked_cast_mut::<__m256i, [u64; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + if SHIFT >= 4usize { + return b; + } + let result = cross_block_alignr_256x1( + self, + self.cvt_to_bytes_u64x4(b).val.0, + self.cvt_to_bytes_u64x4(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u64x4( + self, + a: u64x4, + b: u64x4, + ) -> u64x4 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_256( + self, + self.cvt_to_bytes_u64x4(b).val.0, + self.cvt_to_bytes_u64x4(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let result: [u64; 4usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + a[2usize].wrapping_mul(b[2usize]), + a[3usize].wrapping_mul(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u64x4(self, a: u64x4) -> u64x4 { + a ^ !0 + } + #[inline(always)] + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, shift: u32) -> u64x4 { + _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, shift: u32) -> u64x4 { + _mm256_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + _mm256_srlv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> mask64x4 { + _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> mask64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] < b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] < b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> mask64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] <= b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] <= b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> mask64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] >= b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] >= b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> mask64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 4usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + if a[2usize] > b[2usize] { + true_lane + } else { + false_lane + }, + if a[3usize] > b[3usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> u64x4 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let lo = _mm256_unpacklo_epi64(a.into(), b.into()); + let hi = _mm256_unpackhi_epi64(a.into(), b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into()); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask64x4, + b: u64x4, + c: u64x4, + ) -> u64x4 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let result: [u64; 4usize] = [ + a[0usize].min(b[0usize]), + a[1usize].min(b[1usize]), + a[2usize].min(b[2usize]), + a[3usize].min(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let a: [u64; 4usize] = a.into(); + let b: [u64; 4usize] = b.into(); + let result: [u64; 4usize] = [ + a[0usize].max(b[0usize]), + a[1usize].max(b[1usize]), + a[2usize].max(b[2usize]), + a[3usize].max(b[3usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + u64x8 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4) -> (u64x2, u64x2) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask64x4(self, val: bool) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: bool) -> mask64x4 { + let val: i64 = if val { !0 } else { 0 }; + _mm256_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { + mask64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, bits: u64) -> mask64x4 { + { + let bit_lanes = _mm256_set1_epi64x(bits.cast_signed()); + let bit_mask = _mm256_set_epi64x(8, 4, 2, 1); + _mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) + } + .simd_into(token) + } + ); + kernel(self, bits) + } + #[inline(always)] + fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> u64 { + _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 as u64 + } + ); + kernel(self, a) + } + #[inline(always)] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask64x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x4(lanes); + } + #[inline(always)] + fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_mask64x4(self, a: mask64x4) -> mask64x4 { + self.xor_mask64x4(a, self.splat_mask64x4(true)) + } + #[inline(always)] + fn select_mask64x4( + self, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4, b: mask64x4) -> mask64x4 { + _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn any_true_mask64x4(self, a: mask64x4) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> bool { + _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_true_mask64x4(self, a: mask64x4) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> bool { + _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0b1111 + } + ); + kernel(self, a) + } + #[inline(always)] + fn any_false_mask64x4(self, a: mask64x4) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> bool { + _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0b1111 + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_false_mask64x4(self, a: mask64x4) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> bool { + _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { + mask64x8 { + val: crate::support::Aligned512([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: mask64x4) -> (mask64x2, mask64x2) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_f32x16(self, val: f32) -> f32x16 { + let half = self.splat_f32x8(val); + self.combine_f32x8(half, half) + } + #[inline(always)] + fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m256; 2usize], [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { + crate::transmute::checked_cast_ref::<[__m256; 2usize], [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { + crate::transmute::checked_cast_mut::<[__m256; 2usize], [f32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_alignr_256x2( + self, + self.cvt_to_bytes_f32x16(b).val.0, + self.cvt_to_bytes_f32x16(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_f32x16( + self, + a: f32x16, + b: f32x16, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.slide_within_blocks_f32x8::(a0, b0), + self.slide_within_blocks_f32x8::(a1, b1), + ) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) } #[inline(always)] fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { @@ -9374,6 +11664,17 @@ impl Simd for Avx2 { lo | (hi << 32usize) } #[inline(always)] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let mut lanes = self.as_array_mask8x64(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x64(lanes); + } + #[inline(always)] fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { let (a0, a1) = self.split_mask8x64(a); let (b0, b1) = self.split_mask8x64(b); @@ -10146,6 +12447,17 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask16x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x32(lanes); + } + #[inline(always)] fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { let (a0, a1) = self.split_mask16x32(a); let (b0, b1) = self.split_mask16x32(b); @@ -10184,1103 +12496,1699 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16( - self.simd_eq_mask16x16(a0, b0), - self.simd_eq_mask16x16(a1, b1), + fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16( + self.simd_eq_mask16x16(a0, b0), + self.simd_eq_mask16x16(a1, b1), + ) + } + #[inline(always)] + fn any_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + } + #[inline(always)] + fn all_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + } + #[inline(always)] + fn any_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + } + #[inline(always)] + fn all_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + } + #[inline(always)] + fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + ( + mask16x16 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + mask16x16 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i32x16(self, val: i32) -> i32x16 { + let half = self.splat_i32x8(val); + self.combine_i32x8(half, half) + } + #[inline(always)] + fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { + crate::transmute::checked_cast_ref::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { + crate::transmute::checked_cast_mut::<[__m256i; 2usize], [i32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_alignr_256x2( + self, + self.cvt_to_bytes_i32x16(b).val.0, + self.cvt_to_bytes_i32x16(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i32x16( + self, + a: i32x16, + b: i32x16, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8( + self.slide_within_blocks_i32x8::(a0, b0), + self.slide_within_blocks_i32x8::(a1, b1), + ) + } + #[inline(always)] + fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) + } + #[inline(always)] + fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) + } + #[inline(always)] + fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) + } + #[inline(always)] + fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) + } + #[inline(always)] + fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) + } + #[inline(always)] + fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) + } + #[inline(always)] + fn not_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) + } + #[inline(always)] + fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) + } + #[inline(always)] + fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) + } + #[inline(always)] + fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) + } + #[inline(always)] + fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, _) = self.split_i32x16(a); + let (b0, _) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) + } + #[inline(always)] + fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (_, a1) = self.split_i32x16(a); + let (_, b1) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + } + #[inline(always)] + fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + } + #[inline(always)] + fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_lo = self.zip_low_i32x8(a0, b0); + let lo_hi = self.zip_high_i32x8(a0, b0); + let hi_lo = self.zip_low_i32x8(a1, b1); + let hi_hi = self.zip_high_i32x8(a1, b1); + ( + self.combine_i32x8(lo_lo, lo_hi), + self.combine_i32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn any_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_even = self.unzip_low_i32x8(a0, a1); + let lo_odd = self.unzip_high_i32x8(a0, a1); + let hi_even = self.unzip_low_i32x8(b0, b1); + let hi_odd = self.unzip_high_i32x8(b0, b1); + ( + self.combine_i32x8(lo_even, hi_even), + self.combine_i32x8(lo_odd, hi_odd), + ) } #[inline(always)] - fn all_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_i32x16(b); + let (c0, c1) = self.split_i32x16(c); + self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) } #[inline(always)] - fn any_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) } #[inline(always)] - fn all_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) } #[inline(always)] - fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( - mask16x16 { + i32x8 { val: crate::support::Aligned256(a.val.0[0]), simd: self, }, - mask16x16 { + i32x8 { val: crate::support::Aligned256(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_i32x16(self, val: i32) -> i32x16 { - let half = self.splat_i32x8(val); - self.combine_i32x8(half, half) + fn neg_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { - i32x16 { + fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { + let (a0, a1) = self.split_i32x16(a); + self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) + } + #[inline(always)] + fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_u32x8( + self.reinterpret_u32_i32x8(a0), + self.reinterpret_u32_i32x8(a1), + ) + } + #[inline(always)] + fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) + } + #[inline(always)] + fn splat_u32x16(self, val: u32) -> u32x16 { + let half = self.splat_u32x8(val); + self.combine_u32x8(half, half) + } + #[inline(always)] + fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { - i32x16 { + fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) + fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { - crate::transmute::checked_cast_ref::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) + fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { + crate::transmute::checked_cast_ref::<[__m256i; 2usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { - crate::transmute::checked_cast_mut::<[__m256i; 2usize], [i32; 16usize]>(&mut a.val.0) + fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { + crate::transmute::checked_cast_mut::<[__m256i; 2usize], [u32; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { - i32x16 { + fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { if SHIFT >= 16usize { return b; } let result = cross_block_alignr_256x2( self, - self.cvt_to_bytes_i32x16(b).val.0, - self.cvt_to_bytes_i32x16(a).val.0, + self.cvt_to_bytes_u32x16(b).val.0, + self.cvt_to_bytes_u32x16(a).val.0, SHIFT * 4usize, ); - self.cvt_from_bytes_i32x16(u8x64 { + self.cvt_from_bytes_u32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x16( - self, - a: i32x16, - b: i32x16, - ) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8( - self.slide_within_blocks_i32x8::(a0, b0), - self.slide_within_blocks_i32x8::(a1, b1), + fn slide_within_blocks_u32x16( + self, + a: u32x16, + b: u32x16, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8( + self.slide_within_blocks_u32x8::(a0, b0), + self.slide_within_blocks_u32x8::(a1, b1), + ) + } + #[inline(always)] + fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) + } + #[inline(always)] + fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) + } + #[inline(always)] + fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) + } + #[inline(always)] + fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) + } + #[inline(always)] + fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) + } + #[inline(always)] + fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) + } + #[inline(always)] + fn not_u32x16(self, a: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) + } + #[inline(always)] + fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) + } + #[inline(always)] + fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) + } + #[inline(always)] + fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) + } + #[inline(always)] + fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, _) = self.split_u32x16(a); + let (b0, _) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) + } + #[inline(always)] + fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (_, a1) = self.split_u32x16(a); + let (_, b1) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) + } + #[inline(always)] + fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + } + #[inline(always)] + fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_lo = self.zip_low_u32x8(a0, b0); + let lo_hi = self.zip_high_u32x8(a0, b0); + let hi_lo = self.zip_low_u32x8(a1, b1); + let hi_hi = self.zip_high_u32x8(a1, b1); + ( + self.combine_u32x8(lo_lo, lo_hi), + self.combine_u32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) + fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_even = self.unzip_low_u32x8(a0, a1); + let lo_odd = self.unzip_high_u32x8(a0, a1); + let hi_even = self.unzip_low_u32x8(b0, b1); + let hi_odd = self.unzip_high_u32x8(b0, b1); + ( + self.combine_u32x8(lo_even, hi_even), + self.combine_u32x8(lo_odd, hi_odd), + ) } #[inline(always)] - fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) + fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_u32x16(b); + let (c0, c1) = self.split_u32x16(c); + self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) } #[inline(always)] - fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) + fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) } #[inline(always)] - fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) + fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) } #[inline(always)] - fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) + fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { + ( + u32x8 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + u32x8 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] - fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) + fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u32; 16usize]) -> u32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_u32x8( + token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) } #[inline(always)] - fn not_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) + fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x16, dest: &mut [u32; 16usize]) -> () { + let (v01, v23) = token.split_u32x16(a); + let (v0, v1) = token.split_u32x8(v01); + let (v2, v3) = token.split_u32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); } #[inline(always)] - fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) + fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) } #[inline(always)] - fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) + fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) } #[inline(always)] - fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) + fn splat_mask32x16(self, val: bool) -> mask32x16 { + let half = self.splat_mask32x8(val); + self.combine_mask32x8(half, half) } #[inline(always)] - fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) + fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { + mask32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) + fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) } #[inline(always)] - fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) + fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, bits: u64) -> mask32x16 { + { + let bit_lanes = _mm256_set1_epi32(bits as i32); + mask32x16 { + val: crate::support::Aligned512([ + { + let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128); + _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) + }, + { + let bit_mask = _mm256_setr_epi32( + 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, + ); + _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) + }, + ]), + simd: token, + } + } + } + ); + kernel(self, bits) } #[inline(always)] - fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) + fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { + let (lo, hi) = self.split_mask32x16(a); + let lo = self.to_bitmask_mask32x8(lo); + let hi = self.to_bitmask_mask32x8(hi); + lo | (hi << 8usize) } #[inline(always)] - fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask32x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x16(lanes); } #[inline(always)] - fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) + fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) } #[inline(always)] - fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, _) = self.split_i32x16(a); - let (b0, _) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) + fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) } #[inline(always)] - fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (_, a1) = self.split_i32x16(a); - let (_, b1) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) } #[inline(always)] - fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + fn not_mask32x16(self, a: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) } #[inline(always)] - fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + fn select_mask32x16( + self, + a: mask32x16, + b: mask32x16, + c: mask32x16, + ) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + let (c0, c1) = self.split_mask32x16(c); + self.combine_mask32x8( + self.select_mask32x8(a0, b0, c0), + self.select_mask32x8(a1, b1, c1), + ) } #[inline(always)] - fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_lo = self.zip_low_i32x8(a0, b0); - let lo_hi = self.zip_high_i32x8(a0, b0); - let hi_lo = self.zip_low_i32x8(a1, b1); - let hi_hi = self.zip_high_i32x8(a1, b1); - ( - self.combine_i32x8(lo_lo, lo_hi), - self.combine_i32x8(hi_lo, hi_hi), - ) + fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) } #[inline(always)] - fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_even = self.unzip_low_i32x8(a0, a1); - let lo_odd = self.unzip_high_i32x8(a0, a1); - let hi_even = self.unzip_low_i32x8(b0, b1); - let hi_odd = self.unzip_high_i32x8(b0, b1); - ( - self.combine_i32x8(lo_even, hi_even), - self.combine_i32x8(lo_odd, hi_odd), - ) + fn any_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) } #[inline(always)] - fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + fn all_true_mask32x16(self, a: mask32x16) -> bool { let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_i32x16(b); - let (c0, c1) = self.split_i32x16(c); - self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) + self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) } #[inline(always)] - fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) + fn any_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) } #[inline(always)] - fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + fn all_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) } #[inline(always)] - fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { + fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { ( - i32x8 { + mask32x8 { val: crate::support::Aligned256(a.val.0[0]), simd: self, }, - i32x8 { + mask32x8 { val: crate::support::Aligned256(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u32x8( - self.reinterpret_u32_i32x8(a0), - self.reinterpret_u32_i32x8(a1), - ) - } - #[inline(always)] - fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) - } - #[inline(always)] - fn splat_u32x16(self, val: u32) -> u32x16 { - let half = self.splat_u32x8(val); - self.combine_u32x8(half, half) + fn splat_f64x8(self, val: f64) -> f64x8 { + let half = self.splat_f64x4(val); + self.combine_f64x4(half, half) } #[inline(always)] - fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [u32; 16usize]>(&a.val.0) + fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m256d; 2usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { - crate::transmute::checked_cast_ref::<[__m256i; 2usize], [u32; 16usize]>(&a.val.0) + fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { + crate::transmute::checked_cast_ref::<[__m256d; 2usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { - crate::transmute::checked_cast_mut::<[__m256i; 2usize], [u32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { + crate::transmute::checked_cast_mut::<[__m256d; 2usize], [f64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { - u32x16 { + fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { + fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - if SHIFT >= 16usize { + fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_256x2( self, - self.cvt_to_bytes_u32x16(b).val.0, - self.cvt_to_bytes_u32x16(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_f64x8(b).val.0, + self.cvt_to_bytes_f64x8(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_u32x16(u8x64 { + self.cvt_from_bytes_f64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u32x16( + fn slide_within_blocks_f64x8( self, - a: u32x16, - b: u32x16, - ) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8( - self.slide_within_blocks_u32x8::(a0, b0), - self.slide_within_blocks_u32x8::(a1, b1), + a: f64x8, + b: f64x8, + ) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.slide_within_blocks_f64x4::(a0, b0), + self.slide_within_blocks_f64x4::(a1, b1), ) } #[inline(always)] - fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) - } - #[inline(always)] - fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) - } - #[inline(always)] - fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) - } - #[inline(always)] - fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) - } - #[inline(always)] - fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) - } - #[inline(always)] - fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) - } - #[inline(always)] - fn not_u32x16(self, a: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) - } - #[inline(always)] - fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) - } - #[inline(always)] - fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) - } - #[inline(always)] - fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) - } - #[inline(always)] - fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + fn abs_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) } #[inline(always)] - fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, _) = self.split_u32x16(a); - let (b0, _) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) + fn neg_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) } #[inline(always)] - fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (_, a1) = self.split_u32x16(a); - let (_, b1) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) + fn sqrt_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) } #[inline(always)] - fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) + fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.approximate_recip_f64x4(a0), + self.approximate_recip_f64x4(a1), + ) } #[inline(always)] - fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) } #[inline(always)] - fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_lo = self.zip_low_u32x8(a0, b0); - let lo_hi = self.zip_high_u32x8(a0, b0); - let hi_lo = self.zip_low_u32x8(a1, b1); - let hi_hi = self.zip_high_u32x8(a1, b1); - ( - self.combine_u32x8(lo_lo, lo_hi), - self.combine_u32x8(hi_lo, hi_hi), - ) + fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) } #[inline(always)] - fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_even = self.unzip_low_u32x8(a0, a1); - let lo_odd = self.unzip_high_u32x8(a0, a1); - let hi_even = self.unzip_low_u32x8(b0, b1); - let hi_odd = self.unzip_high_u32x8(b0, b1); - ( - self.combine_u32x8(lo_even, hi_even), - self.combine_u32x8(lo_odd, hi_odd), - ) + fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) } #[inline(always)] - fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_u32x16(b); - let (c0, c1) = self.split_u32x16(c); - self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) + fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) } #[inline(always)] - fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) + fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) } #[inline(always)] - fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) + fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) } #[inline(always)] - fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { - ( - u32x8 { - val: crate::support::Aligned256(a.val.0[0]), - simd: self, - }, - u32x8 { - val: crate::support::Aligned256(a.val.0[1]), - simd: self, - }, - ) + fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) + fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) } #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) } #[inline(always)] - fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) + fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) } #[inline(always)] - fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) + fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, _) = self.split_f64x8(a); + let (b0, _) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) } #[inline(always)] - fn splat_mask32x16(self, val: bool) -> mask32x16 { - let half = self.splat_mask32x8(val); - self.combine_mask32x8(half, half) + fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (_, a1) = self.split_f64x8(a); + let (_, b1) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) } #[inline(always)] - fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { - mask32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) } #[inline(always)] - fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [i32; 16usize]>(&a.val.0) + fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) } #[inline(always)] - fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, bits: u64) -> mask32x16 { - { - let bit_lanes = _mm256_set1_epi32(bits as i32); - mask32x16 { - val: crate::support::Aligned512([ - { - let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128); - _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) - }, - { - let bit_mask = _mm256_setr_epi32( - 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, - ); - _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask) - }, - ]), - simd: token, - } - } - } - ); - kernel(self, bits) + fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_lo = self.zip_low_f64x4(a0, b0); + let lo_hi = self.zip_high_f64x4(a0, b0); + let hi_lo = self.zip_low_f64x4(a1, b1); + let hi_hi = self.zip_high_f64x4(a1, b1); + ( + self.combine_f64x4(lo_lo, lo_hi), + self.combine_f64x4(hi_lo, hi_hi), + ) } #[inline(always)] - fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { - let (lo, hi) = self.split_mask32x16(a); - let lo = self.to_bitmask_mask32x8(lo); - let hi = self.to_bitmask_mask32x8(hi); - lo | (hi << 8usize) + fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_even = self.unzip_low_f64x4(a0, a1); + let lo_odd = self.unzip_high_f64x4(a0, a1); + let hi_even = self.unzip_low_f64x4(b0, b1); + let hi_odd = self.unzip_high_f64x4(b0, b1); + ( + self.combine_f64x4(lo_even, hi_even), + self.combine_f64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) + fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) } #[inline(always)] - fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) + fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) } #[inline(always)] - fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) + fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.max_precise_f64x4(a0, b0), + self.max_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn not_mask32x16(self, a: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.min_precise_f64x4(a0, b0), + self.min_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn select_mask32x16( - self, - a: mask32x16, - b: mask32x16, - c: mask32x16, - ) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - let (c0, c1) = self.split_mask32x16(c); - self.combine_mask32x8( - self.select_mask32x8(a0, b0, c0), - self.select_mask32x8(a1, b1, c1), + fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_add_f64x4(a0, b0, c0), + self.mul_add_f64x4(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) + fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_sub_f64x4(a0, b0, c0), + self.mul_sub_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn any_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) + fn floor_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) } #[inline(always)] - fn all_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) + fn ceil_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) } #[inline(always)] - fn any_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) + fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.round_ties_even_f64x4(a0), + self.round_ties_even_f64x4(a1), + ) } #[inline(always)] - fn all_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) + fn fract_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) } #[inline(always)] - fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { + fn trunc_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) + } + #[inline(always)] + fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) + } + #[inline(always)] + fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { ( - mask32x8 { + f64x4 { val: crate::support::Aligned256(a.val.0[0]), simd: self, }, - mask32x8 { + f64x4 { val: crate::support::Aligned256(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_f64x8(self, val: f64) -> f64x8 { - let half = self.splat_f64x4(val); - self.combine_f64x4(half, half) + fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f32x8( + self.reinterpret_f32_f64x4(a0), + self.reinterpret_f32_f64x4(a1), + ) } #[inline(always)] - fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { - f64x8 { + fn splat_i64x8(self, val: i64) -> i64x8 { + let half = self.splat_i64x4(val); + self.combine_i64x4(half, half) + } + #[inline(always)] + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { - f64x8 { + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { - crate::transmute::checked_transmute_copy::<[__m256d; 2usize], [f64; 8usize]>(&a.val.0) + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { - crate::transmute::checked_cast_ref::<[__m256d; 2usize], [f64; 8usize]>(&a.val.0) + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + crate::transmute::checked_cast_ref::<[__m256i; 2usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { - crate::transmute::checked_cast_mut::<[__m256d; 2usize], [f64; 8usize]>(&mut a.val.0) + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + crate::transmute::checked_cast_mut::<[__m256i; 2usize], [i64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { - f64x8 { + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { if SHIFT >= 8usize { return b; } let result = cross_block_alignr_256x2( self, - self.cvt_to_bytes_f64x8(b).val.0, - self.cvt_to_bytes_f64x8(a).val.0, + self.cvt_to_bytes_i64x8(b).val.0, + self.cvt_to_bytes_i64x8(a).val.0, SHIFT * 8usize, ); - self.cvt_from_bytes_f64x8(u8x64 { + self.cvt_from_bytes_i64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x8( + fn slide_within_blocks_i64x8( self, - a: f64x8, - b: f64x8, - ) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.slide_within_blocks_f64x4::(a0, b0), - self.slide_within_blocks_f64x4::(a1, b1), + a: i64x8, + b: i64x8, + ) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4( + self.slide_within_blocks_i64x4::(a0, b0), + self.slide_within_blocks_i64x4::(a1, b1), + ) + } + #[inline(always)] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), ) } #[inline(always)] - fn abs_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m256i; 2usize], [u64; 8usize]>(&a.val.0) } #[inline(always)] - fn neg_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::<[__m256i; 2usize], [u64; 8usize]>(&a.val.0) } #[inline(always)] - fn sqrt_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::<[__m256i; 2usize], [u64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.approximate_recip_f64x4(a0), - self.approximate_recip_f64x4(a1), - ) + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_alignr_256x2( + self, + self.cvt_to_bytes_u64x8(b).val.0, + self.cvt_to_bytes_u64x8(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), + ) } #[inline(always)] - fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.add_u64x4(a0, b0), self.add_u64x4(a1, b1)) } #[inline(always)] - fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.sub_u64x4(a0, b0), self.sub_u64x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1)) } #[inline(always)] - fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.and_u64x4(a0, b0), self.and_u64x4(a1, b1)) } #[inline(always)] - fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.or_u64x4(a0, b0), self.or_u64x4(a1, b1)) } #[inline(always)] - fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.xor_u64x4(a0, b0), self.xor_u64x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, _) = self.split_f64x8(a); - let (b0, _) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) + fn not_u64x8(self, a: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.not_u64x4(a0), self.not_u64x4(a1)) } #[inline(always)] - fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (_, a1) = self.split_f64x8(a); - let (_, b1) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shl_u64x4(a0, shift), self.shl_u64x4(a1, shift)) } #[inline(always)] - fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shlv_u64x4(a0, b0), self.shlv_u64x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shr_u64x4(a0, shift), self.shr_u64x4(a1, shift)) } #[inline(always)] - fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_lo = self.zip_low_f64x4(a0, b0); - let lo_hi = self.zip_high_f64x4(a0, b0); - let hi_lo = self.zip_low_f64x4(a1, b1); - let hi_hi = self.zip_high_f64x4(a1, b1); - ( - self.combine_f64x4(lo_lo, lo_hi), - self.combine_f64x4(hi_lo, hi_hi), - ) + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shrv_u64x4(a0, b0), self.shrv_u64x4(a1, b1)) } #[inline(always)] - fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_even = self.unzip_low_f64x4(a0, a1); - let lo_odd = self.unzip_high_f64x4(a0, a1); - let hi_even = self.unzip_low_f64x4(b0, b1); - let hi_odd = self.unzip_high_f64x4(b0, b1); - ( - self.combine_f64x4(lo_even, hi_even), - self.combine_f64x4(lo_odd, hi_odd), - ) + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_eq_u64x4(a0, b0), self.simd_eq_u64x4(a1, b1)) } #[inline(always)] - fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_lt_u64x4(a0, b0), self.simd_lt_u64x4(a1, b1)) } #[inline(always)] - fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_le_u64x4(a0, b0), self.simd_le_u64x4(a1, b1)) } #[inline(always)] - fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.max_precise_f64x4(a0, b0), - self.max_precise_f64x4(a1, b1), - ) + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_ge_u64x4(a0, b0), self.simd_ge_u64x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.min_precise_f64x4(a0, b0), - self.min_precise_f64x4(a1, b1), - ) + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_gt_u64x4(a0, b0), self.simd_gt_u64x4(a1, b1)) } #[inline(always)] - fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_add_f64x4(a0, b0, c0), - self.mul_add_f64x4(a1, b1, c1), - ) + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, _) = self.split_u64x8(a); + let (b0, _) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a0, b0), self.zip_high_u64x4(a0, b0)) } #[inline(always)] - fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_sub_f64x4(a0, b0, c0), - self.mul_sub_f64x4(a1, b1, c1), - ) + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (_, a1) = self.split_u64x8(a); + let (_, b1) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a1, b1), self.zip_high_u64x4(a1, b1)) } #[inline(always)] - fn floor_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_low_u64x4(a0, a1), self.unzip_low_u64x4(b0, b1)) } #[inline(always)] - fn ceil_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_high_u64x4(a0, a1), self.unzip_high_u64x4(b0, b1)) } #[inline(always)] - fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.round_ties_even_f64x4(a0), - self.round_ties_even_f64x4(a1), + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_lo = self.zip_low_u64x4(a0, b0); + let lo_hi = self.zip_high_u64x4(a0, b0); + let hi_lo = self.zip_low_u64x4(a1, b1); + let hi_hi = self.zip_high_u64x4(a1, b1); + ( + self.combine_u64x4(lo_lo, lo_hi), + self.combine_u64x4(hi_lo, hi_hi), ) } #[inline(always)] - fn fract_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_even = self.unzip_low_u64x4(a0, a1); + let lo_odd = self.unzip_high_u64x4(a0, a1); + let hi_even = self.unzip_low_u64x4(b0, b1); + let hi_odd = self.unzip_high_u64x4(b0, b1); + ( + self.combine_u64x4(lo_even, hi_even), + self.combine_u64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn trunc_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_u64x8(b); + let (c0, c1) = self.split_u64x8(c); + self.combine_u64x4(self.select_u64x4(a0, b0, c0), self.select_u64x4(a1, b1, c1)) } #[inline(always)] - fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.min_u64x4(a0, b0), self.min_u64x4(a1, b1)) } #[inline(always)] - fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.max_u64x4(a0, b0), self.max_u64x4(a1, b1)) + } + #[inline(always)] + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( - f64x4 { + u64x4 { val: crate::support::Aligned256(a.val.0[0]), simd: self, }, - f64x4 { + u64x4 { val: crate::support::Aligned256(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f32x8( - self.reinterpret_f32_f64x4(a0), - self.reinterpret_f32_f64x4(a1), + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u64; 8usize]) -> u64x8 { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + token.combine_u64x4(out0.simd_into(token), out1.simd_into(token)) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x8, dest: &mut [u64; 8usize]) -> () { + let (v0, v1) = token.split_u64x8(a); + let v0: __m256i = v0.into(); + let v1: __m256i = v1.into(); + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u8x32(self.reinterpret_u8_u64x4(a0), self.reinterpret_u8_u64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u32x8( + self.reinterpret_u32_u64x4(a0), + self.reinterpret_u32_u64x4(a1), ) } #[inline(always)] @@ -11332,6 +14240,17 @@ impl Simd for Avx2 { lo | (hi << 4usize) } #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask64x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x8(lanes); + } + #[inline(always)] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { let (a0, a1) = self.split_mask64x8(a); let (b0, b1) = self.split_mask64x8(b); @@ -11457,16 +14376,15 @@ impl From> for __m256i { impl SimdFrom<__m256i, S> for mask8x32 { #[inline(always)] fn simd_from(simd: S, arch: __m256i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i8; 32usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m256i { #[inline(always)] fn from(value: mask8x32) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i8; 32usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m256i, S> for i16x16 { @@ -11502,16 +14420,15 @@ impl From> for __m256i { impl SimdFrom<__m256i, S> for mask16x16 { #[inline(always)] fn simd_from(simd: S, arch: __m256i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i16; 16usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m256i { #[inline(always)] fn from(value: mask16x16) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i16; 16usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m256i, S> for i32x8 { @@ -11547,16 +14464,15 @@ impl From> for __m256i { impl SimdFrom<__m256i, S> for mask32x8 { #[inline(always)] fn simd_from(simd: S, arch: __m256i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i32; 8usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m256i { #[inline(always)] fn from(value: mask32x8) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i32; 8usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m256d, S> for f64x4 { @@ -11574,7 +14490,22 @@ impl From> for __m256d { crate::transmute::checked_transmute_copy(&value.val) } } -impl SimdFrom<__m256i, S> for mask64x4 { +impl SimdFrom<__m256i, S> for i64x4 { + #[inline(always)] + fn simd_from(simd: S, arch: __m256i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m256i { + #[inline(always)] + fn from(value: i64x4) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m256i, S> for u64x4 { #[inline(always)] fn simd_from(simd: S, arch: __m256i) -> Self { Self { @@ -11583,10 +14514,24 @@ impl SimdFrom<__m256i, S> for mask64x4 { } } } +impl From> for __m256i { + #[inline(always)] + fn from(value: u64x4) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m256i, S> for mask64x4 { + #[inline(always)] + fn simd_from(simd: S, arch: __m256i) -> Self { + let lanes: [i64; 4usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) + } +} impl From> for __m256i { #[inline(always)] fn from(value: mask64x4) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i64; 4usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } crate::kernel!( diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs new file mode 100644 index 000000000..77e3e5025 --- /dev/null +++ b/fearless_simd/src/generated/avx512.rs @@ -0,0 +1,16726 @@ +// Copyright 2025 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// This file is autogenerated by fearless_simd_gen + +#![allow( + clippy::identity_op, + reason = "AVX-512 mask code is generated uniformly for all __mmask widths" +)] +#![allow( + clippy::useless_conversion, + reason = "AVX-512 mask code is generated uniformly for all __mmask widths" +)] +use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; +use crate::{ + f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, +}; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; +#[doc = "A token for AVX-512 intrinsics on `x86` and `x86_64`, representing an Ice Lake feature level."] +#[derive(Clone, Copy, Debug)] +pub struct Avx512 { + _private: (), +} +impl Avx512 { + #[doc = r" Create a SIMD token."] + #[doc = r""] + #[doc = r" # Safety"] + #[doc = r""] + #[doc = r" The Ice Lake AVX-512 CPU feature set must be available."] + #[inline] + pub const unsafe fn new_unchecked() -> Self { + Self { _private: () } + } +} +impl Seal for Avx512 {} +impl ArchTypes for Avx512 { + type f32x4 = crate::support::Aligned128<__m128>; + type i8x16 = crate::support::Aligned128<__m128i>; + type u8x16 = crate::support::Aligned128<__m128i>; + type mask8x16 = __mmask16; + type i16x8 = crate::support::Aligned128<__m128i>; + type u16x8 = crate::support::Aligned128<__m128i>; + type mask16x8 = __mmask8; + type i32x4 = crate::support::Aligned128<__m128i>; + type u32x4 = crate::support::Aligned128<__m128i>; + type mask32x4 = __mmask8; + type f64x2 = crate::support::Aligned128<__m128d>; + type i64x2 = crate::support::Aligned128<__m128i>; + type u64x2 = crate::support::Aligned128<__m128i>; + type mask64x2 = __mmask8; + type f32x8 = crate::support::Aligned256<__m256>; + type i8x32 = crate::support::Aligned256<__m256i>; + type u8x32 = crate::support::Aligned256<__m256i>; + type mask8x32 = __mmask32; + type i16x16 = crate::support::Aligned256<__m256i>; + type u16x16 = crate::support::Aligned256<__m256i>; + type mask16x16 = __mmask16; + type i32x8 = crate::support::Aligned256<__m256i>; + type u32x8 = crate::support::Aligned256<__m256i>; + type mask32x8 = __mmask8; + type f64x4 = crate::support::Aligned256<__m256d>; + type i64x4 = crate::support::Aligned256<__m256i>; + type u64x4 = crate::support::Aligned256<__m256i>; + type mask64x4 = __mmask8; + type f32x16 = crate::support::Aligned512<__m512>; + type i8x64 = crate::support::Aligned512<__m512i>; + type u8x64 = crate::support::Aligned512<__m512i>; + type mask8x64 = __mmask64; + type i16x32 = crate::support::Aligned512<__m512i>; + type u16x32 = crate::support::Aligned512<__m512i>; + type mask16x32 = __mmask32; + type i32x16 = crate::support::Aligned512<__m512i>; + type u32x16 = crate::support::Aligned512<__m512i>; + type mask32x16 = __mmask16; + type f64x8 = crate::support::Aligned512<__m512d>; + type i64x8 = crate::support::Aligned512<__m512i>; + type u64x8 = crate::support::Aligned512<__m512i>; + type mask64x8 = __mmask8; +} +impl Simd for Avx512 { + type f32s = f32x16; + type f64s = f64x8; + type u8s = u8x64; + type i8s = i8x64; + type u16s = u16x32; + type i16s = i16x32; + type u32s = u32x16; + type i32s = i32x16; + type u64s = u64x8; + type i64s = i64x8; + type mask8s = mask8x64; + type mask16s = mask16x32; + type mask32s = mask32x16; + type mask64s = mask64x8; + #[inline(always)] + fn level(self) -> Level { + Level::Avx512(self) + } + #[inline] + fn vectorize R, R>(self, f: F) -> R { + #[target_feature( + enable = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves" + )] + fn vectorize_avx512 R, R>(f: F) -> R { + f() + } + unsafe { vectorize_avx512(f) } + } + #[inline(always)] + fn splat_f32x4(self, val: f32) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f32) -> f32x4 { + _mm_set1_ps(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f32x4(self, val: [f32; 4usize]) -> f32x4 { + f32x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x4(self, val: &[f32; 4usize]) -> f32x4 { + f32x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x4(self, a: f32x4) -> [f32; 4usize] { + crate::transmute::checked_transmute_copy::<__m128, [f32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x4(self, a: &f32x4) -> &[f32; 4usize] { + crate::transmute::checked_cast_ref::<__m128, [f32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x4(self, a: &mut f32x4) -> &mut [f32; 4usize] { + crate::transmute::checked_cast_mut::<__m128, [f32; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x4(self, a: f32x4, dest: &mut [f32; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x4(self, a: u8x16) -> f32x4 { + f32x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x4(self, a: f32x4) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + if SHIFT >= 4usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_f32x4(b).val.0, + self.cvt_to_bytes_f32x4(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_f32x4( + self, + a: f32x4, + b: f32x4, + ) -> f32x4 { + self.slide_f32x4::(a, b) + } + #[inline(always)] + fn abs_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_sqrt_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_rcp14_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_add_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_sub_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_mul_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_div_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + let mask = _mm_set1_ps(-0.0); + _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into())) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f32x4(self, a: f32x4, b: f32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmp_ps_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f32x4(self, a: f32x4, b: f32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmp_ps_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f32x4(self, a: f32x4, b: f32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmp_ps_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f32x4(self, a: f32x4, b: f32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmp_ps_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f32x4(self, a: f32x4, b: f32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmp_ps_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_unpacklo_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_unpackhi_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f32x4(self, a: f32x4, b: f32x4) -> (f32x4, f32x4) { + (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b)) + } + #[inline(always)] + fn deinterleave_f32x4(self, a: f32x4, b: f32x4) -> (f32x4, f32x4) { + (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b)) + } + #[inline(always)] + fn max_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_max_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_min_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_range_ps::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f32x4(self, a: f32x4, b: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x4 { + _mm_range_ps::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x4, + b: f32x4, + c: f32x4, + ) -> f32x4 { + _mm_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x4, + b: f32x4, + c: f32x4, + ) -> f32x4 { + _mm_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f32x4(self, a: f32x4) -> f32x4 { + a - self.trunc_f32x4(a) + } + #[inline(always)] + fn trunc_f32x4(self, a: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f32x4 { + _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f32x4(self, a: mask32x4, b: f32x4, c: f32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x4, + b: f32x4, + c: f32x4, + ) -> f32x4 { + _mm_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f32x4(self, a: f32x4, b: f32x4) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, b: f32x4) -> f32x8 { + _mm256_setr_m128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn reinterpret_f64_f32x4(self, a: f32x4) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> f64x2 { + _mm_castps_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_i32_f32x4(self, a: f32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> i32x4 { + _mm_castps_si128(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_f32x4(self, a: f32x4) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> u8x16 { + _mm_castps_si128(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_f32x4(self, a: f32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> u32x4 { + _mm_castps_si128(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> u32x4 { + _mm_cvttps_epu32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_precise_f32x4(self, a: f32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> u32x4 { + let a = _mm_max_ps(a.into(), _mm_setzero_ps()); + let mut converted = _mm_cvttps_epu32(a); + let exceeds_unsigned_range = _mm_cmp_ps_mask::<17i32>(_mm_set1_ps(4294967040.0), a); + converted = _mm_mask_blend_epi32( + exceeds_unsigned_range, + converted, + _mm_set1_epi32(u32::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_f32x4(self, a: f32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> i32x4 { + _mm_cvttps_epi32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_precise_f32x4(self, a: f32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4) -> i32x4 { + let a = a.into(); + let in_range = _mm_cmp_ps_mask::<17i32>(a, _mm_set1_ps(2147483648.0)); + let mut converted = _mm_mask_cvttps_epi32(_mm_set1_epi32(i32::MAX), in_range, a); + let is_not_nan = _mm_cmp_ps_mask::<7i32>(a, a); + converted = _mm_mask_blend_epi32(is_not_nan, _mm_setzero_si128(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i8x16(self, val: i8) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i8) -> i8x16 { + _mm_set1_epi8(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i8x16(self, val: [i8; 16usize]) -> i8x16 { + i8x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x16(self, val: &[i8; 16usize]) -> i8x16 { + i8x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x16(self, a: i8x16) -> [i8; 16usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i8; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x16(self, a: &i8x16) -> &[i8; 16usize] { + crate::transmute::checked_cast_ref::<__m128i, [i8; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x16(self, a: &mut i8x16) -> &mut [i8; 16usize] { + crate::transmute::checked_cast_mut::<__m128i, [i8; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x16(self, a: i8x16, dest: &mut [i8; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i8x16(self, a: u8x16) -> i8x16 { + i8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i8x16(self, a: i8x16) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + if SHIFT >= 16usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i8x16(b).val.0, + self.cvt_to_bytes_i8x16(a).val.0, + SHIFT, + ); + self.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i8x16( + self, + a: i8x16, + b: i8x16, + ) -> i8x16 { + self.slide_i8x16::(a, b) + } + #[inline(always)] + fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + let dst_even = _mm_mullo_epi16(a.into(), b.into()); + let dst_odd = + _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into())); + _mm_or_si128( + _mm_slli_epi16(dst_odd, 8), + _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i8x16(self, a: i8x16) -> i8x16 { + a ^ !0 + } + #[inline(always)] + fn shl_i8x16(self, a: i8x16, shift: u32) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, shift: u32) -> i8x16 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val)); + let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val)); + let lo_shifted = _mm_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm_sll_epi16(hi_16, shift_count); + _mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + let val = a.into(); + let counts = b.into(); + let zero = _mm_setzero_si128(); + let value_extend = zero; + let lo_values = _mm_unpacklo_epi8(val, value_extend); + let hi_values = _mm_unpackhi_epi8(val, value_extend); + let lo_counts = _mm_unpacklo_epi8(counts, zero); + let hi_counts = _mm_unpackhi_epi8(counts, zero); + let byte_mask = _mm_set1_epi16(0x00ff); + let lo_shifted = _mm_and_si128(_mm_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = _mm_and_si128(_mm_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i8x16(self, a: i8x16, shift: u32) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, shift: u32) -> i8x16 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val)); + let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val)); + let lo_shifted = _mm_sra_epi16(lo_16, shift_count); + let hi_shifted = _mm_sra_epi16(hi_16, shift_count); + _mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + let val = a.into(); + let counts = b.into(); + let zero = _mm_setzero_si128(); + let value_extend = _mm_cmpgt_epi8(zero, val); + let lo_values = _mm_unpacklo_epi8(val, value_extend); + let hi_values = _mm_unpackhi_epi8(val, value_extend); + let lo_counts = _mm_unpacklo_epi8(counts, zero); + let hi_counts = _mm_unpackhi_epi8(counts, zero); + let byte_mask = _mm_set1_epi16(0x00ff); + let lo_shifted = _mm_and_si128(_mm_srav_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = _mm_and_si128(_mm_srav_epi16(hi_values, hi_counts), byte_mask); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpeq_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmplt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmple_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpge_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpgt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_permutex2var_epi8( + a.into(), + _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_permutex2var_epi8( + a.into(), + _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i8x16(self, a: i8x16, b: i8x16) -> (i8x16, i8x16) { + (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b)) + } + #[inline(always)] + fn deinterleave_i8x16(self, a: i8x16, b: i8x16) -> (i8x16, i8x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x16, + b: i8x16, + ) -> (i8x16, i8x16) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi8( + a, + _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b, + ) + .simd_into(token), + _mm_permutex2var_epi8( + a, + _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i8x16(self, a: mask8x16, b: i8x16, c: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x16, + b: i8x16, + c: i8x16, + ) -> i8x16 { + _mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_min_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x16 { + _mm_max_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i8x16(self, a: i8x16, b: i8x16) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, b: i8x16) -> i8x32 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn neg_i8x16(self, a: i8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16) -> i8x16 { + _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i8x16(self, a: i8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i8x16(self, a: i8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u8x16(self, val: u8) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u8) -> u8x16 { + _mm_set1_epi8(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u8x16(self, val: [u8; 16usize]) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x16(self, val: &[u8; 16usize]) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x16(self, a: u8x16) -> [u8; 16usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u8; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x16(self, a: &u8x16) -> &[u8; 16usize] { + crate::transmute::checked_cast_ref::<__m128i, [u8; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x16(self, a: &mut u8x16) -> &mut [u8; 16usize] { + crate::transmute::checked_cast_mut::<__m128i, [u8; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x16(self, a: u8x16, dest: &mut [u8; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x16(self, a: u8x16) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x16(self, a: u8x16) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + if SHIFT >= 16usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_u8x16(b).val.0, + self.cvt_to_bytes_u8x16(a).val.0, + SHIFT, + ); + self.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u8x16( + self, + a: u8x16, + b: u8x16, + ) -> u8x16 { + self.slide_u8x16::(a, b) + } + #[inline(always)] + fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + let dst_even = _mm_mullo_epi16(a.into(), b.into()); + let dst_odd = + _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into())); + _mm_or_si128( + _mm_slli_epi16(dst_odd, 8), + _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u8x16(self, a: u8x16) -> u8x16 { + a ^ !0 + } + #[inline(always)] + fn shl_u8x16(self, a: u8x16, shift: u32) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, shift: u32) -> u8x16 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128()); + let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128()); + let lo_shifted = _mm_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm_sll_epi16(hi_16, shift_count); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + let val = a.into(); + let counts = b.into(); + let zero = _mm_setzero_si128(); + let value_extend = zero; + let lo_values = _mm_unpacklo_epi8(val, value_extend); + let hi_values = _mm_unpackhi_epi8(val, value_extend); + let lo_counts = _mm_unpacklo_epi8(counts, zero); + let hi_counts = _mm_unpackhi_epi8(counts, zero); + let byte_mask = _mm_set1_epi16(0x00ff); + let lo_shifted = _mm_and_si128(_mm_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = _mm_and_si128(_mm_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u8x16(self, a: u8x16, shift: u32) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, shift: u32) -> u8x16 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128()); + let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128()); + let lo_shifted = _mm_srl_epi16(lo_16, shift_count); + let hi_shifted = _mm_srl_epi16(hi_16, shift_count); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + let val = a.into(); + let counts = b.into(); + let zero = _mm_setzero_si128(); + let value_extend = zero; + let lo_values = _mm_unpacklo_epi8(val, value_extend); + let hi_values = _mm_unpackhi_epi8(val, value_extend); + let lo_counts = _mm_unpacklo_epi8(counts, zero); + let hi_counts = _mm_unpackhi_epi8(counts, zero); + let byte_mask = _mm_set1_epi16(0x00ff); + let lo_shifted = _mm_and_si128(_mm_srlv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = _mm_and_si128(_mm_srlv_epi16(hi_values, hi_counts), byte_mask); + _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpeq_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmplt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmple_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpge_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> mask8x16 { + mask8x16 { + val: _mm_cmpgt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_permutex2var_epi8( + a.into(), + _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_permutex2var_epi8( + a.into(), + _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u8x16(self, a: u8x16, b: u8x16) -> (u8x16, u8x16) { + (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b)) + } + #[inline(always)] + fn deinterleave_u8x16(self, a: u8x16, b: u8x16) -> (u8x16, u8x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x16, + b: u8x16, + ) -> (u8x16, u8x16) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi8( + a, + _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b, + ) + .simd_into(token), + _mm_permutex2var_epi8( + a, + _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u8x16(self, a: mask8x16, b: u8x16, c: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x16, + b: u8x16, + c: u8x16, + ) -> u8x16 { + _mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_min_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x16 { + _mm_max_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, b: u8x16) -> u8x32 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn widen_u8x16(self, a: u8x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16) -> u16x16 { + _mm256_cvtepu8_epi16(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u8x16(self, a: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask8x16(self, val: bool) -> mask8x16 { + mask8x16 { + val: (if val { 65535u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask8x16(self, val: [i8; 16usize]) -> mask8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i8; 16usize]) -> mask8x16 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask8x16 { + val: _mm_movepi8_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask8x16(self, a: mask8x16) -> [i8; 16usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask8x16) -> [i8; 16usize] { + let lanes = _mm_movm_epi8(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16 { + mask8x16 { + val: (bits & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask8x16(self, a: mask8x16) -> u64 { + u64::from((a).val) & 65535u64 + } + #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask8x16 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { + mask8x16 { + val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { + mask8x16 { + val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { + mask8x16 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask8x16(self, a: mask8x16) -> mask8x16 { + mask8x16 { + val: ((!u64::from((a).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask8x16( + self, + a: mask8x16, + b: mask8x16, + c: mask8x16, + ) -> mask8x16 { + mask8x16 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { + mask8x16 { + val: (!u64::from(a.val ^ b.val) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask8x16(self, a: mask8x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask8x16(self, a: mask8x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 65535u64 + } + #[inline(always)] + fn any_false_mask8x16(self, a: mask8x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 65535u64 + } + #[inline(always)] + fn all_false_mask8x16(self, a: mask8x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 0 + } + #[inline(always)] + fn combine_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x32 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64; + mask8x32 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn splat_i16x8(self, val: i16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i16) -> i16x8 { + _mm_set1_epi16(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i16x8(self, val: [i16; 8usize]) -> i16x8 { + i16x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x8(self, val: &[i16; 8usize]) -> i16x8 { + i16x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x8(self, a: i16x8) -> [i16; 8usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x8(self, a: &i16x8) -> &[i16; 8usize] { + crate::transmute::checked_cast_ref::<__m128i, [i16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x8(self, a: &mut i16x8) -> &mut [i16; 8usize] { + crate::transmute::checked_cast_mut::<__m128i, [i16; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x8(self, a: i16x8, dest: &mut [i16; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x8(self, a: u8x16) -> i16x8 { + i16x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x8(self, a: i16x8) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + if SHIFT >= 8usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i16x8(b).val.0, + self.cvt_to_bytes_i16x8(a).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i16x8( + self, + a: i16x8, + b: i16x8, + ) -> i16x8 { + self.slide_i16x8::(a, b) + } + #[inline(always)] + fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i16x8(self, a: i16x8) -> i16x8 { + a ^ !0 + } + #[inline(always)] + fn shl_i16x8(self, a: i16x8, shift: u32) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, shift: u32) -> i16x8 { + _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i16x8(self, a: i16x8, shift: u32) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, shift: u32) -> i16x8 { + _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_srav_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpeq_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmplt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmple_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpge_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpgt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_permutex2var_epi16( + a.into(), + _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_permutex2var_epi16( + a.into(), + _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i16x8(self, a: i16x8, b: i16x8) -> (i16x8, i16x8) { + (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b)) + } + #[inline(always)] + fn deinterleave_i16x8(self, a: i16x8, b: i16x8) -> (i16x8, i16x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x8, + b: i16x8, + ) -> (i16x8, i16x8) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i16x8(self, a: mask16x8, b: i16x8, c: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x8, + b: i16x8, + c: i16x8, + ) -> i16x8 { + _mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_min_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { + _mm_max_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i16x8(self, a: i16x8, b: i16x8) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x16 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn neg_i16x8(self, a: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8) -> i16x8 { + _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i16x8(self, a: i16x8) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i16x8(self, a: i16x8) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u16x8(self, val: u16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u16) -> u16x8 { + _mm_set1_epi16(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x8(self, a: u16x8) -> [u16; 8usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x8(self, a: &u16x8) -> &[u16; 8usize] { + crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x8(self, a: &mut u16x8) -> &mut [u16; 8usize] { + crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x8(self, a: u16x8, dest: &mut [u16; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x8(self, a: u8x16) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x8(self, a: u16x8) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + if SHIFT >= 8usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_u16x8(b).val.0, + self.cvt_to_bytes_u16x8(a).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u16x8( + self, + a: u16x8, + b: u16x8, + ) -> u16x8 { + self.slide_u16x8::(a, b) + } + #[inline(always)] + fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u16x8(self, a: u16x8) -> u16x8 { + a ^ !0 + } + #[inline(always)] + fn shl_u16x8(self, a: u16x8, shift: u32) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, shift: u32) -> u16x8 { + _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u16x8(self, a: u16x8, shift: u32) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, shift: u32) -> u16x8 { + _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_srlv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpeq_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmplt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmple_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpge_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> mask16x8 { + mask16x8 { + val: _mm_cmpgt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_permutex2var_epi16( + a.into(), + _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_permutex2var_epi16( + a.into(), + _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u16x8(self, a: u16x8, b: u16x8) -> (u16x8, u16x8) { + (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b)) + } + #[inline(always)] + fn deinterleave_u16x8(self, a: u16x8, b: u16x8) -> (u16x8, u16x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x8, + b: u16x8, + ) -> (u16x8, u16x8) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u16x8(self, a: mask16x8, b: u16x8, c: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x8, + b: u16x8, + c: u16x8, + ) -> u16x8 { + _mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_min_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x8 { + _mm_max_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u16x8(self, a: u16x8, b: u16x8) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, b: u16x8) -> u16x16 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn reinterpret_u8_u16x8(self, a: u16x8) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u16x8(self, a: u16x8) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask16x8(self, val: bool) -> mask16x8 { + mask16x8 { + val: (if val { 255u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask16x8(self, val: [i16; 8usize]) -> mask16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i16; 8usize]) -> mask16x8 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask16x8 { + val: _mm_movepi16_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask16x8(self, a: mask16x8) -> [i16; 8usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask16x8) -> [i16; 8usize] { + let lanes = _mm_movm_epi16(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8 { + mask16x8 { + val: (bits & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask16x8(self, a: mask16x8) -> u64 { + u64::from((a).val) & 255u64 + } + #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask16x8 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { + mask16x8 { + val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { + mask16x8 { + val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { + mask16x8 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask16x8(self, a: mask16x8) -> mask16x8 { + mask16x8 { + val: ((!u64::from((a).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask16x8( + self, + a: mask16x8, + b: mask16x8, + c: mask16x8, + ) -> mask16x8 { + mask16x8 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { + mask16x8 { + val: (!u64::from(a.val ^ b.val) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask16x8(self, a: mask16x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask16x8(self, a: mask16x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 255u64 + } + #[inline(always)] + fn any_false_mask16x8(self, a: mask16x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 255u64 + } + #[inline(always)] + fn all_false_mask16x8(self, a: mask16x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 0 + } + #[inline(always)] + fn combine_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x16 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64; + mask16x16 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn splat_i32x4(self, val: i32) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i32) -> i32x4 { + _mm_set1_epi32(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i32x4(self, val: [i32; 4usize]) -> i32x4 { + i32x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i32x4(self, val: &[i32; 4usize]) -> i32x4 { + i32x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i32x4(self, a: i32x4) -> [i32; 4usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i32x4(self, a: &i32x4) -> &[i32; 4usize] { + crate::transmute::checked_cast_ref::<__m128i, [i32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i32x4(self, a: &mut i32x4) -> &mut [i32; 4usize] { + crate::transmute::checked_cast_mut::<__m128i, [i32; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i32x4(self, a: i32x4, dest: &mut [i32; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i32x4(self, a: u8x16) -> i32x4 { + i32x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i32x4(self, a: i32x4) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + if SHIFT >= 4usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i32x4(b).val.0, + self.cvt_to_bytes_i32x4(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i32x4( + self, + a: i32x4, + b: i32x4, + ) -> i32x4 { + self.slide_i32x4::(a, b) + } + #[inline(always)] + fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i32x4(self, a: i32x4) -> i32x4 { + a ^ !0 + } + #[inline(always)] + fn shl_i32x4(self, a: i32x4, shift: u32) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, shift: u32) -> i32x4 { + _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i32x4(self, a: i32x4, shift: u32) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, shift: u32) -> i32x4 { + _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_srav_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpeq_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmplt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmple_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpge_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpgt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i32x4(self, a: i32x4, b: i32x4) -> (i32x4, i32x4) { + (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b)) + } + #[inline(always)] + fn deinterleave_i32x4(self, a: i32x4, b: i32x4) -> (i32x4, i32x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x4, + b: i32x4, + ) -> (i32x4, i32x4) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token), + _mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i32x4(self, a: mask32x4, b: i32x4, c: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x4, + b: i32x4, + c: i32x4, + ) -> i32x4 { + _mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_min_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x4 { + _mm_max_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i32x4(self, a: i32x4, b: i32x4) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, b: i32x4) -> i32x8 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn neg_i32x4(self, a: i32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4) -> i32x4 { + _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i32x4(self, a: i32x4) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i32x4(self, a: i32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4) -> f32x4 { + _mm_cvtepi32_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u32x4(self, val: u32) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u32) -> u32x4 { + _mm_set1_epi32(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u32x4(self, val: [u32; 4usize]) -> u32x4 { + u32x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u32x4(self, val: &[u32; 4usize]) -> u32x4 { + u32x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u32x4(self, a: u32x4) -> [u32; 4usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u32x4(self, a: &u32x4) -> &[u32; 4usize] { + crate::transmute::checked_cast_ref::<__m128i, [u32; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u32x4(self, a: &mut u32x4) -> &mut [u32; 4usize] { + crate::transmute::checked_cast_mut::<__m128i, [u32; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u32x4(self, a: u32x4, dest: &mut [u32; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u32x4(self, a: u8x16) -> u32x4 { + u32x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u32x4(self, a: u32x4) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + if SHIFT >= 4usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_u32x4(b).val.0, + self.cvt_to_bytes_u32x4(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u32x4( + self, + a: u32x4, + b: u32x4, + ) -> u32x4 { + self.slide_u32x4::(a, b) + } + #[inline(always)] + fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u32x4(self, a: u32x4) -> u32x4 { + a ^ !0 + } + #[inline(always)] + fn shl_u32x4(self, a: u32x4, shift: u32) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, shift: u32) -> u32x4 { + _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u32x4(self, a: u32x4, shift: u32) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, shift: u32) -> u32x4 { + _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_srlv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpeq_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmplt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmple_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpge_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> mask32x4 { + mask32x4 { + val: _mm_cmpgt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u32x4(self, a: u32x4, b: u32x4) -> (u32x4, u32x4) { + (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b)) + } + #[inline(always)] + fn deinterleave_u32x4(self, a: u32x4, b: u32x4) -> (u32x4, u32x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x4, + b: u32x4, + ) -> (u32x4, u32x4) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token), + _mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u32x4(self, a: mask32x4, b: u32x4, c: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x4, + b: u32x4, + c: u32x4, + ) -> u32x4 { + _mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_min_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x4 { + _mm_max_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u32x4(self, a: u32x4, b: u32x4) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, b: u32x4) -> u32x8 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn reinterpret_u8_u32x4(self, a: u32x4) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4) -> f32x4 { + _mm512_castps512_ps128(_mm512_cvtepu32_ps(_mm512_zextsi128_si512(a.into()))) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask32x4(self, val: bool) -> mask32x4 { + mask32x4 { + val: (if val { 15u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask32x4(self, val: [i32; 4usize]) -> mask32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i32; 4usize]) -> mask32x4 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask32x4 { + val: _mm_movepi32_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask32x4(self, a: mask32x4) -> [i32; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask32x4) -> [i32; 4usize] { + let lanes = _mm_movm_epi32(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4 { + mask32x4 { + val: (bits & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask32x4(self, a: mask32x4) -> u64 { + u64::from((a).val) & 15u64 + } + #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask32x4 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { + mask32x4 { + val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { + mask32x4 { + val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { + mask32x4 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask32x4(self, a: mask32x4) -> mask32x4 { + mask32x4 { + val: ((!u64::from((a).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask32x4( + self, + a: mask32x4, + b: mask32x4, + c: mask32x4, + ) -> mask32x4 { + mask32x4 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { + mask32x4 { + val: (!u64::from(a.val ^ b.val) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask32x4(self, a: mask32x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask32x4(self, a: mask32x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits == 15u64 + } + #[inline(always)] + fn any_false_mask32x4(self, a: mask32x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits != 15u64 + } + #[inline(always)] + fn all_false_mask32x4(self, a: mask32x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits == 0 + } + #[inline(always)] + fn combine_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x8 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64; + mask32x8 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn splat_f64x2(self, val: f64) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f64) -> f64x2 { + _mm_set1_pd(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f64x2(self, val: [f64; 2usize]) -> f64x2 { + f64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f64x2(self, val: &[f64; 2usize]) -> f64x2 { + f64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f64x2(self, a: f64x2) -> [f64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128d, [f64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f64x2(self, a: &f64x2) -> &[f64; 2usize] { + crate::transmute::checked_cast_ref::<__m128d, [f64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f64x2(self, a: &mut f64x2) -> &mut [f64; 2usize] { + crate::transmute::checked_cast_mut::<__m128d, [f64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f64x2(self, a: f64x2, dest: &mut [f64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f64x2(self, a: u8x16) -> f64x2 { + f64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f64x2(self, a: f64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_f64x2(b).val.0, + self.cvt_to_bytes_f64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_f64x2( + self, + a: f64x2, + b: f64x2, + ) -> f64x2 { + self.slide_f64x2::(a, b) + } + #[inline(always)] + fn abs_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_sqrt_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_rcp14_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_add_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_sub_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_mul_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_div_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + let mask = _mm_set1_pd(-0.0); + _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into())) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f64x2(self, a: f64x2, b: f64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmp_pd_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f64x2(self, a: f64x2, b: f64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmp_pd_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f64x2(self, a: f64x2, b: f64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmp_pd_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f64x2(self, a: f64x2, b: f64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmp_pd_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f64x2(self, a: f64x2, b: f64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmp_pd_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_unpacklo_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_unpackhi_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f64x2(self, a: f64x2, b: f64x2) -> (f64x2, f64x2) { + (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b)) + } + #[inline(always)] + fn deinterleave_f64x2(self, a: f64x2, b: f64x2) -> (f64x2, f64x2) { + (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b)) + } + #[inline(always)] + fn max_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_max_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_min_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_range_pd::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f64x2(self, a: f64x2, b: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x2 { + _mm_range_pd::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x2, + b: f64x2, + c: f64x2, + ) -> f64x2 { + _mm_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x2, + b: f64x2, + c: f64x2, + ) -> f64x2 { + _mm_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f64x2(self, a: f64x2) -> f64x2 { + a - self.trunc_f64x2(a) + } + #[inline(always)] + fn trunc_f64x2(self, a: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f64x2 { + _mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f64x2(self, a: mask64x2, b: f64x2, c: f64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x2, + b: f64x2, + c: f64x2, + ) -> f64x2 { + _mm_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, b: f64x2) -> f64x4 { + _mm256_setr_m128d(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn reinterpret_f32_f64x2(self, a: f64x2) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> f32x4 { + _mm_castpd_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i64x2(self, val: i64) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i64) -> i64x2 { + _mm_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [i64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [i64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i64x2(b).val.0, + self.cvt_to_bytes_i64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i64x2(self, a: i64x2) -> i64x2 { + a ^ !0 + } + #[inline(always)] + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, shift: u32) -> i64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, shift: u32) -> i64x2 { + _mm_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_srav_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpeq_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmplt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmple_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpge_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpgt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(2, 0), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(3, 1), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) + } + #[inline(always)] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x2, + b: i64x2, + ) -> (i64x2, i64x2) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi64(a, _mm_set_epi64x(2, 0), b).simd_into(token), + _mm_permutex2var_epi64(a, _mm_set_epi64x(3, 1), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x2, + b: i64x2, + c: i64x2, + ) -> i64x2 { + _mm_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_min_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_max_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x4 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn neg_i64x2(self, a: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2) -> i64x2 { + _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u64x2(self, val: u64) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u64) -> u64x2 { + _mm_set1_epi64x(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [u64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [u64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_u64x2(b).val.0, + self.cvt_to_bytes_u64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u64x2( + self, + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) + } + #[inline(always)] + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u64x2(self, a: u64x2) -> u64x2 { + a ^ !0 + } + #[inline(always)] + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, shift: u32) -> u64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, shift: u32) -> u64x2 { + _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_srlv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpeq_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmplt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmple_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpge_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> mask64x2 { + mask64x2 { + val: _mm_cmpgt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(2, 0), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(3, 1), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) + } + #[inline(always)] + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x2, + b: u64x2, + ) -> (u64x2, u64x2) { + let a = a.into(); + let b = b.into(); + ( + _mm_permutex2var_epi64(a, _mm_set_epi64x(2, 0), b).simd_into(token), + _mm_permutex2var_epi64(a, _mm_set_epi64x(3, 1), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x2, + b: u64x2, + c: u64x2, + ) -> u64x2 { + _mm_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_min_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x2 { + _mm_max_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, b: u64x2) -> u64x4 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask64x2(self, val: bool) -> mask64x2 { + mask64x2 { + val: (if val { 3u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i64; 2usize]) -> mask64x2 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask64x2 { + val: _mm_movepi64_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask64x2) -> [i64; 2usize] { + let lanes = _mm_movm_epi64(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + mask64x2 { + val: (bits & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + u64::from((a).val) & 3u64 + } + #[inline(always)] + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask64x2 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + mask64x2 { + val: ((u64::from((a).val) & u64::from((b).val)) & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + mask64x2 { + val: ((u64::from((a).val) | u64::from((b).val)) & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + mask64x2 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + mask64x2 { + val: ((!u64::from((a).val)) & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask64x2( + self, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + mask64x2 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + mask64x2 { + val: (!u64::from(a.val ^ b.val) & 3u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask64x2(self, a: mask64x2) -> bool { + let bits = u64::from((a).val) & 3u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask64x2(self, a: mask64x2) -> bool { + let bits = u64::from((a).val) & 3u64; + bits == 3u64 + } + #[inline(always)] + fn any_false_mask64x2(self, a: mask64x2) -> bool { + let bits = u64::from((a).val) & 3u64; + bits != 3u64 + } + #[inline(always)] + fn all_false_mask64x2(self, a: mask64x2) -> bool { + let bits = u64::from((a).val) & 3u64; + bits == 0 + } + #[inline(always)] + fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 2usize)) & 15u64; + mask64x4 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn splat_f32x8(self, val: f32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f32) -> f32x8 { + _mm256_set1_ps(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256, [f32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { + crate::transmute::checked_cast_ref::<__m256, [f32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { + crate::transmute::checked_cast_mut::<__m256, [f32; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x8, + b: f32x8, + shift: usize, + ) -> f32x8 { + if shift >= 8usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_f32x8(a).val.0, + idx, + token.cvt_to_bytes_f32x8(b).val.0, + ); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_f32x8( + self, + a: f32x8, + b: f32x8, + ) -> f32x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_f32x8(a).val.0; + let b = self.cvt_to_bytes_f32x8(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn abs_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_sqrt_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_rcp14_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_add_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_sub_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_mul_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_div_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + let mask = _mm256_set1_ps(-0.0); + _mm256_or_ps( + _mm256_and_ps(mask, b.into()), + _mm256_andnot_ps(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmp_ps_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmp_ps_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmp_ps_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmp_ps_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmp_ps_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_permutex2var_ps( + a.into(), + _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_permutex2var_ps( + a.into(), + _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_permutex2var_ps( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_permutex2var_ps( + a.into(), + _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x8, + b: f32x8, + ) -> (f32x8, f32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm256_permutex2var_ps(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x8, + b: f32x8, + ) -> (f32x8, f32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm256_permutex2var_ps(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_max_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_min_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_range_ps::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x8 { + _mm256_range_ps::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x8, + b: f32x8, + c: f32x8, + ) -> f32x8 { + _mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x8, + b: f32x8, + c: f32x8, + ) -> f32x8 { + _mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f32x8(self, a: f32x8) -> f32x8 { + a - self.trunc_f32x8(a) + } + #[inline(always)] + fn trunc_f32x8(self, a: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f32x8 { + _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x8, + b: f32x8, + c: f32x8, + ) -> f32x8 { + _mm256_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, b: f32x8) -> f32x16 { + _mm512_insertf32x8::<1>(_mm512_castps256_ps512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> (f32x4, f32x4) { + ( + _mm256_extractf128_ps::<0>(a.into()).simd_into(token), + _mm256_extractf128_ps::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> f64x4 { + _mm256_castps_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> i32x8 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> u8x32 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> u32x8 { + _mm256_castps_si256(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> u32x8 { + _mm256_cvttps_epu32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> u32x8 { + let a = _mm256_max_ps(a.into(), _mm256_setzero_ps()); + let mut converted = _mm256_cvttps_epu32(a); + let exceeds_unsigned_range = + _mm256_cmp_ps_mask::<17i32>(_mm256_set1_ps(4294967040.0), a); + converted = _mm256_mask_blend_epi32( + exceeds_unsigned_range, + converted, + _mm256_set1_epi32(u32::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> i32x8 { + _mm256_cvttps_epi32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8) -> i32x8 { + let a = a.into(); + let in_range = _mm256_cmp_ps_mask::<17i32>(a, _mm256_set1_ps(2147483648.0)); + let mut converted = + _mm256_mask_cvttps_epi32(_mm256_set1_epi32(i32::MAX), in_range, a); + let is_not_nan = _mm256_cmp_ps_mask::<7i32>(a, a); + converted = _mm256_mask_blend_epi32(is_not_nan, _mm256_setzero_si256(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i8x32(self, val: i8) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i8) -> i8x32 { + _mm256_set1_epi8(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { + crate::transmute::checked_cast_ref::<__m256i, [i8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { + crate::transmute::checked_cast_mut::<__m256i, [i8; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x32, + b: i8x32, + shift: usize, + ) -> i8x32 { + if shift >= 32usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_i8x32(a).val.0, + idx, + token.cvt_to_bytes_i8x32(b).val.0, + ); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i8x32( + self, + a: i8x32, + b: i8x32, + ) -> i8x32 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 16usize { + return b; + } + let a = self.cvt_to_bytes_i8x32(a).val.0; + let b = self.cvt_to_bytes_i8x32(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT); + self.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + let dst_even = _mm256_mullo_epi16(a.into(), b.into()); + let dst_odd = _mm256_mullo_epi16( + _mm256_srli_epi16::<8>(a.into()), + _mm256_srli_epi16::<8>(b.into()), + ); + _mm256_or_si256( + _mm256_slli_epi16(dst_odd, 8), + _mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i8x32(self, a: i8x32) -> i8x32 { + a ^ !0 + } + #[inline(always)] + fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, shift: u32) -> i8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = + _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let hi_16 = + _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let lo_shifted = _mm256_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm256_sll_epi16(hi_16, shift_count); + _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + let val = a.into(); + let counts = b.into(); + let zero = _mm256_setzero_si256(); + let value_extend = zero; + let lo_values = _mm256_unpacklo_epi8(val, value_extend); + let hi_values = _mm256_unpackhi_epi8(val, value_extend); + let lo_counts = _mm256_unpacklo_epi8(counts, zero); + let hi_counts = _mm256_unpackhi_epi8(counts, zero); + let byte_mask = _mm256_set1_epi16(0x00ff); + let lo_shifted = + _mm256_and_si256(_mm256_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm256_and_si256(_mm256_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, shift: u32) -> i8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = + _mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let hi_16 = + _mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val)); + let lo_shifted = _mm256_sra_epi16(lo_16, shift_count); + let hi_shifted = _mm256_sra_epi16(hi_16, shift_count); + _mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + let val = a.into(); + let counts = b.into(); + let zero = _mm256_setzero_si256(); + let value_extend = _mm256_cmpgt_epi8(zero, val); + let lo_values = _mm256_unpacklo_epi8(val, value_extend); + let hi_values = _mm256_unpackhi_epi8(val, value_extend); + let lo_counts = _mm256_unpacklo_epi8(counts, zero); + let hi_counts = _mm256_unpackhi_epi8(counts, zero); + let byte_mask = _mm256_set1_epi16(0x00ff); + let lo_shifted = + _mm256_and_si256(_mm256_srav_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm256_and_si256(_mm256_srav_epi16(hi_values, hi_counts), byte_mask); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpeq_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmplt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmple_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpge_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpgt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10, + 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25, + 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, + 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, + 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x32, + b: i8x32, + ) -> (i8x32, i8x32) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, + 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, + 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x32, + b: i8x32, + ) -> (i8x32, i8x32) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, + 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, + 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x32, + b: i8x32, + c: i8x32, + ) -> i8x32 { + _mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_min_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x32 { + _mm256_max_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, b: i8x32) -> i8x64 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32) -> (i8x16, i8x16) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i8x32(self, a: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32) -> i8x32 { + _mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u8x32(self, val: u8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u8) -> u8x32 { + _mm256_set1_epi8(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { + crate::transmute::checked_cast_ref::<__m256i, [u8; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { + crate::transmute::checked_cast_mut::<__m256i, [u8; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x32, + b: u8x32, + shift: usize, + ) -> u8x32 { + if shift >= 32usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_u8x32(a).val.0, + idx, + token.cvt_to_bytes_u8x32(b).val.0, + ); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u8x32( + self, + a: u8x32, + b: u8x32, + ) -> u8x32 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 16usize { + return b; + } + let a = self.cvt_to_bytes_u8x32(a).val.0; + let b = self.cvt_to_bytes_u8x32(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT); + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + let dst_even = _mm256_mullo_epi16(a.into(), b.into()); + let dst_odd = _mm256_mullo_epi16( + _mm256_srli_epi16::<8>(a.into()), + _mm256_srli_epi16::<8>(b.into()), + ); + _mm256_or_si256( + _mm256_slli_epi16(dst_odd, 8), + _mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u8x32(self, a: u8x32) -> u8x32 { + a ^ !0 + } + #[inline(always)] + fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, shift: u32) -> u8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); + let lo_shifted = _mm256_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm256_sll_epi16(hi_16, shift_count); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + let val = a.into(); + let counts = b.into(); + let zero = _mm256_setzero_si256(); + let value_extend = zero; + let lo_values = _mm256_unpacklo_epi8(val, value_extend); + let hi_values = _mm256_unpackhi_epi8(val, value_extend); + let lo_counts = _mm256_unpacklo_epi8(counts, zero); + let hi_counts = _mm256_unpackhi_epi8(counts, zero); + let byte_mask = _mm256_set1_epi16(0x00ff); + let lo_shifted = + _mm256_and_si256(_mm256_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm256_and_si256(_mm256_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, shift: u32) -> u8x32 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256()); + let lo_shifted = _mm256_srl_epi16(lo_16, shift_count); + let hi_shifted = _mm256_srl_epi16(hi_16, shift_count); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + let val = a.into(); + let counts = b.into(); + let zero = _mm256_setzero_si256(); + let value_extend = zero; + let lo_values = _mm256_unpacklo_epi8(val, value_extend); + let hi_values = _mm256_unpackhi_epi8(val, value_extend); + let lo_counts = _mm256_unpacklo_epi8(counts, zero); + let hi_counts = _mm256_unpackhi_epi8(counts, zero); + let byte_mask = _mm256_set1_epi16(0x00ff); + let lo_shifted = + _mm256_and_si256(_mm256_srlv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm256_and_si256(_mm256_srlv_epi16(hi_values, hi_counts), byte_mask); + _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpeq_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmplt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmple_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpge_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> mask8x32 { + mask8x32 { + val: _mm256_cmpgt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10, + 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25, + 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, + 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_permutex2var_epi8( + a.into(), + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, + 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x32, + b: u8x32, + ) -> (u8x32, u8x32) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, + 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, + 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x32, + b: u8x32, + ) -> (u8x32, u8x32) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, + 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi8( + a, + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, + 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x32, + b: u8x32, + c: u8x32, + ) -> u8x32 { + _mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_min_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_max_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x64 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32) -> (u8x16, u8x16) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn widen_u8x32(self, a: u8x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32) -> u16x32 { + _mm512_cvtepu8_epi16(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask8x32(self, val: bool) -> mask8x32 { + mask8x32 { + val: (if val { 4294967295u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i8; 32usize]) -> mask8x32 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask8x32 { + val: _mm256_movepi8_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask8x32) -> [i8; 32usize] { + let lanes = _mm256_movm_epi8(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { + mask8x32 { + val: (bits & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { + u64::from((a).val) & 4294967295u64 + } + #[inline(always)] + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask8x32 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + mask8x32 { + val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + mask8x32 { + val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + mask8x32 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask8x32(self, a: mask8x32) -> mask8x32 { + mask8x32 { + val: ((!u64::from((a).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask8x32( + self, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + mask8x32 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + mask8x32 { + val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask8x32(self, a: mask8x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask8x32(self, a: mask8x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits == 4294967295u64 + } + #[inline(always)] + fn any_false_mask8x32(self, a: mask8x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits != 4294967295u64 + } + #[inline(always)] + fn all_false_mask8x32(self, a: mask8x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits == 0 + } + #[inline(always)] + fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 32usize)) & u64::MAX; + mask8x64 { + val: bits, + simd: self, + } + } + #[inline(always)] + fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { + let bits = u64::from(a.val); + ( + mask8x16 { + val: (bits & 65535u64) as _, + simd: self, + }, + mask8x16 { + val: ((bits >> 16usize) & 65535u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i16x16(self, val: i16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i16) -> i16x16 { + _mm256_set1_epi16(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { + crate::transmute::checked_cast_ref::<__m256i, [i16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { + crate::transmute::checked_cast_mut::<__m256i, [i16; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x16, + b: i16x16, + shift: usize, + ) -> i16x16 { + if shift >= 16usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 2usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_i16x16(a).val.0, + idx, + token.cvt_to_bytes_i16x16(b).val.0, + ); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i16x16( + self, + a: i16x16, + b: i16x16, + ) -> i16x16 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 8usize { + return b; + } + let a = self.cvt_to_bytes_i16x16(a).val.0; + let b = self.cvt_to_bytes_i16x16(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 2usize); + self.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i16x16(self, a: i16x16) -> i16x16 { + a ^ !0 + } + #[inline(always)] + fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, shift: u32) -> i16x16 { + _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, shift: u32) -> i16x16 { + _mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_srav_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpeq_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmplt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmple_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpge_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpgt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x16, + b: i16x16, + ) -> (i16x16, i16x16) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x16, + b: i16x16, + ) -> (i16x16, i16x16) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x16, + b: i16x16, + c: i16x16, + ) -> i16x16 { + _mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_min_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x16 { + _mm256_max_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, b: i16x16) -> i16x32 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16) -> (i16x8, i16x8) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i16x16(self, a: i16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16) -> i16x16 { + _mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u16x16(self, val: u16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u16) -> u16x16 { + _mm256_set1_epi16(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { + crate::transmute::checked_cast_ref::<__m256i, [u16; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { + crate::transmute::checked_cast_mut::<__m256i, [u16; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x16, + b: u16x16, + shift: usize, + ) -> u16x16 { + if shift >= 16usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 2usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_u16x16(a).val.0, + idx, + token.cvt_to_bytes_u16x16(b).val.0, + ); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u16x16( + self, + a: u16x16, + b: u16x16, + ) -> u16x16 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 8usize { + return b; + } + let a = self.cvt_to_bytes_u16x16(a).val.0; + let b = self.cvt_to_bytes_u16x16(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 2usize); + self.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u16x16(self, a: u16x16) -> u16x16 { + a ^ !0 + } + #[inline(always)] + fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, shift: u32) -> u16x16 { + _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, shift: u32) -> u16x16 { + _mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_srlv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpeq_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmplt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmple_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpge_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> mask16x16 { + mask16x16 { + val: _mm256_cmpgt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_permutex2var_epi16( + a.into(), + _mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x16, + b: u16x16, + ) -> (u16x16, u16x16) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x16, + b: u16x16, + ) -> (u16x16, u16x16) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + ), + b, + ) + .simd_into(token), + _mm256_permutex2var_epi16( + a, + _mm256_setr_epi16( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x16, + b: u16x16, + c: u16x16, + ) -> u16x16 { + _mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_min_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x16 { + _mm256_max_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, b: u16x16) -> u16x32 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16) -> (u16x8, u16x8) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn narrow_u16x16(self, a: u16x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16) -> u8x16 { + _mm256_cvtepi16_epi8(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask16x16(self, val: bool) -> mask16x16 { + mask16x16 { + val: (if val { 65535u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i16; 16usize]) -> mask16x16 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask16x16 { + val: _mm256_movepi16_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask16x16) -> [i16; 16usize] { + let lanes = _mm256_movm_epi16(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { + mask16x16 { + val: (bits & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { + u64::from((a).val) & 65535u64 + } + #[inline(always)] + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask16x16 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + mask16x16 { + val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + mask16x16 { + val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + mask16x16 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask16x16(self, a: mask16x16) -> mask16x16 { + mask16x16 { + val: ((!u64::from((a).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask16x16( + self, + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { + mask16x16 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + mask16x16 { + val: (!u64::from(a.val ^ b.val) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask16x16(self, a: mask16x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask16x16(self, a: mask16x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 65535u64 + } + #[inline(always)] + fn any_false_mask16x16(self, a: mask16x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 65535u64 + } + #[inline(always)] + fn all_false_mask16x16(self, a: mask16x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 0 + } + #[inline(always)] + fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64; + mask16x32 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + let bits = u64::from(a.val); + ( + mask16x8 { + val: (bits & 255u64) as _, + simd: self, + }, + mask16x8 { + val: ((bits >> 8usize) & 255u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i32x8(self, val: i32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i32) -> i32x8 { + _mm256_set1_epi32(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { + crate::transmute::checked_cast_ref::<__m256i, [i32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { + crate::transmute::checked_cast_mut::<__m256i, [i32; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x8, + b: i32x8, + shift: usize, + ) -> i32x8 { + if shift >= 8usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_i32x8(a).val.0, + idx, + token.cvt_to_bytes_i32x8(b).val.0, + ); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i32x8( + self, + a: i32x8, + b: i32x8, + ) -> i32x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_i32x8(a).val.0; + let b = self.cvt_to_bytes_i32x8(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i32x8(self, a: i32x8) -> i32x8 { + a ^ !0 + } + #[inline(always)] + fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, shift: u32) -> i32x8 { + _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, shift: u32) -> i32x8 { + _mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_srav_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpeq_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmplt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmple_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpge_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpgt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x8, + b: i32x8, + ) -> (i32x8, i32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x8, + b: i32x8, + ) -> (i32x8, i32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x8, + b: i32x8, + c: i32x8, + ) -> i32x8 { + _mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_min_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x8 { + _mm256_max_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, b: i32x8) -> i32x16 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8) -> (i32x4, i32x4) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i32x8(self, a: i32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8) -> i32x8 { + _mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8) -> f32x8 { + _mm256_cvtepi32_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u32x8(self, val: u32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u32) -> u32x8 { + _mm256_set1_epi32(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { + u32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { + u32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { + crate::transmute::checked_cast_ref::<__m256i, [u32; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { + crate::transmute::checked_cast_mut::<__m256i, [u32; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { + u32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x8, + b: u32x8, + shift: usize, + ) -> u32x8 { + if shift >= 8usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_u32x8(a).val.0, + idx, + token.cvt_to_bytes_u32x8(b).val.0, + ); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u32x8( + self, + a: u32x8, + b: u32x8, + ) -> u32x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_u32x8(a).val.0; + let b = self.cvt_to_bytes_u32x8(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u32x8(self, a: u32x8) -> u32x8 { + a ^ !0 + } + #[inline(always)] + fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, shift: u32) -> u32x8 { + _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, shift: u32) -> u32x8 { + _mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_srlv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpeq_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmplt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmple_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpge_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> mask32x8 { + mask32x8 { + val: _mm256_cmpgt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_permutex2var_epi32( + a.into(), + _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x8, + b: u32x8, + ) -> (u32x8, u32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x8, + b: u32x8, + ) -> (u32x8, u32x8) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x8, + b: u32x8, + c: u32x8, + ) -> u32x8 { + _mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_min_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x8 { + _mm256_max_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, b: u32x8) -> u32x16 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8) -> (u32x4, u32x4) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8) -> f32x8 { + _mm512_castps512_ps256(_mm512_cvtepu32_ps(_mm512_zextsi256_si512(a.into()))) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask32x8(self, val: bool) -> mask32x8 { + mask32x8 { + val: (if val { 255u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i32; 8usize]) -> mask32x8 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask32x8 { + val: _mm256_movepi32_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask32x8) -> [i32; 8usize] { + let lanes = _mm256_movm_epi32(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { + mask32x8 { + val: (bits & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { + u64::from((a).val) & 255u64 + } + #[inline(always)] + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask32x8 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + mask32x8 { + val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + mask32x8 { + val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + mask32x8 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask32x8(self, a: mask32x8) -> mask32x8 { + mask32x8 { + val: ((!u64::from((a).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask32x8( + self, + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { + mask32x8 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + mask32x8 { + val: (!u64::from(a.val ^ b.val) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask32x8(self, a: mask32x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask32x8(self, a: mask32x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 255u64 + } + #[inline(always)] + fn any_false_mask32x8(self, a: mask32x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 255u64 + } + #[inline(always)] + fn all_false_mask32x8(self, a: mask32x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 0 + } + #[inline(always)] + fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64; + mask32x16 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { + let bits = u64::from(a.val); + ( + mask32x4 { + val: (bits & 15u64) as _, + simd: self, + }, + mask32x4 { + val: ((bits >> 4usize) & 15u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_f64x4(self, val: f64) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f64) -> f64x4 { + _mm256_set1_pd(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { + f64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { + f64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256d, [f64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { + crate::transmute::checked_cast_ref::<__m256d, [f64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { + crate::transmute::checked_cast_mut::<__m256d, [f64; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { + f64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x4, + b: f64x4, + shift: usize, + ) -> f64x4 { + if shift >= 4usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_f64x4(a).val.0, + idx, + token.cvt_to_bytes_f64x4(b).val.0, + ); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_f64x4( + self, + a: f64x4, + b: f64x4, + ) -> f64x4 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_f64x4(a).val.0; + let b = self.cvt_to_bytes_f64x4(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn abs_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_sqrt_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_rcp14_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_add_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_sub_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_mul_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_div_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + let mask = _mm256_set1_pd(-0.0); + _mm256_or_pd( + _mm256_and_pd(mask, b.into()), + _mm256_andnot_pd(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmp_pd_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmp_pd_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmp_pd_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmp_pd_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmp_pd_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x4, + b: f64x4, + ) -> (f64x4, f64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 4, 1, 5), b).simd_into(token), + _mm256_permutex2var_pd(a, _mm256_setr_epi64x(2, 6, 3, 7), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x4, + b: f64x4, + ) -> (f64x4, f64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 2, 4, 6), b).simd_into(token), + _mm256_permutex2var_pd(a, _mm256_setr_epi64x(1, 3, 5, 7), b).simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_max_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_min_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_range_pd::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + _mm256_range_pd::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x4, + b: f64x4, + c: f64x4, + ) -> f64x4 { + _mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x4, + b: f64x4, + c: f64x4, + ) -> f64x4 { + _mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f64x4(self, a: f64x4) -> f64x4 { + a - self.trunc_f64x4(a) + } + #[inline(always)] + fn trunc_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x4, + b: f64x4, + c: f64x4, + ) -> f64x4 { + _mm256_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x8 { + _mm512_insertf64x4::<1>(_mm512_castpd256_pd512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> (f64x2, f64x2) { + ( + _mm256_extractf128_pd::<0>(a.into()).simd_into(token), + _mm256_extractf128_pd::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f32x8 { + _mm256_castpd_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i64x4(self, val: i64) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i64) -> i64x4 { + _mm256_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256i, [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + crate::transmute::checked_cast_ref::<__m256i, [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + crate::transmute::checked_cast_mut::<__m256i, [i64; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x4, + b: i64x4, + shift: usize, + ) -> i64x4 { + if shift >= 4usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_i64x4(a).val.0, + idx, + token.cvt_to_bytes_i64x4(b).val.0, + ); + token.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i64x4( + self, + a: i64x4, + b: i64x4, + ) -> i64x4 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_i64x4(a).val.0; + let b = self.cvt_to_bytes_i64x4(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i64x4(self, a: i64x4) -> i64x4 { + a ^ !0 + } + #[inline(always)] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, shift: u32) -> i64x4 { + _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, shift: u32) -> i64x4 { + _mm256_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_srav_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpeq_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmplt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmple_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpge_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpgt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x4, + b: i64x4, + ) -> (i64x4, i64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 4, 1, 5), b) + .simd_into(token), + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(2, 6, 3, 7), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x4, + b: i64x4, + ) -> (i64x4, i64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 2, 4, 6), b) + .simd_into(token), + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(1, 3, 5, 7), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x4, + b: i64x4, + c: i64x4, + ) -> i64x4 { + _mm256_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_min_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x4 { + _mm256_max_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, b: i64x4) -> i64x8 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4) -> (i64x2, i64x2) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i64x4(self, a: i64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4) -> i64x4 { + _mm256_sub_epi64(_mm256_setzero_si256(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u64x4(self, val: u64) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u64) -> u64x4 { + _mm256_set1_epi64x(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + crate::transmute::checked_transmute_copy::<__m256i, [u64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + crate::transmute::checked_cast_ref::<__m256i, [u64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + crate::transmute::checked_cast_mut::<__m256i, [u64; 4usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x4, + b: u64x4, + shift: usize, + ) -> u64x4 { + if shift >= 4usize { + return b; + } + let idx = _mm256_add_epi8( + _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ), + _mm256_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm256_permutex2var_epi8( + token.cvt_to_bytes_u64x4(a).val.0, + idx, + token.cvt_to_bytes_u64x4(b).val.0, + ); + token.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u64x4( + self, + a: u64x4, + b: u64x4, + ) -> u64x4 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_u64x4(a).val.0; + let b = self.cvt_to_bytes_u64x4(b).val.0; + let result = dyn_alignr_256(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_and_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_or_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_xor_si256(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u64x4(self, a: u64x4) -> u64x4 { + a ^ !0 + } + #[inline(always)] + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, shift: u32) -> u64x4 { + _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, shift: u32) -> u64x4 { + _mm256_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_srlv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpeq_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmplt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmple_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpge_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> mask64x4 { + mask64x4 { + val: _mm256_cmpgt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into()) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x4, + b: u64x4, + ) -> (u64x4, u64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 4, 1, 5), b) + .simd_into(token), + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(2, 6, 3, 7), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x4, + b: u64x4, + ) -> (u64x4, u64x4) { + let a = a.into(); + let b = b.into(); + ( + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 2, 4, 6), b) + .simd_into(token), + _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(1, 3, 5, 7), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x4, + b: u64x4, + c: u64x4, + ) -> u64x4 { + _mm256_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_min_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x4 { + _mm256_max_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, b: u64x4) -> u64x8 { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4) -> (u64x2, u64x2) { + ( + _mm256_extracti128_si256::<0>(a.into()).simd_into(token), + _mm256_extracti128_si256::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4) -> u8x32 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4) -> u32x8 { + __m256i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask64x4(self, val: bool) -> mask64x4 { + mask64x4 { + val: (if val { 15u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i64; 4usize]) -> mask64x4 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask64x4 { + val: _mm256_movepi64_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask64x4) -> [i64; 4usize] { + let lanes = _mm256_movm_epi64(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { + mask64x4 { + val: (bits & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + u64::from((a).val) & 15u64 + } + #[inline(always)] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask64x4 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + mask64x4 { + val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + mask64x4 { + val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + mask64x4 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask64x4(self, a: mask64x4) -> mask64x4 { + mask64x4 { + val: ((!u64::from((a).val)) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask64x4( + self, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + mask64x4 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + mask64x4 { + val: (!u64::from(a.val ^ b.val) & 15u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask64x4(self, a: mask64x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask64x4(self, a: mask64x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits == 15u64 + } + #[inline(always)] + fn any_false_mask64x4(self, a: mask64x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits != 15u64 + } + #[inline(always)] + fn all_false_mask64x4(self, a: mask64x4) -> bool { + let bits = u64::from((a).val) & 15u64; + bits == 0 + } + #[inline(always)] + fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { + let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64; + mask64x8 { + val: bits as _, + simd: self, + } + } + #[inline(always)] + fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + let bits = u64::from(a.val); + ( + mask64x2 { + val: (bits & 3u64) as _, + simd: self, + }, + mask64x2 { + val: ((bits >> 2usize) & 3u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_f32x16(self, val: f32) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f32) -> f32x16 { + _mm512_set1_ps(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { + crate::transmute::checked_transmute_copy::<__m512, [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { + crate::transmute::checked_cast_ref::<__m512, [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { + crate::transmute::checked_cast_mut::<__m512, [f32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x16, + b: f32x16, + shift: usize, + ) -> f32x16 { + if shift >= 16usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_f32x16(a).val.0, + idx, + token.cvt_to_bytes_f32x16(b).val.0, + ); + token.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_f32x16( + self, + a: f32x16, + b: f32x16, + ) -> f32x16 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_f32x16(a).val.0; + let b = self.cvt_to_bytes_f32x16(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_andnot_ps(_mm512_set1_ps(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_xor_ps(a.into(), _mm512_set1_ps(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_sqrt_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_rcp14_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_add_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_sub_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_mul_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_div_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + let mask = _mm512_set1_ps(-0.0); + _mm512_or_ps( + _mm512_and_ps(mask, b.into()), + _mm512_andnot_ps(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmp_ps_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmp_ps_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmp_ps_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmp_ps_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmp_ps_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_permutex2var_ps( + a.into(), + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_permutex2var_ps( + a.into(), + _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_permutex2var_ps( + a.into(), + _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_permutex2var_ps( + a.into(), + _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x16, + b: f32x16, + ) -> (f32x16, f32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_ps( + a, + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b, + ) + .simd_into(token), + _mm512_permutex2var_ps( + a, + _mm512_setr_epi32( + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x16, + b: f32x16, + ) -> (f32x16, f32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_ps( + a, + _mm512_setr_epi32( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_ps( + a, + _mm512_setr_epi32( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_max_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_min_ps(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_range_ps::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, b: f32x16) -> f32x16 { + _mm512_range_ps::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x16, + b: f32x16, + c: f32x16, + ) -> f32x16 { + _mm512_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f32x16, + b: f32x16, + c: f32x16, + ) -> f32x16 { + _mm512_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_roundscale_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_roundscale_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_roundscale_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f32x16(self, a: f32x16) -> f32x16 { + a - self.trunc_f32x16(a) + } + #[inline(always)] + fn trunc_f32x16(self, a: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f32x16 { + _mm512_roundscale_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x16, + b: f32x16, + c: f32x16, + ) -> f32x16 { + _mm512_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> (f32x8, f32x8) { + ( + _mm512_castps512_ps256(a.into()).simd_into(token), + _mm512_extractf32x8_ps::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> f64x8 { + _mm512_castps_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> i32x16 { + _mm512_castps_si512(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f32; 16usize]) -> f32x16 { + let lanes: __m512 = + crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src); + _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, dest: &mut [f32; 16usize]) -> () { + let lanes = _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> u8x64 { + _mm512_castps_si512(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> u32x16 { + _mm512_castps_si512(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> u32x16 { + _mm512_cvttps_epu32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> u32x16 { + let a = _mm512_max_ps(a.into(), _mm512_setzero_ps()); + let mut converted = _mm512_cvttps_epu32(a); + let exceeds_unsigned_range = + _mm512_cmp_ps_mask::<17i32>(_mm512_set1_ps(4294967040.0), a); + converted = _mm512_mask_blend_epi32( + exceeds_unsigned_range, + converted, + _mm512_set1_epi32(u32::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> i32x16 { + _mm512_cvttps_epi32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16) -> i32x16 { + let a = a.into(); + let in_range = _mm512_cmp_ps_mask::<17i32>(a, _mm512_set1_ps(2147483648.0)); + let mut converted = + _mm512_mask_cvttps_epi32(_mm512_set1_epi32(i32::MAX), in_range, a); + let is_not_nan = _mm512_cmp_ps_mask::<7i32>(a, a); + converted = _mm512_mask_blend_epi32(is_not_nan, _mm512_setzero_si512(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i8x64(self, val: i8) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i8) -> i8x64 { + _mm512_set1_epi8(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::<__m512i, [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { + crate::transmute::checked_cast_ref::<__m512i, [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { + crate::transmute::checked_cast_mut::<__m512i, [i8; 64usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x64, + b: i8x64, + shift: usize, + ) -> i8x64 { + if shift >= 64usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_i8x64(a).val.0, + idx, + token.cvt_to_bytes_i8x64(b).val.0, + ); + token.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i8x64( + self, + a: i8x64, + b: i8x64, + ) -> i8x64 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 16usize { + return b; + } + let a = self.cvt_to_bytes_i8x64(a).val.0; + let b = self.cvt_to_bytes_i8x64(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT); + self.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + let dst_even = _mm512_mullo_epi16(a.into(), b.into()); + let dst_odd = _mm512_mullo_epi16( + _mm512_srli_epi16::<8>(a.into()), + _mm512_srli_epi16::<8>(b.into()), + ); + _mm512_or_si512( + _mm512_slli_epi16(dst_odd, 8), + _mm512_and_si512(dst_even, _mm512_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i8x64(self, a: i8x64) -> i8x64 { + a ^ !0 + } + #[inline(always)] + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, shift: u32) -> i8x64 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm512_unpacklo_epi8( + val, + _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)), + ); + let hi_16 = _mm512_unpackhi_epi8( + val, + _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)), + ); + let lo_shifted = _mm512_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm512_sll_epi16(hi_16, shift_count); + _mm512_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + let val = a.into(); + let counts = b.into(); + let zero = _mm512_setzero_si512(); + let value_extend = zero; + let lo_values = _mm512_unpacklo_epi8(val, value_extend); + let hi_values = _mm512_unpackhi_epi8(val, value_extend); + let lo_counts = _mm512_unpacklo_epi8(counts, zero); + let hi_counts = _mm512_unpackhi_epi8(counts, zero); + let byte_mask = _mm512_set1_epi16(0x00ff); + let lo_shifted = + _mm512_and_si512(_mm512_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm512_and_si512(_mm512_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, shift: u32) -> i8x64 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm512_unpacklo_epi8( + val, + _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)), + ); + let hi_16 = _mm512_unpackhi_epi8( + val, + _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)), + ); + let lo_shifted = _mm512_sra_epi16(lo_16, shift_count); + let hi_shifted = _mm512_sra_epi16(hi_16, shift_count); + _mm512_packs_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + let val = a.into(); + let counts = b.into(); + let zero = _mm512_setzero_si512(); + let value_extend = _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(zero, val)); + let lo_values = _mm512_unpacklo_epi8(val, value_extend); + let hi_values = _mm512_unpackhi_epi8(val, value_extend); + let lo_counts = _mm512_unpacklo_epi8(counts, zero); + let hi_counts = _mm512_unpackhi_epi8(counts, zero); + let byte_mask = _mm512_set1_epi16(0x00ff); + let lo_shifted = + _mm512_and_si512(_mm512_srav_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm512_and_si512(_mm512_srav_epi16(hi_values, hi_counts), byte_mask); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpeq_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmplt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmple_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpge_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpgt_epi8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86, + 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, + 66, 2, 65, 1, 64, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, + 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, + 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, + 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98, + 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60, + 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22, + 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99, + 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61, + 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23, + 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x64, + b: i8x64, + ) -> (i8x64, i8x64) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, + 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, + 4, 67, 3, 66, 2, 65, 1, 64, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, + 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, + 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, + 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i8x64, + b: i8x64, + ) -> (i8x64, i8x64) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, + 98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, + 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, + 99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, + 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x64, + b: i8x64, + c: i8x64, + ) -> i8x64 { + _mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_min_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, b: i8x64) -> i8x64 { + _mm512_max_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64) -> (i8x32, i8x32) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i8x64(self, a: i8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64) -> i8x64 { + _mm512_sub_epi8(_mm512_setzero_si512(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u8x64(self, val: u8) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u8) -> u8x64 { + _mm512_set1_epi8(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { + crate::transmute::checked_transmute_copy::<__m512i, [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { + crate::transmute::checked_cast_ref::<__m512i, [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { + crate::transmute::checked_cast_mut::<__m512i, [u8; 64usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x64, + b: u8x64, + shift: usize, + ) -> u8x64 { + if shift >= 64usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_u8x64(a).val.0, + idx, + token.cvt_to_bytes_u8x64(b).val.0, + ); + token.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u8x64( + self, + a: u8x64, + b: u8x64, + ) -> u8x64 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 16usize { + return b; + } + let a = self.cvt_to_bytes_u8x64(a).val.0; + let b = self.cvt_to_bytes_u8x64(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT); + self.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + let dst_even = _mm512_mullo_epi16(a.into(), b.into()); + let dst_odd = _mm512_mullo_epi16( + _mm512_srli_epi16::<8>(a.into()), + _mm512_srli_epi16::<8>(b.into()), + ); + _mm512_or_si512( + _mm512_slli_epi16(dst_odd, 8), + _mm512_and_si512(dst_even, _mm512_set1_epi16(0xFF)), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u8x64(self, a: u8x64) -> u8x64 { + a ^ !0 + } + #[inline(always)] + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, shift: u32) -> u8x64 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm512_unpacklo_epi8(val, _mm512_setzero_si512()); + let hi_16 = _mm512_unpackhi_epi8(val, _mm512_setzero_si512()); + let lo_shifted = _mm512_sll_epi16(lo_16, shift_count); + let hi_shifted = _mm512_sll_epi16(hi_16, shift_count); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + let val = a.into(); + let counts = b.into(); + let zero = _mm512_setzero_si512(); + let value_extend = zero; + let lo_values = _mm512_unpacklo_epi8(val, value_extend); + let hi_values = _mm512_unpackhi_epi8(val, value_extend); + let lo_counts = _mm512_unpacklo_epi8(counts, zero); + let hi_counts = _mm512_unpackhi_epi8(counts, zero); + let byte_mask = _mm512_set1_epi16(0x00ff); + let lo_shifted = + _mm512_and_si512(_mm512_sllv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm512_and_si512(_mm512_sllv_epi16(hi_values, hi_counts), byte_mask); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, shift: u32) -> u8x64 { + let val = a.into(); + let shift_count = _mm_cvtsi32_si128(shift.cast_signed()); + let lo_16 = _mm512_unpacklo_epi8(val, _mm512_setzero_si512()); + let hi_16 = _mm512_unpackhi_epi8(val, _mm512_setzero_si512()); + let lo_shifted = _mm512_srl_epi16(lo_16, shift_count); + let hi_shifted = _mm512_srl_epi16(hi_16, shift_count); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + let val = a.into(); + let counts = b.into(); + let zero = _mm512_setzero_si512(); + let value_extend = zero; + let lo_values = _mm512_unpacklo_epi8(val, value_extend); + let hi_values = _mm512_unpackhi_epi8(val, value_extend); + let lo_counts = _mm512_unpacklo_epi8(counts, zero); + let hi_counts = _mm512_unpackhi_epi8(counts, zero); + let byte_mask = _mm512_set1_epi16(0x00ff); + let lo_shifted = + _mm512_and_si512(_mm512_srlv_epi16(lo_values, lo_counts), byte_mask); + let hi_shifted = + _mm512_and_si512(_mm512_srlv_epi16(hi_values, hi_counts), byte_mask); + _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpeq_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmplt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmple_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpge_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> mask8x64 { + mask8x64 { + val: _mm512_cmpgt_epu8_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86, + 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, + 66, 2, 65, 1, 64, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, + 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, + 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, + 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98, + 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60, + 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22, + 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_permutex2var_epi8( + a.into(), + _mm512_set_epi8( + 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99, + 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61, + 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23, + 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x64, + b: u8x64, + ) -> (u8x64, u8x64) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, + 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, + 4, 67, 3, 66, 2, 65, 1, 64, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, + 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, + 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, + 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u8x64, + b: u8x64, + ) -> (u8x64, u8x64) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, + 98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, + 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi8( + a, + _mm512_set_epi8( + 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, + 99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, + 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask8x64, + b: u8x64, + c: u8x64, + ) -> u8x64 { + _mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_min_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_max_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64) -> (u8x32, u8x32) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u8; 64usize]) -> u8x64 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src); + _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, dest: &mut [u8; 64usize]) -> () { + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask8x64(self, val: bool) -> mask8x64 { + mask8x64 { + val: if val { u64::MAX } else { 0 }, + simd: self, + } + } + #[inline(always)] + fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i8; 64usize]) -> mask8x64 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask8x64 { + val: _mm512_movepi8_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask8x64) -> [i8; 64usize] { + let lanes = _mm512_movm_epi8(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { + mask8x64 { + val: bits & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { + u64::from((a).val) & u64::MAX + } + #[inline(always)] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask8x64 { + val: bits, + simd: self, + }; + } + #[inline(always)] + fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + mask8x64 { + val: (u64::from((a).val) & u64::from((b).val)) & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + mask8x64 { + val: (u64::from((a).val) | u64::from((b).val)) & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + mask8x64 { + val: (u64::from((a).val) ^ u64::from((b).val)) & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn not_mask8x64(self, a: mask8x64) -> mask8x64 { + mask8x64 { + val: (!u64::from((a).val)) & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn select_mask8x64( + self, + a: mask8x64, + b: mask8x64, + c: mask8x64, + ) -> mask8x64 { + mask8x64 { + val: ((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + mask8x64 { + val: !u64::from(a.val ^ b.val) & u64::MAX, + simd: self, + } + } + #[inline(always)] + fn any_true_mask8x64(self, a: mask8x64) -> bool { + let bits = u64::from((a).val) & u64::MAX; + bits != 0 + } + #[inline(always)] + fn all_true_mask8x64(self, a: mask8x64) -> bool { + let bits = u64::from((a).val) & u64::MAX; + bits == u64::MAX + } + #[inline(always)] + fn any_false_mask8x64(self, a: mask8x64) -> bool { + let bits = u64::from((a).val) & u64::MAX; + bits != u64::MAX + } + #[inline(always)] + fn all_false_mask8x64(self, a: mask8x64) -> bool { + let bits = u64::from((a).val) & u64::MAX; + bits == 0 + } + #[inline(always)] + fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + let bits = u64::from(a.val); + ( + mask8x32 { + val: (bits & 4294967295u64) as _, + simd: self, + }, + mask8x32 { + val: ((bits >> 32usize) & 4294967295u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i16x32(self, val: i16) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i16) -> i16x32 { + _mm512_set1_epi16(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::<__m512i, [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { + crate::transmute::checked_cast_ref::<__m512i, [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { + crate::transmute::checked_cast_mut::<__m512i, [i16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x32, + b: i16x32, + shift: usize, + ) -> i16x32 { + if shift >= 32usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 2usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_i16x32(a).val.0, + idx, + token.cvt_to_bytes_i16x32(b).val.0, + ); + token.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i16x32( + self, + a: i16x32, + b: i16x32, + ) -> i16x32 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 8usize { + return b; + } + let a = self.cvt_to_bytes_i16x32(a).val.0; + let b = self.cvt_to_bytes_i16x32(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 2usize); + self.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i16x32(self, a: i16x32) -> i16x32 { + a ^ !0 + } + #[inline(always)] + fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, shift: u32) -> i16x32 { + _mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, shift: u32) -> i16x32 { + _mm512_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_srav_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpeq_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmplt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmple_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpge_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpgt_epi16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, + 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, + 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, + 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, + 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x32, + b: i16x32, + ) -> (i16x32, i16x32) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, + 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, + 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i16x32, + b: i16x32, + ) -> (i16x32, i16x32) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, + 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, + 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x32, + b: i16x32, + c: i16x32, + ) -> i16x32 { + _mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_min_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, b: i16x32) -> i16x32 { + _mm512_max_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32) -> (i16x16, i16x16) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i16x32(self, a: i16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32) -> i16x32 { + _mm512_sub_epi16(_mm512_setzero_si512(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u16x32(self, val: u16) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u16) -> u16x32 { + _mm512_set1_epi16(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { + crate::transmute::checked_transmute_copy::<__m512i, [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { + crate::transmute::checked_cast_ref::<__m512i, [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { + crate::transmute::checked_cast_mut::<__m512i, [u16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x32, + b: u16x32, + shift: usize, + ) -> u16x32 { + if shift >= 32usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 2usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_u16x32(a).val.0, + idx, + token.cvt_to_bytes_u16x32(b).val.0, + ); + token.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u16x32( + self, + a: u16x32, + b: u16x32, + ) -> u16x32 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 8usize { + return b; + } + let a = self.cvt_to_bytes_u16x32(a).val.0; + let b = self.cvt_to_bytes_u16x32(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 2usize); + self.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u16x32(self, a: u16x32) -> u16x32 { + a ^ !0 + } + #[inline(always)] + fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, shift: u32) -> u16x32 { + _mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_sllv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, shift: u32) -> u16x32 { + _mm512_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_srlv_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpeq_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmplt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmple_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpge_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> mask16x32 { + mask16x32 { + val: _mm512_cmpgt_epu16_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, + 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, + 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, + 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_permutex2var_epi16( + a.into(), + _mm512_set_epi16( + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, + 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x32, + b: u16x32, + ) -> (u16x32, u16x32) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, + 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, + 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u16x32, + b: u16x32, + ) -> (u16x32, u16x32) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, + 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi16( + a, + _mm512_set_epi16( + 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, + 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask16x32, + b: u16x32, + c: u16x32, + ) -> u16x32 { + _mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_min_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, b: u16x32) -> u16x32 { + _mm512_max_epu16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32) -> (u16x16, u16x16) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u16; 32usize]) -> u16x32 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src); + _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, dest: &mut [u16; 32usize]) -> () { + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn narrow_u16x32(self, a: u16x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32) -> u8x32 { + _mm512_cvtepi16_epi8(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask16x32(self, val: bool) -> mask16x32 { + mask16x32 { + val: (if val { 4294967295u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i16; 32usize]) -> mask16x32 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask16x32 { + val: _mm512_movepi16_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask16x32) -> [i16; 32usize] { + let lanes = _mm512_movm_epi16(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { + mask16x32 { + val: (bits & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { + u64::from((a).val) & 4294967295u64 + } + #[inline(always)] + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask16x32 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + mask16x32 { + val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + mask16x32 { + val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + mask16x32 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask16x32(self, a: mask16x32) -> mask16x32 { + mask16x32 { + val: ((!u64::from((a).val)) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask16x32( + self, + a: mask16x32, + b: mask16x32, + c: mask16x32, + ) -> mask16x32 { + mask16x32 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + mask16x32 { + val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask16x32(self, a: mask16x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask16x32(self, a: mask16x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits == 4294967295u64 + } + #[inline(always)] + fn any_false_mask16x32(self, a: mask16x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits != 4294967295u64 + } + #[inline(always)] + fn all_false_mask16x32(self, a: mask16x32) -> bool { + let bits = u64::from((a).val) & 4294967295u64; + bits == 0 + } + #[inline(always)] + fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + let bits = u64::from(a.val); + ( + mask16x16 { + val: (bits & 65535u64) as _, + simd: self, + }, + mask16x16 { + val: ((bits >> 16usize) & 65535u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i32x16(self, val: i32) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i32) -> i32x16 { + _mm512_set1_epi32(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<__m512i, [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { + crate::transmute::checked_cast_ref::<__m512i, [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { + crate::transmute::checked_cast_mut::<__m512i, [i32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x16, + b: i32x16, + shift: usize, + ) -> i32x16 { + if shift >= 16usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_i32x16(a).val.0, + idx, + token.cvt_to_bytes_i32x16(b).val.0, + ); + token.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i32x16( + self, + a: i32x16, + b: i32x16, + ) -> i32x16 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_i32x16(a).val.0; + let b = self.cvt_to_bytes_i32x16(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i32x16(self, a: i32x16) -> i32x16 { + a ^ !0 + } + #[inline(always)] + fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, shift: u32) -> i32x16 { + _mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, shift: u32) -> i32x16 { + _mm512_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_srav_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpeq_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmplt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmple_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpge_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpgt_epi32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x16, + b: i32x16, + ) -> (i32x16, i32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i32x16, + b: i32x16, + ) -> (i32x16, i32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x16, + b: i32x16, + c: i32x16, + ) -> i32x16 { + _mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_min_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, b: i32x16) -> i32x16 { + _mm512_max_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16) -> (i32x8, i32x8) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i32x16(self, a: i32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16) -> i32x16 { + _mm512_sub_epi32(_mm512_setzero_si512(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16) -> f32x16 { + _mm512_cvtepi32_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u32x16(self, val: u32) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u32) -> u32x16 { + _mm512_set1_epi32(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { + u32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { + u32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { + crate::transmute::checked_transmute_copy::<__m512i, [u32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { + crate::transmute::checked_cast_ref::<__m512i, [u32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { + crate::transmute::checked_cast_mut::<__m512i, [u32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { + u32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x16, + b: u32x16, + shift: usize, + ) -> u32x16 { + if shift >= 16usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 4usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_u32x16(a).val.0, + idx, + token.cvt_to_bytes_u32x16(b).val.0, + ); + token.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u32x16( + self, + a: u32x16, + b: u32x16, + ) -> u32x16 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 4usize { + return b; + } + let a = self.cvt_to_bytes_u32x16(a).val.0; + let b = self.cvt_to_bytes_u32x16(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 4usize); + self.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_add_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_sub_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_mullo_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u32x16(self, a: u32x16) -> u32x16 { + a ^ !0 + } + #[inline(always)] + fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, shift: u32) -> u32x16 { + _mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_sllv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, shift: u32) -> u32x16 { + _mm512_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_srlv_epi32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpeq_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmplt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmple_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpge_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> mask32x16 { + mask32x16 { + val: _mm512_cmpgt_epu32_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_permutex2var_epi32( + a.into(), + _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x16, + b: u32x16, + ) -> (u32x16, u32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u32x16, + b: u32x16, + ) -> (u32x16, u32x16) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + ), + b, + ) + .simd_into(token), + _mm512_permutex2var_epi32( + a, + _mm512_setr_epi32( + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ), + b, + ) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask32x16, + b: u32x16, + c: u32x16, + ) -> u32x16 { + _mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_min_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, b: u32x16) -> u32x16 { + _mm512_max_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16) -> (u32x8, u32x8) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u32; 16usize]) -> u32x16 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src); + _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, dest: &mut [u32; 16usize]) -> () { + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16) -> f32x16 { + _mm512_cvtepu32_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask32x16(self, val: bool) -> mask32x16 { + mask32x16 { + val: (if val { 65535u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i32; 16usize]) -> mask32x16 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask32x16 { + val: _mm512_movepi32_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask32x16) -> [i32; 16usize] { + let lanes = _mm512_movm_epi32(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { + mask32x16 { + val: (bits & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { + u64::from((a).val) & 65535u64 + } + #[inline(always)] + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask32x16 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + mask32x16 { + val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + mask32x16 { + val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + mask32x16 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask32x16(self, a: mask32x16) -> mask32x16 { + mask32x16 { + val: ((!u64::from((a).val)) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask32x16( + self, + a: mask32x16, + b: mask32x16, + c: mask32x16, + ) -> mask32x16 { + mask32x16 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + mask32x16 { + val: (!u64::from(a.val ^ b.val) & 65535u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask32x16(self, a: mask32x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask32x16(self, a: mask32x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 65535u64 + } + #[inline(always)] + fn any_false_mask32x16(self, a: mask32x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits != 65535u64 + } + #[inline(always)] + fn all_false_mask32x16(self, a: mask32x16) -> bool { + let bits = u64::from((a).val) & 65535u64; + bits == 0 + } + #[inline(always)] + fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { + let bits = u64::from(a.val); + ( + mask32x8 { + val: (bits & 255u64) as _, + simd: self, + }, + mask32x8 { + val: ((bits >> 8usize) & 255u64) as _, + simd: self, + }, + ) + } + #[inline(always)] + fn splat_f64x8(self, val: f64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: f64) -> f64x8 { + _mm512_set1_pd(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { + f64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { + f64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { + crate::transmute::checked_transmute_copy::<__m512d, [f64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { + crate::transmute::checked_cast_ref::<__m512d, [f64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { + crate::transmute::checked_cast_mut::<__m512d, [f64; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { + f64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x8, + b: f64x8, + shift: usize, + ) -> f64x8 { + if shift >= 8usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_f64x8(a).val.0, + idx, + token.cvt_to_bytes_f64x8(b).val.0, + ); + token.cvt_from_bytes_f64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_f64x8( + self, + a: f64x8, + b: f64x8, + ) -> f64x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_f64x8(a).val.0; + let b = self.cvt_to_bytes_f64x8(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_f64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn abs_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_andnot_pd(_mm512_set1_pd(-0.0), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_xor_pd(a.into(), _mm512_set1_pd(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_sqrt_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_rcp14_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_add_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_sub_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_mul_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_div_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + let mask = _mm512_set1_pd(-0.0); + _mm512_or_pd( + _mm512_and_pd(mask, b.into()), + _mm512_andnot_pd(mask, a.into()), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmp_pd_mask::<0i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmp_pd_mask::<17i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmp_pd_mask::<18i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmp_pd_mask::<29i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmp_pd_mask::<30i32>(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_permutex2var_pd( + a.into(), + _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_permutex2var_pd( + a.into(), + _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_permutex2var_pd( + a.into(), + _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_permutex2var_pd( + a.into(), + _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x8, + b: f64x8, + ) -> (f64x8, f64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm512_permutex2var_pd(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x8, + b: f64x8, + ) -> (f64x8, f64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm512_permutex2var_pd(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_max_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_min_pd(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_range_pd::<5i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, b: f64x8) -> f64x8 { + _mm512_range_pd::<4i32>(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x8, + b: f64x8, + c: f64x8, + ) -> f64x8 { + _mm512_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: f64x8, + b: f64x8, + c: f64x8, + ) -> f64x8 { + _mm512_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn floor_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_roundscale_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn ceil_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_roundscale_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_roundscale_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn fract_f64x8(self, a: f64x8) -> f64x8 { + a - self.trunc_f64x8(a) + } + #[inline(always)] + fn trunc_f64x8(self, a: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f64x8 { + _mm512_roundscale_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into()) + .simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x8, + b: f64x8, + c: f64x8, + ) -> f64x8 { + _mm512_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> (f64x4, f64x4) { + ( + _mm512_castpd512_pd256(a.into()).simd_into(token), + _mm512_extractf64x4_pd::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> f32x16 { + _mm512_castpd_ps(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_i64x8(self, val: i64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: i64) -> i64x8 { + _mm512_set1_epi64(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + crate::transmute::checked_transmute_copy::<__m512i, [i64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + crate::transmute::checked_cast_ref::<__m512i, [i64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + crate::transmute::checked_cast_mut::<__m512i, [i64; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x8, + b: i64x8, + shift: usize, + ) -> i64x8 { + if shift >= 8usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_i64x8(a).val.0, + idx, + token.cvt_to_bytes_i64x8(b).val.0, + ); + token.cvt_from_bytes_i64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_i64x8( + self, + a: i64x8, + b: i64x8, + ) -> i64x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_i64x8(a).val.0; + let b = self.cvt_to_bytes_i64x8(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_i64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + a ^ !0 + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, shift: u32) -> i64x8 { + _mm512_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, shift: u32) -> i64x8 { + _mm512_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_srav_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpeq_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmplt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmple_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpge_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpgt_epi64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x8, + b: i64x8, + ) -> (i64x8, i64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: i64x8, + b: i64x8, + ) -> (i64x8, i64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x8, + b: i64x8, + c: i64x8, + ) -> i64x8 { + _mm512_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_min_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, b: i64x8) -> i64x8 { + _mm512_max_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8) -> (i64x4, i64x4) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8) -> i64x8 { + _mm512_sub_epi64(_mm512_setzero_si512(), a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u64x8(self, val: u64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: u64) -> u64x8 { + _mm512_set1_epi64(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<__m512i, [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::<__m512i, [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::<__m512i, [u64; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x8, + b: u64x8, + shift: usize, + ) -> u64x8 { + if shift >= 8usize { + return b; + } + let idx = _mm512_add_epi8( + _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, + 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4, 3, 2, 1, 0, + ), + _mm512_set1_epi8((shift * 8usize) as i8), + ); + let result = _mm512_permutex2var_epi8( + token.cvt_to_bytes_u64x8(a).val.0, + idx, + token.cvt_to_bytes_u64x8(b).val.0, + ); + token.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, b, SHIFT) + } + #[inline(always)] + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 2usize { + return b; + } + let a = self.cvt_to_bytes_u64x8(a).val.0; + let b = self.cvt_to_bytes_u64x8(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT * 8usize); + self.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_mullo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_and_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_or_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_xor_si512(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_u64x8(self, a: u64x8) -> u64x8 { + a ^ !0 + } + #[inline(always)] + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, shift: u32) -> u64x8 { + _mm512_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_sllv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, shift: u32) -> u64x8 { + _mm512_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_srlv_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpeq_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmplt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmple_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpge_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> mask64x8 { + mask64x8 { + val: _mm512_cmpgt_epu64_mask(a.into(), b.into()), + simd: token, + } + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_permutex2var_epi64( + a.into(), + _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), + b.into(), + ) + .simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x8, + b: u64x8, + ) -> (u64x8, u64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b) + .simd_into(token), + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: u64x8, + b: u64x8, + ) -> (u64x8, u64x8) { + let a = a.into(); + let b = b.into(); + ( + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b) + .simd_into(token), + _mm512_permutex2var_epi64(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b) + .simd_into(token), + ) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + a: mask64x8, + b: u64x8, + c: u64x8, + ) -> u64x8 { + _mm512_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_min_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, b: u64x8) -> u64x8 { + _mm512_max_epu64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8) -> (u64x4, u64x4) { + ( + _mm512_castsi512_si256(a.into()).simd_into(token), + _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token), + ) + } + ); + kernel(self, a) + } + #[inline(always)] + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u64; 8usize]) -> u64x8 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src); + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, dest: &mut [u64; 8usize]) -> () { + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); + crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8) -> u8x64 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8) -> u32x16 { + __m512i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask64x8(self, val: bool) -> mask64x8 { + mask64x8 { + val: (if val { 255u64 } else { 0 }) as _, + simd: self, + } + } + #[inline(always)] + fn load_array_mask64x8(self, val: [i64; 8usize]) -> mask64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, val: [i64; 8usize]) -> mask64x8 { + let lanes = crate::transmute::checked_transmute_copy(&val); + mask64x8 { + val: _mm512_movepi64_mask(lanes), + simd: token, + } + } + ); + kernel(self, val) + } + #[inline(always)] + fn as_array_mask64x8(self, a: mask64x8) -> [i64; 8usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: mask64x8) -> [i64; 8usize] { + let lanes = _mm512_movm_epi64(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + ); + kernel(self, a) + } + #[inline(always)] + fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8 { + mask64x8 { + val: (bits & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn to_bitmask_mask64x8(self, a: mask64x8) -> u64 { + u64::from((a).val) & 255u64 + } + #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let bit = 1u64 << index; + let bits = u64::from((a).val); + let bits = if value { bits | bit } else { bits & !bit }; + *a = mask64x8 { + val: (bits) as _, + simd: self, + }; + } + #[inline(always)] + fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { + mask64x8 { + val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn or_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { + mask64x8 { + val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn xor_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { + mask64x8 { + val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn not_mask64x8(self, a: mask64x8) -> mask64x8 { + mask64x8 { + val: ((!u64::from((a).val)) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn select_mask64x8( + self, + a: mask64x8, + b: mask64x8, + c: mask64x8, + ) -> mask64x8 { + mask64x8 { + val: (((u64::from((a).val) & u64::from((b).val)) + | ((!u64::from((a).val)) & u64::from((c).val))) + & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn simd_eq_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { + mask64x8 { + val: (!u64::from(a.val ^ b.val) & 255u64) as _, + simd: self, + } + } + #[inline(always)] + fn any_true_mask64x8(self, a: mask64x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 0 + } + #[inline(always)] + fn all_true_mask64x8(self, a: mask64x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 255u64 + } + #[inline(always)] + fn any_false_mask64x8(self, a: mask64x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits != 255u64 + } + #[inline(always)] + fn all_false_mask64x8(self, a: mask64x8) -> bool { + let bits = u64::from((a).val) & 255u64; + bits == 0 + } + #[inline(always)] + fn split_mask64x8(self, a: mask64x8) -> (mask64x4, mask64x4) { + let bits = u64::from(a.val); + ( + mask64x4 { + val: (bits & 15u64) as _, + simd: self, + }, + mask64x4 { + val: ((bits >> 4usize) & 15u64) as _, + simd: self, + }, + ) + } +} +impl SimdFrom<__mmask16, S> for mask8x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask16) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask16 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask8x16) -> Self { + value.to_bitmask() as __mmask16 + } +} +impl SimdFrom<__mmask8, S> for mask16x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask16x8) -> Self { + value.to_bitmask() as __mmask8 + } +} +impl SimdFrom<__mmask8, S> for mask32x4 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask32x4) -> Self { + value.to_bitmask() as __mmask8 + } +} +impl SimdFrom<__mmask8, S> for mask64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask64x2) -> Self { + value.to_bitmask() as __mmask8 + } +} +impl SimdFrom<__mmask32, S> for mask8x32 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask32) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask32 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask8x32) -> Self { + value.to_bitmask() as __mmask32 + } +} +impl SimdFrom<__mmask16, S> for mask16x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask16) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask16 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask16x16) -> Self { + value.to_bitmask() as __mmask16 + } +} +impl SimdFrom<__mmask8, S> for mask32x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask32x8) -> Self { + value.to_bitmask() as __mmask8 + } +} +impl SimdFrom<__mmask8, S> for mask64x4 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask64x4) -> Self { + value.to_bitmask() as __mmask8 + } +} +impl SimdFrom<__m512, S> for f32x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512 { + #[inline(always)] + fn from(value: f32x16) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for i8x64 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: i8x64) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for u8x64 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: u8x64) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__mmask64, S> for mask8x64 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask64) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask64 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask8x64) -> Self { + value.to_bitmask() as __mmask64 + } +} +impl SimdFrom<__m512i, S> for i16x32 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: i16x32) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for u16x32 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: u16x32) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__mmask32, S> for mask16x32 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask32) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask32 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask16x32) -> Self { + value.to_bitmask() as __mmask32 + } +} +impl SimdFrom<__m512i, S> for i32x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: i32x16) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for u32x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: u32x16) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__mmask16, S> for mask32x16 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask16) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask16 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask32x16) -> Self { + value.to_bitmask() as __mmask16 + } +} +impl SimdFrom<__m512d, S> for f64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512d) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512d { + #[inline(always)] + fn from(value: f64x8) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for i64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: i64x8) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m512i, S> for u64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __m512i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m512i { + #[inline(always)] + fn from(value: u64x8) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__mmask8, S> for mask64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: __mmask8) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } +} +impl From> for __mmask8 { + #[inline(always)] + #[allow( + trivial_numeric_casts, + reason = "generated uniformly for all __mmask widths" + )] + fn from(value: mask64x8) -> Self { + value.to_bitmask() as __mmask8 + } +} +crate::kernel!( + #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"] + #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"] + #[doc = r" Rust doesn't currently let you do math on const generics."] + #[inline(always)] + fn dyn_alignr_128(token: Avx512, a: __m128i, b: __m128i, shift: usize) -> __m128i { + match shift { + 0usize => _mm_alignr_epi8::<0i32>(a, b), + 1usize => _mm_alignr_epi8::<1i32>(a, b), + 2usize => _mm_alignr_epi8::<2i32>(a, b), + 3usize => _mm_alignr_epi8::<3i32>(a, b), + 4usize => _mm_alignr_epi8::<4i32>(a, b), + 5usize => _mm_alignr_epi8::<5i32>(a, b), + 6usize => _mm_alignr_epi8::<6i32>(a, b), + 7usize => _mm_alignr_epi8::<7i32>(a, b), + 8usize => _mm_alignr_epi8::<8i32>(a, b), + 9usize => _mm_alignr_epi8::<9i32>(a, b), + 10usize => _mm_alignr_epi8::<10i32>(a, b), + 11usize => _mm_alignr_epi8::<11i32>(a, b), + 12usize => _mm_alignr_epi8::<12i32>(a, b), + 13usize => _mm_alignr_epi8::<13i32>(a, b), + 14usize => _mm_alignr_epi8::<14i32>(a, b), + 15usize => _mm_alignr_epi8::<15i32>(a, b), + _ => unreachable!(), + } + } +); +crate::kernel!( + #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"] + #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"] + #[doc = r" Rust doesn't currently let you do math on const generics."] + #[inline(always)] + fn dyn_alignr_256(token: Avx512, a: __m256i, b: __m256i, shift: usize) -> __m256i { + match shift { + 0usize => _mm256_alignr_epi8::<0i32>(a, b), + 1usize => _mm256_alignr_epi8::<1i32>(a, b), + 2usize => _mm256_alignr_epi8::<2i32>(a, b), + 3usize => _mm256_alignr_epi8::<3i32>(a, b), + 4usize => _mm256_alignr_epi8::<4i32>(a, b), + 5usize => _mm256_alignr_epi8::<5i32>(a, b), + 6usize => _mm256_alignr_epi8::<6i32>(a, b), + 7usize => _mm256_alignr_epi8::<7i32>(a, b), + 8usize => _mm256_alignr_epi8::<8i32>(a, b), + 9usize => _mm256_alignr_epi8::<9i32>(a, b), + 10usize => _mm256_alignr_epi8::<10i32>(a, b), + 11usize => _mm256_alignr_epi8::<11i32>(a, b), + 12usize => _mm256_alignr_epi8::<12i32>(a, b), + 13usize => _mm256_alignr_epi8::<13i32>(a, b), + 14usize => _mm256_alignr_epi8::<14i32>(a, b), + 15usize => _mm256_alignr_epi8::<15i32>(a, b), + _ => unreachable!(), + } + } +); +crate::kernel!( + #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"] + #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"] + #[doc = r" Rust doesn't currently let you do math on const generics."] + #[inline(always)] + fn dyn_alignr_512(token: Avx512, a: __m512i, b: __m512i, shift: usize) -> __m512i { + match shift { + 0usize => _mm512_alignr_epi8::<0i32>(a, b), + 1usize => _mm512_alignr_epi8::<1i32>(a, b), + 2usize => _mm512_alignr_epi8::<2i32>(a, b), + 3usize => _mm512_alignr_epi8::<3i32>(a, b), + 4usize => _mm512_alignr_epi8::<4i32>(a, b), + 5usize => _mm512_alignr_epi8::<5i32>(a, b), + 6usize => _mm512_alignr_epi8::<6i32>(a, b), + 7usize => _mm512_alignr_epi8::<7i32>(a, b), + 8usize => _mm512_alignr_epi8::<8i32>(a, b), + 9usize => _mm512_alignr_epi8::<9i32>(a, b), + 10usize => _mm512_alignr_epi8::<10i32>(a, b), + 11usize => _mm512_alignr_epi8::<11i32>(a, b), + 12usize => _mm512_alignr_epi8::<12i32>(a, b), + 13usize => _mm512_alignr_epi8::<13i32>(a, b), + 14usize => _mm512_alignr_epi8::<14i32>(a, b), + 15usize => _mm512_alignr_epi8::<15i32>(a, b), + _ => unreachable!(), + } + } +); diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 6ec6d23b9..89393c87c 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -6,9 +6,9 @@ use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; use core::ops::*; #[cfg(all(feature = "libm", not(feature = "std")))] @@ -98,6 +98,8 @@ impl ArchTypes for Fallback { type u32x4 = crate::support::Aligned128<[u32; 4usize]>; type mask32x4 = crate::support::Aligned128<[i32; 4usize]>; type f64x2 = crate::support::Aligned128<[f64; 2usize]>; + type i64x2 = crate::support::Aligned128<[i64; 2usize]>; + type u64x2 = crate::support::Aligned128<[u64; 2usize]>; type mask64x2 = crate::support::Aligned128<[i64; 2usize]>; type f32x8 = crate::support::Aligned256<[f32; 8usize]>; type i8x32 = crate::support::Aligned256<[i8; 32usize]>; @@ -110,6 +112,8 @@ impl ArchTypes for Fallback { type u32x8 = crate::support::Aligned256<[u32; 8usize]>; type mask32x8 = crate::support::Aligned256<[i32; 8usize]>; type f64x4 = crate::support::Aligned256<[f64; 4usize]>; + type i64x4 = crate::support::Aligned256<[i64; 4usize]>; + type u64x4 = crate::support::Aligned256<[u64; 4usize]>; type mask64x4 = crate::support::Aligned256<[i64; 4usize]>; type f32x16 = crate::support::Aligned512<[f32; 16usize]>; type i8x64 = crate::support::Aligned512<[i8; 64usize]>; @@ -122,6 +126,8 @@ impl ArchTypes for Fallback { type u32x16 = crate::support::Aligned512<[u32; 16usize]>; type mask32x16 = crate::support::Aligned512<[i32; 16usize]>; type f64x8 = crate::support::Aligned512<[f64; 8usize]>; + type i64x8 = crate::support::Aligned512<[i64; 8usize]>; + type u64x8 = crate::support::Aligned512<[u64; 8usize]>; type mask64x8 = crate::support::Aligned512<[i64; 8usize]>; } impl Simd for Fallback { @@ -133,6 +139,8 @@ impl Simd for Fallback { type i16s = i16x8; type u32s = u32x4; type i32s = i32x4; + type u64s = u64x2; + type i64s = i64x2; type mask8s = mask8x16; type mask16s = mask16x8; type mask32s = mask32x4; @@ -1811,8 +1819,24 @@ impl Simd for Fallback { } #[inline(always)] fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16 { - let lanes: [i8; 16usize] = - core::array::from_fn(|i| if ((bits >> i) & 1) != 0 { !0 } else { 0 }); + let lanes: [i8; 16usize] = [ + if bits & 1 != 0 { !0 } else { 0 }, + if (bits >> 1usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 2usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 3usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 4usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 5usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 6usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 7usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 8usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 9usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 10usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 11usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 12usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 13usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 14usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 15usize) & 1 != 0 { !0 } else { 0 }, + ]; lanes.simd_into(self) } #[inline(always)] @@ -1829,6 +1853,17 @@ impl Simd for Fallback { bits } #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask8x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x16(lanes); + } + #[inline(always)] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { [ i8::bitand(a.val.0[0usize], &b.val.0[0usize]), @@ -2968,8 +3003,16 @@ impl Simd for Fallback { } #[inline(always)] fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8 { - let lanes: [i16; 8usize] = - core::array::from_fn(|i| if ((bits >> i) & 1) != 0 { !0 } else { 0 }); + let lanes: [i16; 8usize] = [ + if bits & 1 != 0 { !0 } else { 0 }, + if (bits >> 1usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 2usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 3usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 4usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 5usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 6usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 7usize) & 1 != 0 { !0 } else { 0 }, + ]; lanes.simd_into(self) } #[inline(always)] @@ -2986,6 +3029,17 @@ impl Simd for Fallback { bits } #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask16x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x8(lanes); + } + #[inline(always)] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { [ i16::bitand(a.val.0[0usize], &b.val.0[0usize]), @@ -3817,8 +3871,12 @@ impl Simd for Fallback { } #[inline(always)] fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4 { - let lanes: [i32; 4usize] = - core::array::from_fn(|i| if ((bits >> i) & 1) != 0 { !0 } else { 0 }); + let lanes: [i32; 4usize] = [ + if bits & 1 != 0 { !0 } else { 0 }, + if (bits >> 1usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 2usize) & 1 != 0 { !0 } else { 0 }, + if (bits >> 3usize) & 1 != 0 { !0 } else { 0 }, + ]; lanes.simd_into(self) } #[inline(always)] @@ -3835,6 +3893,17 @@ impl Simd for Fallback { bits } #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask32x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x4(lanes); + } + #[inline(always)] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { [ i32::bitand(a.val.0[0usize], &b.val.0[0usize]), @@ -4202,14 +4271,518 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] - fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { - let mut result = [0.0; 4usize]; + fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { + let mut result = [0.0; 4usize]; + result[0..2usize].copy_from_slice(&a.val.0); + result[2usize..4usize].copy_from_slice(&b.val.0); + result.simd_into(self) + } + #[inline(always)] + fn reinterpret_f32_f64x2(self, a: f64x2) -> f32x4 { + a.bitcast() + } + #[inline(always)] + fn splat_i64x2(self, val: i64) -> i64x2 { + [val; 2usize].simd_into(self) + } + #[inline(always)] + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::support::Aligned128(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::support::Aligned128(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let mut dest = [Default::default(); 2usize]; + dest[..2usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[2usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::wrapping_add(a[0usize], b[0usize]), + i64::wrapping_add(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::wrapping_sub(a[0usize], b[0usize]), + i64::wrapping_sub(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::wrapping_mul(a[0usize], b[0usize]), + i64::wrapping_mul(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::bitand(a[0usize], &b[0usize]), + i64::bitand(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::bitor(a[0usize], &b[0usize]), + i64::bitor(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::bitxor(a[0usize], &b[0usize]), + i64::bitxor(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn not_i64x2(self, a: i64x2) -> i64x2 { + [i64::not(a[0usize]), i64::not(a[1usize])].simd_into(self) + } + #[inline(always)] + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + [i64::shl(a[0usize], shift), i64::shl(a[1usize], shift)].simd_into(self) + } + #[inline(always)] + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::shl(a[0usize], &b[0usize]), + i64::shl(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + [i64::shr(a[0usize], shift), i64::shr(a[1usize], shift)].simd_into(self) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::shr(a[0usize], &b[0usize]), + i64::shr(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + [ + -(i64::eq(&a[0usize], &b[0usize]) as i64), + -(i64::eq(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + [ + -(i64::lt(&a[0usize], &b[0usize]) as i64), + -(i64::lt(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + [ + -(i64::le(&a[0usize], &b[0usize]) as i64), + -(i64::le(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + [ + -(i64::ge(&a[0usize], &b[0usize]) as i64), + -(i64::ge(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + [ + -(i64::gt(&a[0usize], &b[0usize]) as i64), + -(i64::gt(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [a[0usize], b[0usize]].simd_into(self) + } + #[inline(always)] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [a[1usize], b[1usize]].simd_into(self) + } + #[inline(always)] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [a[0usize], b[0usize]].simd_into(self) + } + #[inline(always)] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [a[1usize], b[1usize]].simd_into(self) + } + #[inline(always)] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) + } + #[inline(always)] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b)) + } + #[inline(always)] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { + [ + if a.val.0[0usize] != 0 { + b[0usize] + } else { + c[0usize] + }, + if a.val.0[1usize] != 0 { + b[1usize] + } else { + c[1usize] + }, + ] + .simd_into(self) + } + #[inline(always)] + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::min(a[0usize], b[0usize]), + i64::min(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + [ + i64::max(a[0usize], b[0usize]), + i64::max(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { + let mut result = [0; 4usize]; + result[0..2usize].copy_from_slice(&a.val.0); + result[2usize..4usize].copy_from_slice(&b.val.0); + result.simd_into(self) + } + #[inline(always)] + fn neg_i64x2(self, a: i64x2) -> i64x2 { + [i64::neg(a[0usize]), i64::neg(a[1usize])].simd_into(self) + } + #[inline(always)] + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { + a.bitcast() + } + #[inline(always)] + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { + a.bitcast() + } + #[inline(always)] + fn splat_u64x2(self, val: u64) -> u64x2 { + [val; 2usize].simd_into(self) + } + #[inline(always)] + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::support::Aligned128(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { + val: crate::support::Aligned128(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let mut dest = [Default::default(); 2usize]; + dest[..2usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[2usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_u64x2( + self, + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) + } + #[inline(always)] + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::wrapping_add(a[0usize], b[0usize]), + u64::wrapping_add(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::wrapping_sub(a[0usize], b[0usize]), + u64::wrapping_sub(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::wrapping_mul(a[0usize], b[0usize]), + u64::wrapping_mul(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::bitand(a[0usize], &b[0usize]), + u64::bitand(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::bitor(a[0usize], &b[0usize]), + u64::bitor(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::bitxor(a[0usize], &b[0usize]), + u64::bitxor(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn not_u64x2(self, a: u64x2) -> u64x2 { + [u64::not(a[0usize]), u64::not(a[1usize])].simd_into(self) + } + #[inline(always)] + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + [u64::shl(a[0usize], shift), u64::shl(a[1usize], shift)].simd_into(self) + } + #[inline(always)] + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::shl(a[0usize], &b[0usize]), + u64::shl(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + [u64::shr(a[0usize], shift), u64::shr(a[1usize], shift)].simd_into(self) + } + #[inline(always)] + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::shr(a[0usize], &b[0usize]), + u64::shr(a[1usize], &b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + [ + -(u64::eq(&a[0usize], &b[0usize]) as i64), + -(u64::eq(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + [ + -(u64::lt(&a[0usize], &b[0usize]) as i64), + -(u64::lt(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + [ + -(u64::le(&a[0usize], &b[0usize]) as i64), + -(u64::le(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + [ + -(u64::ge(&a[0usize], &b[0usize]) as i64), + -(u64::ge(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + [ + -(u64::gt(&a[0usize], &b[0usize]) as i64), + -(u64::gt(&a[1usize], &b[1usize]) as i64), + ] + .simd_into(self) + } + #[inline(always)] + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [a[0usize], b[0usize]].simd_into(self) + } + #[inline(always)] + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [a[1usize], b[1usize]].simd_into(self) + } + #[inline(always)] + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [a[0usize], b[0usize]].simd_into(self) + } + #[inline(always)] + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [a[1usize], b[1usize]].simd_into(self) + } + #[inline(always)] + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) + } + #[inline(always)] + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b)) + } + #[inline(always)] + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { + [ + if a.val.0[0usize] != 0 { + b[0usize] + } else { + c[0usize] + }, + if a.val.0[1usize] != 0 { + b[1usize] + } else { + c[1usize] + }, + ] + .simd_into(self) + } + #[inline(always)] + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::min(a[0usize], b[0usize]), + u64::min(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + [ + u64::max(a[0usize], b[0usize]), + u64::max(a[1usize], b[1usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { + let mut result = [0; 4usize]; result[0..2usize].copy_from_slice(&a.val.0); result[2usize..4usize].copy_from_slice(&b.val.0); result.simd_into(self) } #[inline(always)] - fn reinterpret_f32_f64x2(self, a: f64x2) -> f32x4 { + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { + a.bitcast() + } + #[inline(always)] + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { a.bitcast() } #[inline(always)] @@ -4230,8 +4803,10 @@ impl Simd for Fallback { } #[inline(always)] fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { - let lanes: [i64; 2usize] = - core::array::from_fn(|i| if ((bits >> i) & 1) != 0 { !0 } else { 0 }); + let lanes: [i64; 2usize] = [ + if bits & 1 != 0 { !0 } else { 0 }, + if (bits >> 1usize) & 1 != 0 { !0 } else { 0 }, + ]; lanes.simd_into(self) } #[inline(always)] @@ -4248,6 +4823,17 @@ impl Simd for Fallback { bits } #[inline(always)] + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let mut lanes = self.as_array_mask64x2(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x2(lanes); + } + #[inline(always)] fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { [ i64::bitand(a.val.0[0usize], &b.val.0[0usize]), @@ -5237,6 +5823,17 @@ impl Simd for Fallback { lo | (hi << 16usize) } #[inline(always)] + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask8x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x32(lanes); + } + #[inline(always)] fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { let (a0, a1) = self.split_mask8x32(a); let (b0, b1) = self.split_mask8x32(b); @@ -5890,6 +6487,17 @@ impl Simd for Fallback { lo | (hi << 8usize) } #[inline(always)] + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask16x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x16(lanes); + } + #[inline(always)] fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { let (a0, a1) = self.split_mask16x16(a); let (b0, b1) = self.split_mask16x16(b); @@ -6523,6 +7131,17 @@ impl Simd for Fallback { lo | (hi << 4usize) } #[inline(always)] + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask32x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x8(lanes); + } + #[inline(always)] fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { let (a0, a1) = self.split_mask32x8(a); let (b0, b1) = self.split_mask32x8(b); @@ -6912,6 +7531,529 @@ impl Simd for Fallback { ) } #[inline(always)] + fn splat_i64x4(self, val: i64) -> i64x4 { + let half = self.splat_i64x2(val); + self.combine_i64x2(half, half) + } + #[inline(always)] + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::support::Aligned256(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::support::Aligned256(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let mut dest = [Default::default(); 4usize]; + dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_i64x4( + self, + a: i64x4, + b: i64x4, + ) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2( + self.slide_within_blocks_i64x2::(a0, b0), + self.slide_within_blocks_i64x2::(a1, b1), + ) + } + #[inline(always)] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) + } + #[inline(always)] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) + } + #[inline(always)] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) + } + #[inline(always)] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) + } + #[inline(always)] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) + } + #[inline(always)] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) + } + #[inline(always)] + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) + } + #[inline(always)] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) + } + #[inline(always)] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) + } + #[inline(always)] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) + } + #[inline(always)] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) + } + #[inline(always)] + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_lo = self.zip_low_i64x2(a0, b0); + let lo_hi = self.zip_high_i64x2(a0, b0); + let hi_lo = self.zip_low_i64x2(a1, b1); + let hi_hi = self.zip_high_i64x2(a1, b1); + ( + self.combine_i64x2(lo_lo, lo_hi), + self.combine_i64x2(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_even = self.unzip_low_i64x2(a0, a1); + let lo_odd = self.unzip_high_i64x2(a0, a1); + let hi_even = self.unzip_low_i64x2(b0, b1); + let hi_odd = self.unzip_high_i64x2(b0, b1); + ( + self.combine_i64x2(lo_even, hi_even), + self.combine_i64x2(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_i64x4(b); + let (c0, c1) = self.split_i64x4(c); + self.combine_i64x2(self.select_i64x2(a0, b0, c0), self.select_i64x2(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.min_i64x2(a0, b0), self.min_i64x2(a1, b1)) + } + #[inline(always)] + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.max_i64x2(a0, b0), self.max_i64x2(a1, b1)) + } + #[inline(always)] + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + let mut result = [0; 8usize]; + result[0..4usize].copy_from_slice(&a.val.0); + result[4usize..8usize].copy_from_slice(&b.val.0); + result.simd_into(self) + } + #[inline(always)] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { + let mut b0 = [0; 2usize]; + let mut b1 = [0; 2usize]; + b0.copy_from_slice(&a.val.0[0..2usize]); + b1.copy_from_slice(&a.val.0[2usize..4usize]); + (b0.simd_into(self), b1.simd_into(self)) + } + #[inline(always)] + fn neg_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u8x16(self.reinterpret_u8_i64x2(a0), self.reinterpret_u8_i64x2(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u32x4( + self.reinterpret_u32_i64x2(a0), + self.reinterpret_u32_i64x2(a1), + ) + } + #[inline(always)] + fn splat_u64x4(self, val: u64) -> u64x4 { + let half = self.splat_u64x2(val); + self.combine_u64x2(half, half) + } + #[inline(always)] + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::support::Aligned256(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { + val: crate::support::Aligned256(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let mut dest = [Default::default(); 4usize]; + dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_u64x4( + self, + a: u64x4, + b: u64x4, + ) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2( + self.slide_within_blocks_u64x2::(a0, b0), + self.slide_within_blocks_u64x2::(a1, b1), + ) + } + #[inline(always)] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) + } + #[inline(always)] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) + } + #[inline(always)] + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1)) + } + #[inline(always)] + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.and_u64x2(a0, b0), self.and_u64x2(a1, b1)) + } + #[inline(always)] + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.or_u64x2(a0, b0), self.or_u64x2(a1, b1)) + } + #[inline(always)] + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.xor_u64x2(a0, b0), self.xor_u64x2(a1, b1)) + } + #[inline(always)] + fn not_u64x4(self, a: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.not_u64x2(a0), self.not_u64x2(a1)) + } + #[inline(always)] + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shl_u64x2(a0, shift), self.shl_u64x2(a1, shift)) + } + #[inline(always)] + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shlv_u64x2(a0, b0), self.shlv_u64x2(a1, b1)) + } + #[inline(always)] + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shr_u64x2(a0, shift), self.shr_u64x2(a1, shift)) + } + #[inline(always)] + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shrv_u64x2(a0, b0), self.shrv_u64x2(a1, b1)) + } + #[inline(always)] + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_eq_u64x2(a0, b0), self.simd_eq_u64x2(a1, b1)) + } + #[inline(always)] + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_lt_u64x2(a0, b0), self.simd_lt_u64x2(a1, b1)) + } + #[inline(always)] + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_le_u64x2(a0, b0), self.simd_le_u64x2(a1, b1)) + } + #[inline(always)] + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_ge_u64x2(a0, b0), self.simd_ge_u64x2(a1, b1)) + } + #[inline(always)] + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_gt_u64x2(a0, b0), self.simd_gt_u64x2(a1, b1)) + } + #[inline(always)] + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, _) = self.split_u64x4(a); + let (b0, _) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a0, b0), self.zip_high_u64x2(a0, b0)) + } + #[inline(always)] + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (_, a1) = self.split_u64x4(a); + let (_, b1) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a1, b1), self.zip_high_u64x2(a1, b1)) + } + #[inline(always)] + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_low_u64x2(a0, a1), self.unzip_low_u64x2(b0, b1)) + } + #[inline(always)] + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_high_u64x2(a0, a1), self.unzip_high_u64x2(b0, b1)) + } + #[inline(always)] + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_lo = self.zip_low_u64x2(a0, b0); + let lo_hi = self.zip_high_u64x2(a0, b0); + let hi_lo = self.zip_low_u64x2(a1, b1); + let hi_hi = self.zip_high_u64x2(a1, b1); + ( + self.combine_u64x2(lo_lo, lo_hi), + self.combine_u64x2(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_even = self.unzip_low_u64x2(a0, a1); + let lo_odd = self.unzip_high_u64x2(a0, a1); + let hi_even = self.unzip_low_u64x2(b0, b1); + let hi_odd = self.unzip_high_u64x2(b0, b1); + ( + self.combine_u64x2(lo_even, hi_even), + self.combine_u64x2(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_u64x4(b); + let (c0, c1) = self.split_u64x4(c); + self.combine_u64x2(self.select_u64x2(a0, b0, c0), self.select_u64x2(a1, b1, c1)) + } + #[inline(always)] + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.min_u64x2(a0, b0), self.min_u64x2(a1, b1)) + } + #[inline(always)] + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.max_u64x2(a0, b0), self.max_u64x2(a1, b1)) + } + #[inline(always)] + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + let mut result = [0; 8usize]; + result[0..4usize].copy_from_slice(&a.val.0); + result[4usize..8usize].copy_from_slice(&b.val.0); + result.simd_into(self) + } + #[inline(always)] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { + let mut b0 = [0; 2usize]; + let mut b1 = [0; 2usize]; + b0.copy_from_slice(&a.val.0[0..2usize]); + b1.copy_from_slice(&a.val.0[2usize..4usize]); + (b0.simd_into(self), b1.simd_into(self)) + } + #[inline(always)] + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u8x16(self.reinterpret_u8_u64x2(a0), self.reinterpret_u8_u64x2(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u32x4( + self.reinterpret_u32_u64x2(a0), + self.reinterpret_u32_u64x2(a1), + ) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -6941,6 +8083,17 @@ impl Simd for Fallback { lo | (hi << 2usize) } #[inline(always)] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask64x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x4(lanes); + } + #[inline(always)] fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { let (a0, a1) = self.split_mask64x4(a); let (b0, b1) = self.split_mask64x4(b); @@ -8018,6 +9171,17 @@ impl Simd for Fallback { lo | (hi << 32usize) } #[inline(always)] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let mut lanes = self.as_array_mask8x64(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x64(lanes); + } + #[inline(always)] fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { let (a0, a1) = self.split_mask8x64(a); let (b0, b1) = self.split_mask8x64(b); @@ -8699,6 +9863,17 @@ impl Simd for Fallback { lo | (hi << 16usize) } #[inline(always)] + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask16x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x32(lanes); + } + #[inline(always)] fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { let (a0, a1) = self.split_mask16x32(a); let (b0, b1) = self.split_mask16x32(b); @@ -9344,6 +10519,17 @@ impl Simd for Fallback { lo | (hi << 8usize) } #[inline(always)] + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask32x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x16(lanes); + } + #[inline(always)] fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { let (a0, a1) = self.split_mask32x16(a); let (b0, b1) = self.split_mask32x16(b); @@ -9719,6 +10905,535 @@ impl Simd for Fallback { ) } #[inline(always)] + fn splat_i64x8(self, val: i64) -> i64x8 { + let half = self.splat_i64x4(val); + self.combine_i64x4(half, half) + } + #[inline(always)] + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::support::Aligned512(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::support::Aligned512(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let mut dest = [Default::default(); 8usize]; + dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_i64x8( + self, + a: i64x8, + b: i64x8, + ) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4( + self.slide_within_blocks_i64x4::(a0, b0), + self.slide_within_blocks_i64x4::(a1, b1), + ) + } + #[inline(always)] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + let mut b0 = [0; 4usize]; + let mut b1 = [0; 4usize]; + b0.copy_from_slice(&a.val.0[0..4usize]); + b1.copy_from_slice(&a.val.0[4usize..8usize]); + (b0.simd_into(self), b1.simd_into(self)) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), + ) + } + #[inline(always)] + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::support::Aligned512(val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::support::Aligned512(*val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + a.val.0 + } + #[inline(always)] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + &a.val.0 + } + #[inline(always)] + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + &mut a.val.0 + } + #[inline(always)] + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + *dest = a.val.0; + } + #[inline(always)] + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let mut dest = [Default::default(); 8usize]; + dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]); + dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]); + dest.simd_into(self) + } + #[inline(always)] + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), + ) + } + #[inline(always)] + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.add_u64x4(a0, b0), self.add_u64x4(a1, b1)) + } + #[inline(always)] + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.sub_u64x4(a0, b0), self.sub_u64x4(a1, b1)) + } + #[inline(always)] + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1)) + } + #[inline(always)] + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.and_u64x4(a0, b0), self.and_u64x4(a1, b1)) + } + #[inline(always)] + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.or_u64x4(a0, b0), self.or_u64x4(a1, b1)) + } + #[inline(always)] + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.xor_u64x4(a0, b0), self.xor_u64x4(a1, b1)) + } + #[inline(always)] + fn not_u64x8(self, a: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.not_u64x4(a0), self.not_u64x4(a1)) + } + #[inline(always)] + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shl_u64x4(a0, shift), self.shl_u64x4(a1, shift)) + } + #[inline(always)] + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shlv_u64x4(a0, b0), self.shlv_u64x4(a1, b1)) + } + #[inline(always)] + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shr_u64x4(a0, shift), self.shr_u64x4(a1, shift)) + } + #[inline(always)] + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shrv_u64x4(a0, b0), self.shrv_u64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_eq_u64x4(a0, b0), self.simd_eq_u64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_lt_u64x4(a0, b0), self.simd_lt_u64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_le_u64x4(a0, b0), self.simd_le_u64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_ge_u64x4(a0, b0), self.simd_ge_u64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_gt_u64x4(a0, b0), self.simd_gt_u64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, _) = self.split_u64x8(a); + let (b0, _) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a0, b0), self.zip_high_u64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (_, a1) = self.split_u64x8(a); + let (_, b1) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a1, b1), self.zip_high_u64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_low_u64x4(a0, a1), self.unzip_low_u64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_high_u64x4(a0, a1), self.unzip_high_u64x4(b0, b1)) + } + #[inline(always)] + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_lo = self.zip_low_u64x4(a0, b0); + let lo_hi = self.zip_high_u64x4(a0, b0); + let hi_lo = self.zip_low_u64x4(a1, b1); + let hi_hi = self.zip_high_u64x4(a1, b1); + ( + self.combine_u64x4(lo_lo, lo_hi), + self.combine_u64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_even = self.unzip_low_u64x4(a0, a1); + let lo_odd = self.unzip_high_u64x4(a0, a1); + let hi_even = self.unzip_low_u64x4(b0, b1); + let hi_odd = self.unzip_high_u64x4(b0, b1); + ( + self.combine_u64x4(lo_even, hi_even), + self.combine_u64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_u64x8(b); + let (c0, c1) = self.split_u64x8(c); + self.combine_u64x4(self.select_u64x4(a0, b0, c0), self.select_u64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.min_u64x4(a0, b0), self.min_u64x4(a1, b1)) + } + #[inline(always)] + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.max_u64x4(a0, b0), self.max_u64x4(a1, b1)) + } + #[inline(always)] + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { + let mut b0 = [0; 4usize]; + let mut b1 = [0; 4usize]; + b0.copy_from_slice(&a.val.0[0..4usize]); + b1.copy_from_slice(&a.val.0[4usize..8usize]); + (b0.simd_into(self), b1.simd_into(self)) + } + #[inline(always)] + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + [ + src[0usize], + src[4usize], + src[1usize], + src[5usize], + src[2usize], + src[6usize], + src[3usize], + src[7usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + *dest = [ + a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], + ]; + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u8x32(self.reinterpret_u8_u64x4(a0), self.reinterpret_u8_u64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u32x8( + self.reinterpret_u32_u64x4(a0), + self.reinterpret_u32_u64x4(a1), + ) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) @@ -9748,6 +11463,17 @@ impl Simd for Fallback { lo | (hi << 4usize) } #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask64x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x8(lanes); + } + #[inline(always)] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { let (a0, a1) = self.split_mask64x8(a); let (b0, b1) = self.split_mask64x8(b); diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index fa092c741..656848614 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -6,9 +6,9 @@ use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; use core::arch::aarch64::*; #[doc = "A token for Neon intrinsics on aarch64, representing the \"neon\" level."] @@ -35,6 +35,8 @@ impl ArchTypes for Neon { type u32x4 = crate::support::Aligned128; type mask32x4 = crate::support::Aligned128; type f64x2 = crate::support::Aligned128; + type i64x2 = crate::support::Aligned128; + type u64x2 = crate::support::Aligned128; type mask64x2 = crate::support::Aligned128; type f32x8 = crate::support::Aligned256; type i8x32 = crate::support::Aligned256; @@ -47,6 +49,8 @@ impl ArchTypes for Neon { type u32x8 = crate::support::Aligned256; type mask32x8 = crate::support::Aligned256; type f64x4 = crate::support::Aligned256; + type i64x4 = crate::support::Aligned256; + type u64x4 = crate::support::Aligned256; type mask64x4 = crate::support::Aligned256; type f32x16 = crate::support::Aligned512; type i8x64 = crate::support::Aligned512; @@ -59,6 +63,8 @@ impl ArchTypes for Neon { type u32x16 = crate::support::Aligned512; type mask32x16 = crate::support::Aligned512; type f64x8 = crate::support::Aligned512; + type i64x8 = crate::support::Aligned512; + type u64x8 = crate::support::Aligned512; type mask64x8 = crate::support::Aligned512; } impl Simd for Neon { @@ -70,6 +76,8 @@ impl Simd for Neon { type i16s = i16x8; type u32s = u32x4; type i32s = i32x4; + type u64s = u64x2; + type i64s = i64x2; type mask8s = mask8x16; type mask16s = mask16x8; type mask32s = mask32x4; @@ -1347,6 +1355,17 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask8x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x16(lanes); + } + #[inline(always)] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { crate::kernel!( #[inline(always)] @@ -2237,6 +2256,17 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask16x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x8(lanes); + } + #[inline(always)] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3136,6 +3166,17 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask32x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x4(lanes); + } + #[inline(always)] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { crate::kernel!( #[inline(always)] @@ -3672,625 +3713,2841 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] - fn splat_mask64x2(self, val: bool) -> mask64x2 { + fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, val: bool) -> mask64x2 { - let val: i64 = if val { !0 } else { 0 }; + fn kernel(token: Neon, val: i64) -> i64x2 { vdupq_n_s64(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { - mask64x2 { + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_vext_128( + self, + self.cvt_to_bytes_i64x2(a).val.0, + self.cvt_to_bytes_i64x2(b).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, bits: u64) -> mask64x2 { - let shifts = - crate::transmute::checked_transmute_copy::<[i64; 2], int64x2_t>(&[63, 62]); - let shifted = vshlq_u64(vdupq_n_u64(bits), shifts); - let mask = vcltq_s64(vreinterpretq_s64_u64(shifted), vdupq_n_s64(0)); - vreinterpretq_s64_u64(mask).simd_into(token) + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + vaddq_s64(a.into(), b.into()).simd_into(token) } ); - kernel(self, bits) + kernel(self, a, b) } #[inline(always)] - fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> u64 { - let weights = - crate::transmute::checked_transmute_copy::<[u64; 2], uint64x2_t>(&[1, 2]); - let bits = vandq_u64(vreinterpretq_u64_s64(a.into()), weights); - vaddvq_u64(bits) + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + vsubq_s64(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { vandq_s64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { vorrq_s64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { veorq_s64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + fn not_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> mask64x2 { + fn kernel(token: Neon, a: i64x2) -> i64x2 { vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a.into()))).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn select_mask64x2( - self, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, shift: u32) -> i64x2 { + vshlq_s64(a.into(), vdupq_n_s64(shift as i64)).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + vshlq_s64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, shift: u32) -> i64x2 { + vshlq_s64(a.into(), vdupq_n_s64(-(shift as i64))).simd_into(token) + } + ); + kernel(self, a, shift) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + vshlq_s64(a.into(), vnegq_s64(b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> mask64x2 { + vreinterpretq_s64_u64(vceqq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcltq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcleq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcgeq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcgtq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let x = a.into(); + let y = b.into(); + vzip1q_s64(x, y).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let x = a.into(); + let y = b.into(); + vzip2q_s64(x, y).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let x = a.into(); + let y = b.into(); + vuzp1q_s64(x, y).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let x = a.into(); + let y = b.into(); + vuzp2q_s64(x, y).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) + } + #[inline(always)] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b)) + } + #[inline(always)] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] fn kernel( token: Neon, a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { + b: i64x2, + c: i64x2, + ) -> i64x2 { vbslq_s64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token) } ); kernel(self, a, b, c) } #[inline(always)] - fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { - vreinterpretq_s64_u64(vceqq_s64(a.into(), b.into())).simd_into(token) + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask64x2(self, a: mask64x2) -> bool { + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> bool { - vmaxvq_u32(vreinterpretq_u32_s64(a.into())) != 0 + fn kernel(token: Neon, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_true_mask64x2(self, a: mask64x2) -> bool { + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { + i64x4 { + val: crate::support::Aligned256(int64x2x2_t(a.val.0, b.val.0)), + simd: self, + } + } + #[inline(always)] + fn neg_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> bool { - vminvq_u32(vreinterpretq_u32_s64(a.into())) == 0xffffffff + fn kernel(token: Neon, a: i64x2) -> i64x2 { + vnegq_s64(a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn any_false_mask64x2(self, a: mask64x2) -> bool { + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> bool { - vminvq_u32(vreinterpretq_u32_s64(a.into())) != 0xffffffff + fn kernel(token: Neon, a: i64x2) -> u8x16 { + vreinterpretq_u8_s64(a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn all_false_mask64x2(self, a: mask64x2) -> bool { + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Neon, a: mask64x2) -> bool { - vmaxvq_u32(vreinterpretq_u32_s64(a.into())) == 0 + fn kernel(token: Neon, a: i64x2) -> u32x4 { + vreinterpretq_u32_s64(a.into()).simd_into(token) } ); kernel(self, a) } #[inline(always)] - fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { - mask64x4 { - val: crate::support::Aligned256(int64x2x2_t(a.val.0, b.val.0)), - simd: self, - } - } - #[inline(always)] - fn splat_f32x8(self, val: f32) -> f32x8 { - let half = self.splat_f32x4(val); - self.combine_f32x4(half, half) + fn splat_u64x2(self, val: u64) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, val: u64) -> u64x2 { + vdupq_n_u64(val).simd_into(token) + } + ); + kernel(self, val) } #[inline(always)] - fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { - f32x8 { + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { - f32x8 { + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { - f32x8 { + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { - u8x32 { + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - if SHIFT >= 8usize { + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + if SHIFT >= 2usize { return b; } - let result = { - let a_bytes = self.cvt_to_bytes_f32x8(a).val.0; - let b_bytes = self.cvt_to_bytes_f32x8(b).val.0; - let a_blocks = [a_bytes.0, a_bytes.1]; - let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT * 4usize; - uint8x16x2_t( - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 0, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 1, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - ) - }; - self.cvt_from_bytes_f32x8(u8x32 { - val: crate::support::Aligned256(result), + let result = dyn_vext_128( + self, + self.cvt_to_bytes_u64x2(a).val.0, + self.cvt_to_bytes_u64x2(b).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x8( + fn slide_within_blocks_u64x2( self, - a: f32x8, - b: f32x8, - ) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.slide_within_blocks_f32x4::(a0, b0), - self.slide_within_blocks_f32x4::(a1, b1), - ) + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) } #[inline(always)] - fn abs_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vaddq_u64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn neg_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vsubq_u64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn sqrt_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.approximate_recip_f32x4(a0), - self.approximate_recip_f32x4(a1), - ) + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vandq_u64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vorrq_u64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + veorq_u64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) + fn not_u64x2(self, a: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2) -> u64x2 { + vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(a.into()))).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, shift: u32) -> u64x2 { + vshlq_u64(a.into(), vdupq_n_s64(shift as i64)).simd_into(token) + } + ); + kernel(self, a, shift) } #[inline(always)] - fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vshlq_u64(a.into(), vreinterpretq_s64_u64(b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, shift: u32) -> u64x2 { + vshlq_u64(a.into(), vdupq_n_s64(-(shift as i64))).simd_into(token) + } + ); + kernel(self, a, shift) } #[inline(always)] - fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + vshlq_u64(a.into(), vnegq_s64(vreinterpretq_s64_u64(b.into()))).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> mask64x2 { + vreinterpretq_s64_u64(vceqq_u64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcltq_u64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcleq_u64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, _) = self.split_f32x8(a); - let (b0, _) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcgeq_u64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (_, a1) = self.split_f32x8(a); - let (_, b1) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> mask64x2 { + vreinterpretq_s64_u64(vcgtq_u64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let x = a.into(); + let y = b.into(); + vzip1q_u64(x, y).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let x = a.into(); + let y = b.into(); + vzip2q_u64(x, y).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_lo = self.zip_low_f32x4(a0, b0); - let lo_hi = self.zip_high_f32x4(a0, b0); - let hi_lo = self.zip_low_f32x4(a1, b1); - let hi_hi = self.zip_high_f32x4(a1, b1); - ( - self.combine_f32x4(lo_lo, lo_hi), - self.combine_f32x4(hi_lo, hi_hi), - ) + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let x = a.into(); + let y = b.into(); + vuzp1q_u64(x, y).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_even = self.unzip_low_f32x4(a0, a1); - let lo_odd = self.unzip_high_f32x4(a0, a1); - let hi_even = self.unzip_low_f32x4(b0, b1); - let hi_odd = self.unzip_high_f32x4(b0, b1); - ( - self.combine_f32x4(lo_even, hi_even), - self.combine_f32x4(lo_odd, hi_odd), - ) + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let x = a.into(); + let y = b.into(); + vuzp2q_u64(x, y).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) } #[inline(always)] - fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b)) } #[inline(always)] - fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.max_precise_f32x4(a0, b0), + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Neon, + a: mask64x2, + b: u64x2, + c: u64x2, + ) -> u64x2 { + vbslq_u64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { + u64x4 { + val: crate::support::Aligned256(uint64x2x2_t(a.val.0, b.val.0)), + simd: self, + } + } + #[inline(always)] + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2) -> u8x16 { + vreinterpretq_u8_u64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2) -> u32x4 { + vreinterpretq_u32_u64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_mask64x2(self, val: bool) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, val: bool) -> mask64x2 { + let val: i64 = if val { !0 } else { 0 }; + vdupq_n_s64(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { + mask64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, bits: u64) -> mask64x2 { + let shifts = + crate::transmute::checked_transmute_copy::<[i64; 2], int64x2_t>(&[63, 62]); + let shifted = vshlq_u64(vdupq_n_u64(bits), shifts); + let mask = vcltq_s64(vreinterpretq_s64_u64(shifted), vdupq_n_s64(0)); + vreinterpretq_s64_u64(mask).simd_into(token) + } + ); + kernel(self, bits) + } + #[inline(always)] + fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> u64 { + let weights = + crate::transmute::checked_transmute_copy::<[u64; 2], uint64x2_t>(&[1, 2]); + let bits = vandq_u64(vreinterpretq_u64_s64(a.into()), weights); + vaddvq_u64(bits) + } + ); + kernel(self, a) + } + #[inline(always)] + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let mut lanes = self.as_array_mask64x2(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x2(lanes); + } + #[inline(always)] + fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + vandq_s64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + vorrq_s64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + veorq_s64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> mask64x2 { + vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a.into()))).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn select_mask64x2( + self, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Neon, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + vbslq_s64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2, b: mask64x2) -> mask64x2 { + vreinterpretq_s64_u64(vceqq_s64(a.into(), b.into())).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn any_true_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> bool { + vmaxvq_u32(vreinterpretq_u32_s64(a.into())) != 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_true_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> bool { + vminvq_u32(vreinterpretq_u32_s64(a.into())) == 0xffffffff + } + ); + kernel(self, a) + } + #[inline(always)] + fn any_false_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> bool { + vminvq_u32(vreinterpretq_u32_s64(a.into())) != 0xffffffff + } + ); + kernel(self, a) + } + #[inline(always)] + fn all_false_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: mask64x2) -> bool { + vmaxvq_u32(vreinterpretq_u32_s64(a.into())) == 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { + mask64x4 { + val: crate::support::Aligned256(int64x2x2_t(a.val.0, b.val.0)), + simd: self, + } + } + #[inline(always)] + fn splat_f32x8(self, val: f32) -> f32x8 { + let half = self.splat_f32x4(val); + self.combine_f32x4(half, half) + } + #[inline(always)] + fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { + f32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + if SHIFT >= 8usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_f32x8(a).val.0; + let b_bytes = self.cvt_to_bytes_f32x8(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT * 4usize; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_f32x8( + self, + a: f32x8, + b: f32x8, + ) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.slide_within_blocks_f32x4::(a0, b0), + self.slide_within_blocks_f32x4::(a1, b1), + ) + } + #[inline(always)] + fn abs_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) + } + #[inline(always)] + fn neg_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) + } + #[inline(always)] + fn sqrt_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) + } + #[inline(always)] + fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.approximate_recip_f32x4(a0), + self.approximate_recip_f32x4(a1), + ) + } + #[inline(always)] + fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) + } + #[inline(always)] + fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) + } + #[inline(always)] + fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) + } + #[inline(always)] + fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) + } + #[inline(always)] + fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) + } + #[inline(always)] + fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) + } + #[inline(always)] + fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, _) = self.split_f32x8(a); + let (b0, _) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) + } + #[inline(always)] + fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (_, a1) = self.split_f32x8(a); + let (_, b1) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) + } + #[inline(always)] + fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_lo = self.zip_low_f32x4(a0, b0); + let lo_hi = self.zip_high_f32x4(a0, b0); + let hi_lo = self.zip_low_f32x4(a1, b1); + let hi_hi = self.zip_high_f32x4(a1, b1); + ( + self.combine_f32x4(lo_lo, lo_hi), + self.combine_f32x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_even = self.unzip_low_f32x4(a0, a1); + let lo_odd = self.unzip_high_f32x4(a0, a1); + let hi_even = self.unzip_low_f32x4(b0, b1); + let hi_odd = self.unzip_high_f32x4(b0, b1); + ( + self.combine_f32x4(lo_even, hi_even), + self.combine_f32x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) + } + #[inline(always)] + fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) + } + #[inline(always)] + fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.max_precise_f32x4(a0, b0), self.max_precise_f32x4(a1, b1), ) } #[inline(always)] - fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.min_precise_f32x4(a0, b0), - self.min_precise_f32x4(a1, b1), + fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.min_precise_f32x4(a0, b0), + self.min_precise_f32x4(a1, b1), + ) + } + #[inline(always)] + fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_add_f32x4(a0, b0, c0), + self.mul_add_f32x4(a1, b1, c1), + ) + } + #[inline(always)] + fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_sub_f32x4(a0, b0, c0), + self.mul_sub_f32x4(a1, b1, c1), + ) + } + #[inline(always)] + fn floor_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) + } + #[inline(always)] + fn ceil_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + } + #[inline(always)] + fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.round_ties_even_f32x4(a0), + self.round_ties_even_f32x4(a1), + ) + } + #[inline(always)] + fn fract_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + } + #[inline(always)] + fn trunc_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + } + #[inline(always)] + fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + } + #[inline(always)] + fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { + f32x16 { + val: crate::support::Aligned512(float32x4x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + ( + f32x4 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + f32x4 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) + } + #[inline(always)] + fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f64x2( + self.reinterpret_f64_f32x4(a0), + self.reinterpret_f64_f32x4(a1), + ) + } + #[inline(always)] + fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.reinterpret_i32_f32x4(a0), + self.reinterpret_i32_f32x4(a1), + ) + } + #[inline(always)] + fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4( + self.reinterpret_u32_f32x4(a0), + self.reinterpret_u32_f32x4(a1), + ) + } + #[inline(always)] + fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) + } + #[inline(always)] + fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4( + self.cvt_u32_precise_f32x4(a0), + self.cvt_u32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + } + #[inline(always)] + fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.cvt_i32_precise_f32x4(a0), + self.cvt_i32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn splat_i8x32(self, val: i8) -> i8x32 { + let half = self.splat_i8x16(val); + self.combine_i8x16(half, half) + } + #[inline(always)] + fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { + i8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + if SHIFT >= 32usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_i8x32(a).val.0; + let b_bytes = self.cvt_to_bytes_i8x32(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i8x32( + self, + a: i8x32, + b: i8x32, + ) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16( + self.slide_within_blocks_i8x16::(a0, b0), + self.slide_within_blocks_i8x16::(a1, b1), + ) + } + #[inline(always)] + fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) + } + #[inline(always)] + fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) + } + #[inline(always)] + fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) + } + #[inline(always)] + fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) + } + #[inline(always)] + fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) + } + #[inline(always)] + fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) + } + #[inline(always)] + fn not_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) + } + #[inline(always)] + fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) + } + #[inline(always)] + fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) + } + #[inline(always)] + fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) + } + #[inline(always)] + fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) + } + #[inline(always)] + fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) + } + #[inline(always)] + fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, _) = self.split_i8x32(a); + let (b0, _) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) + } + #[inline(always)] + fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (_, a1) = self.split_i8x32(a); + let (_, b1) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) + } + #[inline(always)] + fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_lo = self.zip_low_i8x16(a0, b0); + let lo_hi = self.zip_high_i8x16(a0, b0); + let hi_lo = self.zip_low_i8x16(a1, b1); + let hi_hi = self.zip_high_i8x16(a1, b1); + ( + self.combine_i8x16(lo_lo, lo_hi), + self.combine_i8x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_even = self.unzip_low_i8x16(a0, a1); + let lo_odd = self.unzip_high_i8x16(a0, a1); + let hi_even = self.unzip_low_i8x16(b0, b1); + let hi_odd = self.unzip_high_i8x16(b0, b1); + ( + self.combine_i8x16(lo_even, hi_even), + self.combine_i8x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_i8x32(b); + let (c0, c1) = self.split_i8x32(c); + self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) + } + #[inline(always)] + fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) + } + #[inline(always)] + fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) + } + #[inline(always)] + fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { + i8x64 { + val: crate::support::Aligned512(int8x16x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { + ( + i8x16 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + i8x16 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + } + #[inline(always)] + fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { + let (a0, a1) = self.split_i8x32(a); + self.combine_u32x4( + self.reinterpret_u32_i8x16(a0), + self.reinterpret_u32_i8x16(a1), + ) + } + #[inline(always)] + fn splat_u8x32(self, val: u8) -> u8x32 { + let half = self.splat_u8x16(val); + self.combine_u8x16(half, half) + } + #[inline(always)] + fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + if SHIFT >= 32usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_u8x32(a).val.0; + let b_bytes = self.cvt_to_bytes_u8x32(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u8x32( + self, + a: u8x32, + b: u8x32, + ) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16( + self.slide_within_blocks_u8x16::(a0, b0), + self.slide_within_blocks_u8x16::(a1, b1), + ) + } + #[inline(always)] + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) + } + #[inline(always)] + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) + } + #[inline(always)] + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) + } + #[inline(always)] + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) + } + #[inline(always)] + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) + } + #[inline(always)] + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) + } + #[inline(always)] + fn not_u8x32(self, a: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) + } + #[inline(always)] + fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) + } + #[inline(always)] + fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) + } + #[inline(always)] + fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) + } + #[inline(always)] + fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) + } + #[inline(always)] + fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) + } + #[inline(always)] + fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, _) = self.split_u8x32(a); + let (b0, _) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) + } + #[inline(always)] + fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (_, a1) = self.split_u8x32(a); + let (_, b1) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) + } + #[inline(always)] + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_lo = self.zip_low_u8x16(a0, b0); + let lo_hi = self.zip_high_u8x16(a0, b0); + let hi_lo = self.zip_low_u8x16(a1, b1); + let hi_hi = self.zip_high_u8x16(a1, b1); + ( + self.combine_u8x16(lo_lo, lo_hi), + self.combine_u8x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_even = self.unzip_low_u8x16(a0, a1); + let lo_odd = self.unzip_high_u8x16(a0, a1); + let hi_even = self.unzip_low_u8x16(b0, b1); + let hi_odd = self.unzip_high_u8x16(b0, b1); + ( + self.combine_u8x16(lo_even, hi_even), + self.combine_u8x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_u8x32(b); + let (c0, c1) = self.split_u8x32(c); + self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) + } + #[inline(always)] + fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) + } + #[inline(always)] + fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) + } + #[inline(always)] + fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { + u8x64 { + val: crate::support::Aligned512(uint8x16x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + ( + u8x16 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + u8x16 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) + } + #[inline(always)] + fn widen_u8x32(self, a: u8x32) -> u16x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) + } + #[inline(always)] + fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u32x4( + self.reinterpret_u32_u8x16(a0), + self.reinterpret_u32_u8x16(a1), + ) + } + #[inline(always)] + fn splat_mask8x32(self, val: bool) -> mask8x32 { + let half = self.splat_mask8x16(val); + self.combine_mask8x16(half, half) + } + #[inline(always)] + fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { + mask8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { + let lo = self.from_bitmask_mask8x16(bits); + let hi = self.from_bitmask_mask8x16(bits >> 16usize); + self.combine_mask8x16(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { + let (lo, hi) = self.split_mask8x32(a); + let lo = self.to_bitmask_mask8x16(lo); + let hi = self.to_bitmask_mask8x16(hi); + lo | (hi << 16usize) + } + #[inline(always)] + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask8x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x32(lanes); + } + #[inline(always)] + fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) + } + #[inline(always)] + fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) + } + #[inline(always)] + fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) + } + #[inline(always)] + fn not_mask8x32(self, a: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) + } + #[inline(always)] + fn select_mask8x32( + self, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + let (c0, c1) = self.split_mask8x32(c); + self.combine_mask8x16( + self.select_mask8x16(a0, b0, c0), + self.select_mask8x16(a1, b1, c1), + ) + } + #[inline(always)] + fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) + } + #[inline(always)] + fn any_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) + } + #[inline(always)] + fn all_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) + } + #[inline(always)] + fn any_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + } + #[inline(always)] + fn all_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + } + #[inline(always)] + fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { + mask8x64 { + val: crate::support::Aligned512(int8x16x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { + ( + mask8x16 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + mask8x16 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i16x16(self, val: i16) -> i16x16 { + let half = self.splat_i16x8(val); + self.combine_i16x8(half, half) + } + #[inline(always)] + fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { + i16x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + if SHIFT >= 16usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_i16x16(a).val.0; + let b_bytes = self.cvt_to_bytes_i16x16(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT * 2usize; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i16x16( + self, + a: i16x16, + b: i16x16, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8( + self.slide_within_blocks_i16x8::(a0, b0), + self.slide_within_blocks_i16x8::(a1, b1), + ) + } + #[inline(always)] + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) + } + #[inline(always)] + fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) + } + #[inline(always)] + fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) + } + #[inline(always)] + fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) + } + #[inline(always)] + fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) + } + #[inline(always)] + fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) + } + #[inline(always)] + fn not_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) + } + #[inline(always)] + fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) + } + #[inline(always)] + fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) + } + #[inline(always)] + fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) + } + #[inline(always)] + fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) + } + #[inline(always)] + fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) + } + #[inline(always)] + fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, _) = self.split_i16x16(a); + let (b0, _) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) + } + #[inline(always)] + fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (_, a1) = self.split_i16x16(a); + let (_, b1) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) + } + #[inline(always)] + fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) + } + #[inline(always)] + fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_lo = self.zip_low_i16x8(a0, b0); + let lo_hi = self.zip_high_i16x8(a0, b0); + let hi_lo = self.zip_low_i16x8(a1, b1); + let hi_hi = self.zip_high_i16x8(a1, b1); + ( + self.combine_i16x8(lo_lo, lo_hi), + self.combine_i16x8(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_even = self.unzip_low_i16x8(a0, a1); + let lo_odd = self.unzip_high_i16x8(a0, a1); + let hi_even = self.unzip_low_i16x8(b0, b1); + let hi_odd = self.unzip_high_i16x8(b0, b1); + ( + self.combine_i16x8(lo_even, hi_even), + self.combine_i16x8(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_i16x16(b); + let (c0, c1) = self.split_i16x16(c); + self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) + } + #[inline(always)] + fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) + } + #[inline(always)] + fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) + } + #[inline(always)] + fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { + i16x32 { + val: crate::support::Aligned512(int16x8x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { + ( + i16x8 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + i16x8 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, ) } #[inline(always)] - fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_add_f32x4(a0, b0, c0), - self.mul_add_f32x4(a1, b1, c1), + fn neg_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) + } + #[inline(always)] + fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { + let (a0, a1) = self.split_i16x16(a); + self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) + } + #[inline(always)] + fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { + let (a0, a1) = self.split_i16x16(a); + self.combine_u32x4( + self.reinterpret_u32_i16x8(a0), + self.reinterpret_u32_i16x8(a1), ) } #[inline(always)] - fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_sub_f32x4(a0, b0, c0), - self.mul_sub_f32x4(a1, b1, c1), + fn splat_u16x16(self, val: u16) -> u16x16 { + let half = self.splat_u16x8(val); + self.combine_u16x8(half, half) + } + #[inline(always)] + fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { + crate::transmute::checked_cast_ref::(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + if SHIFT >= 16usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_u16x16(a).val.0; + let b_bytes = self.cvt_to_bytes_u16x16(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT * 2usize; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u16x16( + self, + a: u16x16, + b: u16x16, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8( + self.slide_within_blocks_u16x8::(a0, b0), + self.slide_within_blocks_u16x8::(a1, b1), ) } #[inline(always)] - fn floor_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) + fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) + } + #[inline(always)] + fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) + } + #[inline(always)] + fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) + } + #[inline(always)] + fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) + } + #[inline(always)] + fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) + } + #[inline(always)] + fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) + } + #[inline(always)] + fn not_u16x16(self, a: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) + } + #[inline(always)] + fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) + } + #[inline(always)] + fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) + } + #[inline(always)] + fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) + } + #[inline(always)] + fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) + } + #[inline(always)] + fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) + } + #[inline(always)] + fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, _) = self.split_u16x16(a); + let (b0, _) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) + } + #[inline(always)] + fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (_, a1) = self.split_u16x16(a); + let (_, b1) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) } #[inline(always)] - fn ceil_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) } #[inline(always)] - fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.round_ties_even_f32x4(a0), - self.round_ties_even_f32x4(a1), + fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_lo = self.zip_low_u16x8(a0, b0); + let lo_hi = self.zip_high_u16x8(a0, b0); + let hi_lo = self.zip_low_u16x8(a1, b1); + let hi_hi = self.zip_high_u16x8(a1, b1); + ( + self.combine_u16x8(lo_lo, lo_hi), + self.combine_u16x8(hi_lo, hi_hi), ) } #[inline(always)] - fn fract_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_even = self.unzip_low_u16x8(a0, a1); + let lo_odd = self.unzip_high_u16x8(a0, a1); + let hi_even = self.unzip_low_u16x8(b0, b1); + let hi_odd = self.unzip_high_u16x8(b0, b1); + ( + self.combine_u16x8(lo_even, hi_even), + self.combine_u16x8(lo_odd, hi_odd), + ) } #[inline(always)] - fn trunc_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_u16x16(b); + let (c0, c1) = self.split_u16x16(c); + self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) } #[inline(always)] - fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) } #[inline(always)] - fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { - f32x16 { - val: crate::support::Aligned512(float32x4x4_t( + fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) + } + #[inline(always)] + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { + val: crate::support::Aligned512(uint16x8x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { ( - f32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - f32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f64x2( - self.reinterpret_f64_f32x4(a0), - self.reinterpret_f64_f32x4(a1), - ) + fn narrow_u16x16(self, a: u16x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x16) -> u8x16 { + let converted: uint16x8x2_t = a.into(); + let low = vmovn_u16(converted.0); + let high = vmovn_u16(converted.1); + vcombine_u8(low, high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.reinterpret_i32_f32x4(a0), - self.reinterpret_i32_f32x4(a1), + fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) + } + #[inline(always)] + fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u32x4( + self.reinterpret_u32_u16x8(a0), + self.reinterpret_u32_u16x8(a1), ) } #[inline(always)] - fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + fn splat_mask16x16(self, val: bool) -> mask16x16 { + let half = self.splat_mask16x8(val); + self.combine_mask16x8(half, half) } #[inline(always)] - fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.reinterpret_u32_f32x4(a0), - self.reinterpret_u32_f32x4(a1), - ) + fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { + mask16x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) + fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.cvt_u32_precise_f32x4(a0), - self.cvt_u32_precise_f32x4(a1), + fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { + let lo = self.from_bitmask_mask16x8(bits); + let hi = self.from_bitmask_mask16x8(bits >> 8usize); + self.combine_mask16x8(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { + let (lo, hi) = self.split_mask16x16(a); + let lo = self.to_bitmask_mask16x8(lo); + let hi = self.to_bitmask_mask16x8(hi); + lo | (hi << 8usize) + } + #[inline(always)] + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask16x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x16(lanes); + } + #[inline(always)] + fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) + } + #[inline(always)] + fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) + } + #[inline(always)] + fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) + } + #[inline(always)] + fn not_mask16x16(self, a: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) + } + #[inline(always)] + fn select_mask16x16( + self, + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + let (c0, c1) = self.split_mask16x16(c); + self.combine_mask16x8( + self.select_mask16x8(a0, b0, c0), + self.select_mask16x8(a1, b1, c1), ) } #[inline(always)] - fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) } #[inline(always)] - fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.cvt_i32_precise_f32x4(a0), - self.cvt_i32_precise_f32x4(a1), + fn any_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) + } + #[inline(always)] + fn all_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) + } + #[inline(always)] + fn any_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) + } + #[inline(always)] + fn all_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) + } + #[inline(always)] + fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { + mask16x32 { + val: crate::support::Aligned512(int16x8x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + ( + mask16x8 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + mask16x8 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, ) } #[inline(always)] - fn splat_i8x32(self, val: i8) -> i8x32 { - let half = self.splat_i8x16(val); - self.combine_i8x16(half, half) + fn splat_i32x8(self, val: i32) -> i32x8 { + let half = self.splat_i32x4(val); + self.combine_i32x4(half, half) } #[inline(always)] - fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { - i8x32 { + fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { + i32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - if SHIFT >= 32usize { + fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + if SHIFT >= 8usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i8x32(a).val.0; - let b_bytes = self.cvt_to_bytes_i8x32(b).val.0; + let a_bytes = self.cvt_to_bytes_i32x8(a).val.0; + let b_bytes = self.cvt_to_bytes_i32x8(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1]; let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT; + let shift_bytes = SHIFT * 4usize; uint8x16x2_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -4312,286 +6569,291 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i8x32(u8x32 { + self.cvt_from_bytes_i32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x32( + fn slide_within_blocks_i32x8( self, - a: i8x32, - b: i8x32, - ) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16( - self.slide_within_blocks_i8x16::(a0, b0), - self.slide_within_blocks_i8x16::(a1, b1), + a: i32x8, + b: i32x8, + ) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4( + self.slide_within_blocks_i32x4::(a0, b0), + self.slide_within_blocks_i32x4::(a1, b1), ) } #[inline(always)] - fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) } #[inline(always)] - fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) } #[inline(always)] - fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) } #[inline(always)] - fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) } #[inline(always)] - fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) } #[inline(always)] - fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) + fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) } #[inline(always)] - fn not_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) + fn not_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) } #[inline(always)] - fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) + fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) } #[inline(always)] - fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) + fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) } #[inline(always)] - fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) + fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) } #[inline(always)] - fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) + fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) } #[inline(always)] - fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) + fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) } #[inline(always)] - fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) + fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) } #[inline(always)] - fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) + fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) } #[inline(always)] - fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) + fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) } #[inline(always)] - fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) + fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) } #[inline(always)] - fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, _) = self.split_i8x32(a); - let (b0, _) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) + fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, _) = self.split_i32x8(a); + let (b0, _) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) } #[inline(always)] - fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (_, a1) = self.split_i8x32(a); - let (_, b1) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) + fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (_, a1) = self.split_i32x8(a); + let (_, b1) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) } #[inline(always)] - fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) + fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) } #[inline(always)] - fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) + fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) } #[inline(always)] - fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_lo = self.zip_low_i8x16(a0, b0); - let lo_hi = self.zip_high_i8x16(a0, b0); - let hi_lo = self.zip_low_i8x16(a1, b1); - let hi_hi = self.zip_high_i8x16(a1, b1); + fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_lo = self.zip_low_i32x4(a0, b0); + let lo_hi = self.zip_high_i32x4(a0, b0); + let hi_lo = self.zip_low_i32x4(a1, b1); + let hi_hi = self.zip_high_i32x4(a1, b1); ( - self.combine_i8x16(lo_lo, lo_hi), - self.combine_i8x16(hi_lo, hi_hi), + self.combine_i32x4(lo_lo, lo_hi), + self.combine_i32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_even = self.unzip_low_i8x16(a0, a1); - let lo_odd = self.unzip_high_i8x16(a0, a1); - let hi_even = self.unzip_low_i8x16(b0, b1); - let hi_odd = self.unzip_high_i8x16(b0, b1); + fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_even = self.unzip_low_i32x4(a0, a1); + let lo_odd = self.unzip_high_i32x4(a0, a1); + let hi_even = self.unzip_low_i32x4(b0, b1); + let hi_odd = self.unzip_high_i32x4(b0, b1); ( - self.combine_i8x16(lo_even, hi_even), - self.combine_i8x16(lo_odd, hi_odd), + self.combine_i32x4(lo_even, hi_even), + self.combine_i32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_i8x32(b); - let (c0, c1) = self.split_i8x32(c); - self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) + fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_i32x8(b); + let (c0, c1) = self.split_i32x8(c); + self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) } #[inline(always)] - fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) + fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) } #[inline(always)] - fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) + fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) } #[inline(always)] - fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { - i8x64 { - val: crate::support::Aligned512(int8x16x4_t( + fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { + i32x16 { + val: crate::support::Aligned512(int32x4x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { + fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { ( - i8x16 { + i32x4 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - i8x16 { + i32x4 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn neg_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) + fn neg_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) } #[inline(always)] - fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { + let (a0, a1) = self.split_i32x8(a); + self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) } #[inline(always)] - fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { - let (a0, a1) = self.split_i8x32(a); + fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { + let (a0, a1) = self.split_i32x8(a); self.combine_u32x4( - self.reinterpret_u32_i8x16(a0), - self.reinterpret_u32_i8x16(a1), + self.reinterpret_u32_i32x4(a0), + self.reinterpret_u32_i32x4(a1), ) } #[inline(always)] - fn splat_u8x32(self, val: u8) -> u8x32 { - let half = self.splat_u8x16(val); - self.combine_u8x16(half, half) + fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) } #[inline(always)] - fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { - u8x32 { + fn splat_u32x8(self, val: u32) -> u32x8 { + let half = self.splat_u32x4(val); + self.combine_u32x4(half, half) + } + #[inline(always)] + fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { - u8x32 { + fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { - u8x32 { + fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - if SHIFT >= 32usize { + fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + if SHIFT >= 8usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_u8x32(a).val.0; - let b_bytes = self.cvt_to_bytes_u8x32(b).val.0; + let a_bytes = self.cvt_to_bytes_u32x8(a).val.0; + let b_bytes = self.cvt_to_bytes_u32x8(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1]; let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT; + let shift_bytes = SHIFT * 4usize; uint8x16x2_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -4613,396 +6875,404 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_u8x32(u8x32 { + self.cvt_from_bytes_u32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x32( + fn slide_within_blocks_u32x8( self, - a: u8x32, - b: u8x32, - ) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16( - self.slide_within_blocks_u8x16::(a0, b0), - self.slide_within_blocks_u8x16::(a1, b1), + a: u32x8, + b: u32x8, + ) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4( + self.slide_within_blocks_u32x4::(a0, b0), + self.slide_within_blocks_u32x4::(a1, b1), ) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) + fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) } #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) + fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) } #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) + fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) } #[inline(always)] - fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) + fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) } #[inline(always)] - fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) + fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) } #[inline(always)] - fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) + fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) } #[inline(always)] - fn not_u8x32(self, a: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) + fn not_u32x8(self, a: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) } #[inline(always)] - fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) + fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) } #[inline(always)] - fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) + fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) } #[inline(always)] - fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) + fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) } #[inline(always)] - fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) + fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) } #[inline(always)] - fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) + fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) } #[inline(always)] - fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) + fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) } #[inline(always)] - fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) + fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) } #[inline(always)] - fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) + fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) } #[inline(always)] - fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) + fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) } #[inline(always)] - fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, _) = self.split_u8x32(a); - let (b0, _) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) + fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, _) = self.split_u32x8(a); + let (b0, _) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) } #[inline(always)] - fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (_, a1) = self.split_u8x32(a); - let (_, b1) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) + fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (_, a1) = self.split_u32x8(a); + let (_, b1) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) } #[inline(always)] - fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) + fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) } #[inline(always)] - fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) + fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) } #[inline(always)] - fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_lo = self.zip_low_u8x16(a0, b0); - let lo_hi = self.zip_high_u8x16(a0, b0); - let hi_lo = self.zip_low_u8x16(a1, b1); - let hi_hi = self.zip_high_u8x16(a1, b1); + fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_lo = self.zip_low_u32x4(a0, b0); + let lo_hi = self.zip_high_u32x4(a0, b0); + let hi_lo = self.zip_low_u32x4(a1, b1); + let hi_hi = self.zip_high_u32x4(a1, b1); ( - self.combine_u8x16(lo_lo, lo_hi), - self.combine_u8x16(hi_lo, hi_hi), + self.combine_u32x4(lo_lo, lo_hi), + self.combine_u32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_even = self.unzip_low_u8x16(a0, a1); - let lo_odd = self.unzip_high_u8x16(a0, a1); - let hi_even = self.unzip_low_u8x16(b0, b1); - let hi_odd = self.unzip_high_u8x16(b0, b1); + fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_even = self.unzip_low_u32x4(a0, a1); + let lo_odd = self.unzip_high_u32x4(a0, a1); + let hi_even = self.unzip_low_u32x4(b0, b1); + let hi_odd = self.unzip_high_u32x4(b0, b1); ( - self.combine_u8x16(lo_even, hi_even), - self.combine_u8x16(lo_odd, hi_odd), + self.combine_u32x4(lo_even, hi_even), + self.combine_u32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_u8x32(b); - let (c0, c1) = self.split_u8x32(c); - self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) + fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_u32x8(b); + let (c0, c1) = self.split_u32x8(c); + self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) } #[inline(always)] - fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) + fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) } #[inline(always)] - fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) + fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) } #[inline(always)] - fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { - u8x64 { - val: crate::support::Aligned512(uint8x16x4_t( + fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { + u32x16 { + val: crate::support::Aligned512(uint32x4x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { ( - u8x16 { + u32x4 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - u8x16 { + u32x4 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn widen_u8x32(self, a: u8x32) -> u16x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) + fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) } #[inline(always)] - fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u32x4( - self.reinterpret_u32_u8x16(a0), - self.reinterpret_u32_u8x16(a1), - ) + fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) } #[inline(always)] - fn splat_mask8x32(self, val: bool) -> mask8x32 { - let half = self.splat_mask8x16(val); - self.combine_mask8x16(half, half) + fn splat_mask32x8(self, val: bool) -> mask32x8 { + let half = self.splat_mask32x4(val); + self.combine_mask32x4(half, half) } #[inline(always)] - fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { - mask8x32 { + fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { + mask32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { - let lo = self.from_bitmask_mask8x16(bits); - let hi = self.from_bitmask_mask8x16(bits >> 16usize); - self.combine_mask8x16(lo, hi) + fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { + let lo = self.from_bitmask_mask32x4(bits); + let hi = self.from_bitmask_mask32x4(bits >> 4usize); + self.combine_mask32x4(lo, hi) } #[inline(always)] - fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { - let (lo, hi) = self.split_mask8x32(a); - let lo = self.to_bitmask_mask8x16(lo); - let hi = self.to_bitmask_mask8x16(hi); - lo | (hi << 16usize) + fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { + let (lo, hi) = self.split_mask32x8(a); + let lo = self.to_bitmask_mask32x4(lo); + let hi = self.to_bitmask_mask32x4(hi); + lo | (hi << 4usize) } #[inline(always)] - fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask32x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x8(lanes); } #[inline(always)] - fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) + fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) } #[inline(always)] - fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) + fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) } #[inline(always)] - fn not_mask8x32(self, a: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) + fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) } #[inline(always)] - fn select_mask8x32( + fn not_mask32x8(self, a: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) + } + #[inline(always)] + fn select_mask32x8( self, - a: mask8x32, - b: mask8x32, - c: mask8x32, - ) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - let (c0, c1) = self.split_mask8x32(c); - self.combine_mask8x16( - self.select_mask8x16(a0, b0, c0), - self.select_mask8x16(a1, b1, c1), + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + let (c0, c1) = self.split_mask32x8(c); + self.combine_mask32x4( + self.select_mask32x4(a0, b0, c0), + self.select_mask32x4(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) + fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) } #[inline(always)] - fn any_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) + fn any_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) } #[inline(always)] - fn all_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) + fn all_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) } #[inline(always)] - fn any_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + fn any_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) } #[inline(always)] - fn all_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + fn all_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) } #[inline(always)] - fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { - mask8x64 { - val: crate::support::Aligned512(int8x16x4_t( + fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { + mask32x16 { + val: crate::support::Aligned512(int32x4x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { + fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { ( - mask8x16 { + mask32x4 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - mask8x16 { + mask32x4 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn splat_i16x16(self, val: i16) -> i16x16 { - let half = self.splat_i16x8(val); - self.combine_i16x8(half, half) + fn splat_f64x4(self, val: f64) -> f64x4 { + let half = self.splat_f64x2(val); + self.combine_f64x2(half, half) } #[inline(always)] - fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { - i16x16 { + fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { - i16x16 { + fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { + fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { - i16x16 { + fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { + fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - if SHIFT >= 16usize { + fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + if SHIFT >= 4usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i16x16(a).val.0; - let b_bytes = self.cvt_to_bytes_i16x16(b).val.0; + let a_bytes = self.cvt_to_bytes_f64x4(a).val.0; + let b_bytes = self.cvt_to_bytes_f64x4(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1]; let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT * 2usize; + let shift_bytes = SHIFT * 8usize; uint8x16x2_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -5024,710 +7294,633 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i16x16(u8x32 { + self.cvt_from_bytes_f64x4(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i16x16( + fn slide_within_blocks_f64x4( self, - a: i16x16, - b: i16x16, - ) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8( - self.slide_within_blocks_i16x8::(a0, b0), - self.slide_within_blocks_i16x8::(a1, b1), + a: f64x4, + b: f64x4, + ) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.slide_within_blocks_f64x2::(a0, b0), + self.slide_within_blocks_f64x2::(a1, b1), ) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) - } - #[inline(always)] - fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) - } - #[inline(always)] - fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) - } - #[inline(always)] - fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) - } - #[inline(always)] - fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) - } - #[inline(always)] - fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) - } - #[inline(always)] - fn not_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) - } - #[inline(always)] - fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) - } - #[inline(always)] - fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) - } - #[inline(always)] - fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) - } - #[inline(always)] - fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) - } - #[inline(always)] - fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) - } - #[inline(always)] - fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) - } - #[inline(always)] - fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) + fn abs_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) } #[inline(always)] - fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, _) = self.split_i16x16(a); - let (b0, _) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) + fn neg_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) } #[inline(always)] - fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (_, a1) = self.split_i16x16(a); - let (_, b1) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) } #[inline(always)] - fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.approximate_recip_f64x2(a0), + self.approximate_recip_f64x2(a1), + ) } #[inline(always)] - fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) } #[inline(always)] - fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_lo = self.zip_low_i16x8(a0, b0); - let lo_hi = self.zip_high_i16x8(a0, b0); - let hi_lo = self.zip_low_i16x8(a1, b1); - let hi_hi = self.zip_high_i16x8(a1, b1); - ( - self.combine_i16x8(lo_lo, lo_hi), - self.combine_i16x8(hi_lo, hi_hi), - ) + fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) } #[inline(always)] - fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_even = self.unzip_low_i16x8(a0, a1); - let lo_odd = self.unzip_high_i16x8(a0, a1); - let hi_even = self.unzip_low_i16x8(b0, b1); - let hi_odd = self.unzip_high_i16x8(b0, b1); - ( - self.combine_i16x8(lo_even, hi_even), - self.combine_i16x8(lo_odd, hi_odd), - ) + fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) } #[inline(always)] - fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_i16x16(b); - let (c0, c1) = self.split_i16x16(c); - self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) + fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) } #[inline(always)] - fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) + fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) } #[inline(always)] - fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) + fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) } #[inline(always)] - fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { - i16x32 { - val: crate::support::Aligned512(int16x8x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) } #[inline(always)] - fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { - ( - i16x8 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - i16x8 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, - ) + fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) } #[inline(always)] - fn neg_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) + fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) } #[inline(always)] - fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) + fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) } #[inline(always)] - fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u32x4( - self.reinterpret_u32_i16x8(a0), - self.reinterpret_u32_i16x8(a1), - ) + fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, _) = self.split_f64x4(a); + let (b0, _) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) } #[inline(always)] - fn splat_u16x16(self, val: u16) -> u16x16 { - let half = self.splat_u16x8(val); - self.combine_u16x8(half, half) + fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (_, a1) = self.split_f64x4(a); + let (_, b1) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) } #[inline(always)] - fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { - u16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) } #[inline(always)] - fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { - u16x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) } #[inline(always)] - fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_lo = self.zip_low_f64x2(a0, b0); + let lo_hi = self.zip_high_f64x2(a0, b0); + let hi_lo = self.zip_low_f64x2(a1, b1); + let hi_hi = self.zip_high_f64x2(a1, b1); + ( + self.combine_f64x2(lo_lo, lo_hi), + self.combine_f64x2(hi_lo, hi_hi), + ) } #[inline(always)] - fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_even = self.unzip_low_f64x2(a0, a1); + let lo_odd = self.unzip_high_f64x2(a0, a1); + let hi_even = self.unzip_low_f64x2(b0, b1); + let hi_odd = self.unzip_high_f64x2(b0, b1); + ( + self.combine_f64x2(lo_even, hi_even), + self.combine_f64x2(lo_odd, hi_odd), + ) } #[inline(always)] - fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) } #[inline(always)] - fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) } #[inline(always)] - fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { - u16x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.max_precise_f64x2(a0, b0), + self.max_precise_f64x2(a1, b1), + ) } #[inline(always)] - fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.min_precise_f64x2(a0, b0), + self.min_precise_f64x2(a1, b1), + ) } #[inline(always)] - fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - if SHIFT >= 16usize { - return b; - } - let result = { - let a_bytes = self.cvt_to_bytes_u16x16(a).val.0; - let b_bytes = self.cvt_to_bytes_u16x16(b).val.0; - let a_blocks = [a_bytes.0, a_bytes.1]; - let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT * 2usize; - uint8x16x2_t( - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 0, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 1, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - ) - }; - self.cvt_from_bytes_u16x16(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_add_f64x2(a0, b0, c0), + self.mul_add_f64x2(a1, b1, c1), + ) } #[inline(always)] - fn slide_within_blocks_u16x16( - self, - a: u16x16, - b: u16x16, - ) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8( - self.slide_within_blocks_u16x8::(a0, b0), - self.slide_within_blocks_u16x8::(a1, b1), + fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_sub_f64x2(a0, b0, c0), + self.mul_sub_f64x2(a1, b1, c1), ) } #[inline(always)] - fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) + fn floor_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) } #[inline(always)] - fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) + fn ceil_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) } #[inline(always)] - fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) + fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.round_ties_even_f64x2(a0), + self.round_ties_even_f64x2(a1), + ) } #[inline(always)] - fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) + fn fract_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) } #[inline(always)] - fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) + fn trunc_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) } #[inline(always)] - fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) + fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) } #[inline(always)] - fn not_u16x16(self, a: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) + fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { + f64x8 { + val: crate::support::Aligned512(float64x2x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } } #[inline(always)] - fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) + fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { + ( + f64x2 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + f64x2 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) } #[inline(always)] - fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) + fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f32x4( + self.reinterpret_f32_f64x2(a0), + self.reinterpret_f32_f64x2(a1), + ) } #[inline(always)] - fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) + fn splat_i64x4(self, val: i64) -> i64x4 { + let half = self.splat_i64x2(val); + self.combine_i64x2(half, half) } #[inline(always)] - fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, _) = self.split_u16x16(a); - let (b0, _) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (_, a1) = self.split_u16x16(a); - let (_, b1) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + if SHIFT >= 4usize { + return b; + } + let result = { + let a_bytes = self.cvt_to_bytes_i64x4(a).val.0; + let b_bytes = self.cvt_to_bytes_i64x4(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1]; + let b_blocks = [b_bytes.0, b_bytes.1]; + let shift_bytes = SHIFT * 8usize; + uint8x16x2_t( + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 0, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 1, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) + fn slide_within_blocks_i64x4( + self, + a: i64x4, + b: i64x4, + ) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2( + self.slide_within_blocks_i64x2::(a0, b0), + self.slide_within_blocks_i64x2::(a1, b1), + ) } #[inline(always)] - fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_lo = self.zip_low_u16x8(a0, b0); - let lo_hi = self.zip_high_u16x8(a0, b0); - let hi_lo = self.zip_low_u16x8(a1, b1); - let hi_hi = self.zip_high_u16x8(a1, b1); - ( - self.combine_u16x8(lo_lo, lo_hi), - self.combine_u16x8(hi_lo, hi_hi), - ) + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) } #[inline(always)] - fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_even = self.unzip_low_u16x8(a0, a1); - let lo_odd = self.unzip_high_u16x8(a0, a1); - let hi_even = self.unzip_low_u16x8(b0, b1); - let hi_odd = self.unzip_high_u16x8(b0, b1); - ( - self.combine_u16x8(lo_even, hi_even), - self.combine_u16x8(lo_odd, hi_odd), - ) + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) } #[inline(always)] - fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_u16x16(b); - let (c0, c1) = self.split_u16x16(c); - self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) } #[inline(always)] - fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) } #[inline(always)] - fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { - val: crate::support::Aligned512(uint16x8x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { - ( - u16x8 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - u16x8 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, - ) + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) } #[inline(always)] - fn narrow_u16x16(self, a: u16x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Neon, a: u16x16) -> u8x16 { - let converted: uint16x8x2_t = a.into(); - let low = vmovn_u16(converted.0); - let high = vmovn_u16(converted.1); - vcombine_u8(low, high).simd_into(token) - } - ); - kernel(self, a) + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) } #[inline(always)] - fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) } #[inline(always)] - fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u32x4( - self.reinterpret_u32_u16x8(a0), - self.reinterpret_u32_u16x8(a1), - ) + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) } #[inline(always)] - fn splat_mask16x16(self, val: bool) -> mask16x16 { - let half = self.splat_mask16x8(val); - self.combine_mask16x8(half, half) + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) } #[inline(always)] - fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { - mask16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) } #[inline(always)] - fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) } #[inline(always)] - fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { - let lo = self.from_bitmask_mask16x8(bits); - let hi = self.from_bitmask_mask16x8(bits >> 8usize); - self.combine_mask16x8(lo, hi) + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) } #[inline(always)] - fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { - let (lo, hi) = self.split_mask16x16(a); - let lo = self.to_bitmask_mask16x8(lo); - let hi = self.to_bitmask_mask16x8(hi); - lo | (hi << 8usize) + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) } #[inline(always)] - fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) } #[inline(always)] - fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) } #[inline(always)] - fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) } #[inline(always)] - fn not_mask16x16(self, a: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) } #[inline(always)] - fn select_mask16x16( - self, - a: mask16x16, - b: mask16x16, - c: mask16x16, - ) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - let (c0, c1) = self.split_mask16x16(c); - self.combine_mask16x8( - self.select_mask16x8(a0, b0, c0), - self.select_mask16x8(a1, b1, c1), - ) + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) } #[inline(always)] - fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_lo = self.zip_low_i64x2(a0, b0); + let lo_hi = self.zip_high_i64x2(a0, b0); + let hi_lo = self.zip_low_i64x2(a1, b1); + let hi_hi = self.zip_high_i64x2(a1, b1); + ( + self.combine_i64x2(lo_lo, lo_hi), + self.combine_i64x2(hi_lo, hi_hi), + ) } #[inline(always)] - fn any_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_even = self.unzip_low_i64x2(a0, a1); + let lo_odd = self.unzip_high_i64x2(a0, a1); + let hi_even = self.unzip_low_i64x2(b0, b1); + let hi_odd = self.unzip_high_i64x2(b0, b1); + ( + self.combine_i64x2(lo_even, hi_even), + self.combine_i64x2(lo_odd, hi_odd), + ) } #[inline(always)] - fn all_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_i64x4(b); + let (c0, c1) = self.split_i64x4(c); + self.combine_i64x2(self.select_i64x2(a0, b0, c0), self.select_i64x2(a1, b1, c1)) } #[inline(always)] - fn any_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.min_i64x2(a0, b0), self.min_i64x2(a1, b1)) } #[inline(always)] - fn all_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.max_i64x2(a0, b0), self.max_i64x2(a1, b1)) } #[inline(always)] - fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { - mask16x32 { - val: crate::support::Aligned512(int16x8x4_t( + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + i64x8 { + val: crate::support::Aligned512(int64x2x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { ( - mask16x8 { + i64x2 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - mask16x8 { + i64x2 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn splat_i32x8(self, val: i32) -> i32x8 { - let half = self.splat_i32x4(val); - self.combine_i32x4(half, half) + fn neg_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] - fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { - i32x8 { + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u8x16(self.reinterpret_u8_i64x2(a0), self.reinterpret_u8_i64x2(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u32x4( + self.reinterpret_u32_i64x2(a0), + self.reinterpret_u32_i64x2(a1), + ) + } + #[inline(always)] + fn splat_u64x4(self, val: u64) -> u64x4 { + let half = self.splat_u64x2(val); + self.combine_u64x2(half, half) + } + #[inline(always)] + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { - i32x8 { + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { - i32x8 { + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - if SHIFT >= 8usize { + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + if SHIFT >= 4usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i32x8(a).val.0; - let b_bytes = self.cvt_to_bytes_i32x8(b).val.0; + let a_bytes = self.cvt_to_bytes_u64x4(a).val.0; + let b_bytes = self.cvt_to_bytes_u64x4(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1]; let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT * 4usize; + let shift_bytes = SHIFT * 8usize; uint8x16x2_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -5749,292 +7942,408 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i32x8(u8x32 { + self.cvt_from_bytes_u64x4(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x8( + fn slide_within_blocks_u64x4( self, - a: i32x8, - b: i32x8, - ) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4( - self.slide_within_blocks_i32x4::(a0, b0), - self.slide_within_blocks_i32x4::(a1, b1), + a: u64x4, + b: u64x4, + ) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2( + self.slide_within_blocks_u64x2::(a0, b0), + self.slide_within_blocks_u64x2::(a1, b1), ) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) } #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) } #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1)) } #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.and_u64x2(a0, b0), self.and_u64x2(a1, b1)) } #[inline(always)] - fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.or_u64x2(a0, b0), self.or_u64x2(a1, b1)) } #[inline(always)] - fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.xor_u64x2(a0, b0), self.xor_u64x2(a1, b1)) } #[inline(always)] - fn not_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) + fn not_u64x4(self, a: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.not_u64x2(a0), self.not_u64x2(a1)) } #[inline(always)] - fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shl_u64x2(a0, shift), self.shl_u64x2(a1, shift)) } #[inline(always)] - fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shlv_u64x2(a0, b0), self.shlv_u64x2(a1, b1)) } #[inline(always)] - fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shr_u64x2(a0, shift), self.shr_u64x2(a1, shift)) } #[inline(always)] - fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shrv_u64x2(a0, b0), self.shrv_u64x2(a1, b1)) } #[inline(always)] - fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_eq_u64x2(a0, b0), self.simd_eq_u64x2(a1, b1)) } #[inline(always)] - fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_lt_u64x2(a0, b0), self.simd_lt_u64x2(a1, b1)) } #[inline(always)] - fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_le_u64x2(a0, b0), self.simd_le_u64x2(a1, b1)) } #[inline(always)] - fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_ge_u64x2(a0, b0), self.simd_ge_u64x2(a1, b1)) } #[inline(always)] - fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_gt_u64x2(a0, b0), self.simd_gt_u64x2(a1, b1)) } #[inline(always)] - fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, _) = self.split_i32x8(a); - let (b0, _) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, _) = self.split_u64x4(a); + let (b0, _) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a0, b0), self.zip_high_u64x2(a0, b0)) } #[inline(always)] - fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (_, a1) = self.split_i32x8(a); - let (_, b1) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (_, a1) = self.split_u64x4(a); + let (_, b1) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a1, b1), self.zip_high_u64x2(a1, b1)) } #[inline(always)] - fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_low_u64x2(a0, a1), self.unzip_low_u64x2(b0, b1)) } #[inline(always)] - fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_high_u64x2(a0, a1), self.unzip_high_u64x2(b0, b1)) } #[inline(always)] - fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_lo = self.zip_low_i32x4(a0, b0); - let lo_hi = self.zip_high_i32x4(a0, b0); - let hi_lo = self.zip_low_i32x4(a1, b1); - let hi_hi = self.zip_high_i32x4(a1, b1); + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_lo = self.zip_low_u64x2(a0, b0); + let lo_hi = self.zip_high_u64x2(a0, b0); + let hi_lo = self.zip_low_u64x2(a1, b1); + let hi_hi = self.zip_high_u64x2(a1, b1); ( - self.combine_i32x4(lo_lo, lo_hi), - self.combine_i32x4(hi_lo, hi_hi), + self.combine_u64x2(lo_lo, lo_hi), + self.combine_u64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_even = self.unzip_low_i32x4(a0, a1); - let lo_odd = self.unzip_high_i32x4(a0, a1); - let hi_even = self.unzip_low_i32x4(b0, b1); - let hi_odd = self.unzip_high_i32x4(b0, b1); + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_even = self.unzip_low_u64x2(a0, a1); + let lo_odd = self.unzip_high_u64x2(a0, a1); + let hi_even = self.unzip_low_u64x2(b0, b1); + let hi_odd = self.unzip_high_u64x2(b0, b1); ( - self.combine_i32x4(lo_even, hi_even), - self.combine_i32x4(lo_odd, hi_odd), + self.combine_u64x2(lo_even, hi_even), + self.combine_u64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_i32x8(b); - let (c0, c1) = self.split_i32x8(c); - self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_u64x4(b); + let (c0, c1) = self.split_u64x4(c); + self.combine_u64x2(self.select_u64x2(a0, b0, c0), self.select_u64x2(a1, b1, c1)) } #[inline(always)] - fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.min_u64x2(a0, b0), self.min_u64x2(a1, b1)) } #[inline(always)] - fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.max_u64x2(a0, b0), self.max_u64x2(a1, b1)) } #[inline(always)] - fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { - i32x16 { - val: crate::support::Aligned512(int32x4x4_t( + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + u64x8 { + val: crate::support::Aligned512(uint64x2x4_t( a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, )), simd: self, } } #[inline(always)] - fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { ( - i32x4 { + u64x2 { val: crate::support::Aligned128(a.val.0.0), simd: self, }, - i32x4 { + u64x2 { val: crate::support::Aligned128(a.val.0.1), simd: self, }, ) } #[inline(always)] - fn neg_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u8x16(self.reinterpret_u8_u64x2(a0), self.reinterpret_u8_u64x2(a1)) } #[inline(always)] - fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { - let (a0, a1) = self.split_i32x8(a); - self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u32x4( + self.reinterpret_u32_u64x2(a0), + self.reinterpret_u32_u64x2(a1), + ) } #[inline(always)] - fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_u32x4( - self.reinterpret_u32_i32x4(a0), - self.reinterpret_u32_i32x4(a1), + fn splat_mask64x4(self, val: bool) -> mask64x4 { + let half = self.splat_mask64x2(val); + self.combine_mask64x2(half, half) + } + #[inline(always)] + fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { + mask64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { + let lo = self.from_bitmask_mask64x2(bits); + let hi = self.from_bitmask_mask64x2(bits >> 2usize); + self.combine_mask64x2(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + let (lo, hi) = self.split_mask64x4(a); + let lo = self.to_bitmask_mask64x2(lo); + let hi = self.to_bitmask_mask64x2(hi); + lo | (hi << 2usize) + } + #[inline(always)] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask64x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x4(lanes); + } + #[inline(always)] + fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) + } + #[inline(always)] + fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) + } + #[inline(always)] + fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) + } + #[inline(always)] + fn not_mask64x4(self, a: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) + } + #[inline(always)] + fn select_mask64x4( + self, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + let (c0, c1) = self.split_mask64x4(c); + self.combine_mask64x2( + self.select_mask64x2(a0, b0, c0), + self.select_mask64x2(a1, b1, c1), ) } #[inline(always)] - fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) + fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) } #[inline(always)] - fn splat_u32x8(self, val: u32) -> u32x8 { - let half = self.splat_u32x4(val); - self.combine_u32x4(half, half) + fn any_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) + } + #[inline(always)] + fn all_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) + } + #[inline(always)] + fn any_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) + } + #[inline(always)] + fn all_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + } + #[inline(always)] + fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { + mask64x8 { + val: crate::support::Aligned512(int64x2x4_t( + a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, + )), + simd: self, + } + } + #[inline(always)] + fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + ( + mask64x2 { + val: crate::support::Aligned128(a.val.0.0), + simd: self, + }, + mask64x2 { + val: crate::support::Aligned128(a.val.0.1), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_f32x16(self, val: f32) -> f32x16 { + let half = self.splat_f32x8(val); + self.combine_f32x8(half, half) } #[inline(always)] - fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { + fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { - u32x8 { + fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { - u8x32 { + fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { + u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - if SHIFT >= 8usize { + fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + if SHIFT >= 16usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_u32x8(a).val.0; - let b_bytes = self.cvt_to_bytes_u32x8(b).val.0; - let a_blocks = [a_bytes.0, a_bytes.1]; - let b_blocks = [b_bytes.0, b_bytes.1]; + let a_bytes = self.cvt_to_bytes_f32x16(a).val.0; + let b_bytes = self.cvt_to_bytes_f32x16(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; + let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; let shift_bytes = SHIFT * 4usize; - uint8x16x2_t( + uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( &a_blocks, @@ -6053,396 +8362,399 @@ impl Simd for Neon { ); dyn_vext_128(self, lo, hi, shift_bytes % 16) }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 2, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 3, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, ) }; - self.cvt_from_bytes_u32x8(u8x32 { - val: crate::support::Aligned256(result), + self.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u32x8( + fn slide_within_blocks_f32x16( self, - a: u32x8, - b: u32x8, - ) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4( - self.slide_within_blocks_u32x4::(a0, b0), - self.slide_within_blocks_u32x4::(a1, b1), + a: f32x16, + b: f32x16, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.slide_within_blocks_f32x8::(a0, b0), + self.slide_within_blocks_f32x8::(a1, b1), ) } #[inline(always)] - fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) - } - #[inline(always)] - fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) - } - #[inline(always)] - fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) } #[inline(always)] - fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) } #[inline(always)] - fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) } #[inline(always)] - fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) } #[inline(always)] - fn not_u32x8(self, a: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) } #[inline(always)] - fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) } #[inline(always)] - fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) } #[inline(always)] - fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) } #[inline(always)] - fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) } #[inline(always)] - fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) } #[inline(always)] - fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) } #[inline(always)] - fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) } #[inline(always)] - fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) } #[inline(always)] - fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) } #[inline(always)] - fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, _) = self.split_u32x8(a); - let (b0, _) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, _) = self.split_f32x16(a); + let (b0, _) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) } #[inline(always)] - fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (_, a1) = self.split_u32x8(a); - let (_, b1) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) + fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (_, a1) = self.split_f32x16(a); + let (_, b1) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) } #[inline(always)] - fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) + fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) } #[inline(always)] - fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) + fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) } #[inline(always)] - fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_lo = self.zip_low_u32x4(a0, b0); - let lo_hi = self.zip_high_u32x4(a0, b0); - let hi_lo = self.zip_low_u32x4(a1, b1); - let hi_hi = self.zip_high_u32x4(a1, b1); + fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_lo = self.zip_low_f32x8(a0, b0); + let lo_hi = self.zip_high_f32x8(a0, b0); + let hi_lo = self.zip_low_f32x8(a1, b1); + let hi_hi = self.zip_high_f32x8(a1, b1); ( - self.combine_u32x4(lo_lo, lo_hi), - self.combine_u32x4(hi_lo, hi_hi), + self.combine_f32x8(lo_lo, lo_hi), + self.combine_f32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_even = self.unzip_low_u32x4(a0, a1); - let lo_odd = self.unzip_high_u32x4(a0, a1); - let hi_even = self.unzip_low_u32x4(b0, b1); - let hi_odd = self.unzip_high_u32x4(b0, b1); + fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_even = self.unzip_low_f32x8(a0, a1); + let lo_odd = self.unzip_high_f32x8(a0, a1); + let hi_even = self.unzip_low_f32x8(b0, b1); + let hi_odd = self.unzip_high_f32x8(b0, b1); ( - self.combine_u32x4(lo_even, hi_even), - self.combine_u32x4(lo_odd, hi_odd), + self.combine_f32x8(lo_even, hi_even), + self.combine_f32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_u32x8(b); - let (c0, c1) = self.split_u32x8(c); - self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) - } - #[inline(always)] - fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) - } - #[inline(always)] - fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) + fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) } #[inline(always)] - fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { - u32x16 { - val: crate::support::Aligned512(uint32x4x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) } #[inline(always)] - fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { - ( - u32x4 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - u32x4 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, + fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.max_precise_f32x8(a0, b0), + self.max_precise_f32x8(a1, b1), ) } #[inline(always)] - fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) + fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.min_precise_f32x8(a0, b0), + self.min_precise_f32x8(a1, b1), + ) } #[inline(always)] - fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) + fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_add_f32x8(a0, b0, c0), + self.mul_add_f32x8(a1, b1, c1), + ) } #[inline(always)] - fn splat_mask32x8(self, val: bool) -> mask32x8 { - let half = self.splat_mask32x4(val); - self.combine_mask32x4(half, half) + fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_sub_f32x8(a0, b0, c0), + self.mul_sub_f32x8(a1, b1, c1), + ) } #[inline(always)] - fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { - mask32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn floor_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) } #[inline(always)] - fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn ceil_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) } #[inline(always)] - fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { - let lo = self.from_bitmask_mask32x4(bits); - let hi = self.from_bitmask_mask32x4(bits >> 4usize); - self.combine_mask32x4(lo, hi) + fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.round_ties_even_f32x8(a0), + self.round_ties_even_f32x8(a1), + ) } #[inline(always)] - fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { - let (lo, hi) = self.split_mask32x8(a); - let lo = self.to_bitmask_mask32x4(lo); - let hi = self.to_bitmask_mask32x4(hi); - lo | (hi << 4usize) + fn fract_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) } #[inline(always)] - fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) + fn trunc_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) } #[inline(always)] - fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) + fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) } #[inline(always)] - fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) + fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + ( + f32x8 { + val: crate::support::Aligned256(float32x4x2_t(a.val.0.0, a.val.0.1)), + simd: self, + }, + f32x8 { + val: crate::support::Aligned256(float32x4x2_t(a.val.0.2, a.val.0.3)), + simd: self, + }, + ) } #[inline(always)] - fn not_mask32x8(self, a: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) + fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f64x4( + self.reinterpret_f64_f32x8(a0), + self.reinterpret_f64_f32x8(a1), + ) } #[inline(always)] - fn select_mask32x8( - self, - a: mask32x8, - b: mask32x8, - c: mask32x8, - ) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - let (c0, c1) = self.split_mask32x8(c); - self.combine_mask32x4( - self.select_mask32x4(a0, b0, c0), - self.select_mask32x4(a1, b1, c1), + fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.reinterpret_i32_f32x8(a0), + self.reinterpret_i32_f32x8(a1), ) } #[inline(always)] - fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) + fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { + unsafe { vld4q_f32(src.as_ptr()).simd_into(self) } } #[inline(always)] - fn any_true_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) + fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + unsafe { vst4q_f32(dest.as_mut_ptr(), a.into()) } } #[inline(always)] - fn all_true_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) + fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) } #[inline(always)] - fn any_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) + fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.reinterpret_u32_f32x8(a0), + self.reinterpret_u32_f32x8(a1), + ) } #[inline(always)] - fn all_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) + fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) } #[inline(always)] - fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { - mask32x16 { - val: crate::support::Aligned512(int32x4x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.cvt_u32_precise_f32x8(a0), + self.cvt_u32_precise_f32x8(a1), + ) } #[inline(always)] - fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { - ( - mask32x4 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - mask32x4 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, - ) + fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) } #[inline(always)] - fn splat_f64x4(self, val: f64) -> f64x4 { - let half = self.splat_f64x2(val); - self.combine_f64x2(half, half) + fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.cvt_i32_precise_f32x8(a0), + self.cvt_i32_precise_f32x8(a1), + ) } #[inline(always)] - fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { - f64x4 { + fn splat_i8x64(self, val: i8) -> i8x64 { + let half = self.splat_i8x32(val); + self.combine_i8x32(half, half) + } + #[inline(always)] + fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { + i8x64 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { - f64x4 { + fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { + i8x64 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { + fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { - f64x4 { + fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { + i8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { - u8x32 { + fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { + u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - if SHIFT >= 4usize { + fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + if SHIFT >= 64usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_f64x4(a).val.0; - let b_bytes = self.cvt_to_bytes_f64x4(b).val.0; - let a_blocks = [a_bytes.0, a_bytes.1]; - let b_blocks = [b_bytes.0, b_bytes.1]; - let shift_bytes = SHIFT * 8usize; - uint8x16x2_t( + let a_bytes = self.cvt_to_bytes_i8x64(a).val.0; + let b_bytes = self.cvt_to_bytes_i8x64(b).val.0; + let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; + let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; + let shift_bytes = SHIFT; + uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( &a_blocks, @@ -6461,449 +8773,297 @@ impl Simd for Neon { ); dyn_vext_128(self, lo, hi, shift_bytes % 16) }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 2, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 3, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, ) }; - self.cvt_from_bytes_f64x4(u8x32 { - val: crate::support::Aligned256(result), + self.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x4( + fn slide_within_blocks_i8x64( self, - a: f64x4, - b: f64x4, - ) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.slide_within_blocks_f64x2::(a0, b0), - self.slide_within_blocks_f64x2::(a1, b1), - ) - } - #[inline(always)] - fn abs_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) - } - #[inline(always)] - fn neg_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) - } - #[inline(always)] - fn sqrt_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) - } - #[inline(always)] - fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.approximate_recip_f64x2(a0), - self.approximate_recip_f64x2(a1), - ) - } - #[inline(always)] - fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) - } - #[inline(always)] - fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) - } - #[inline(always)] - fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) - } - #[inline(always)] - fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) - } - #[inline(always)] - fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) - } - #[inline(always)] - fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) - } - #[inline(always)] - fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) - } - #[inline(always)] - fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) - } - #[inline(always)] - fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) - } - #[inline(always)] - fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) - } - #[inline(always)] - fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, _) = self.split_f64x4(a); - let (b0, _) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) - } - #[inline(always)] - fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (_, a1) = self.split_f64x4(a); - let (_, b1) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) - } - #[inline(always)] - fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) - } - #[inline(always)] - fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) - } - #[inline(always)] - fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_lo = self.zip_low_f64x2(a0, b0); - let lo_hi = self.zip_high_f64x2(a0, b0); - let hi_lo = self.zip_low_f64x2(a1, b1); - let hi_hi = self.zip_high_f64x2(a1, b1); - ( - self.combine_f64x2(lo_lo, lo_hi), - self.combine_f64x2(hi_lo, hi_hi), - ) - } - #[inline(always)] - fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_even = self.unzip_low_f64x2(a0, a1); - let lo_odd = self.unzip_high_f64x2(a0, a1); - let hi_even = self.unzip_low_f64x2(b0, b1); - let hi_odd = self.unzip_high_f64x2(b0, b1); - ( - self.combine_f64x2(lo_even, hi_even), - self.combine_f64x2(lo_odd, hi_odd), - ) - } - #[inline(always)] - fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) - } - #[inline(always)] - fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) - } - #[inline(always)] - fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.max_precise_f64x2(a0, b0), - self.max_precise_f64x2(a1, b1), + a: i8x64, + b: i8x64, + ) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32( + self.slide_within_blocks_i8x32::(a0, b0), + self.slide_within_blocks_i8x32::(a1, b1), ) } #[inline(always)] - fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.min_precise_f64x2(a0, b0), - self.min_precise_f64x2(a1, b1), - ) + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) } #[inline(always)] - fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_add_f64x2(a0, b0, c0), - self.mul_add_f64x2(a1, b1, c1), - ) + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) } #[inline(always)] - fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_sub_f64x2(a0, b0, c0), - self.mul_sub_f64x2(a1, b1, c1), - ) + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) } #[inline(always)] - fn floor_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) } #[inline(always)] - fn ceil_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) } #[inline(always)] - fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.round_ties_even_f64x2(a0), - self.round_ties_even_f64x2(a1), - ) + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) } #[inline(always)] - fn fract_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) + fn not_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) } #[inline(always)] - fn trunc_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) } #[inline(always)] - fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) } #[inline(always)] - fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { - f64x8 { - val: crate::support::Aligned512(float64x2x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) } #[inline(always)] - fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { - ( - f64x2 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - f64x2 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, - ) + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) } #[inline(always)] - fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f32x4( - self.reinterpret_f32_f64x2(a0), - self.reinterpret_f32_f64x2(a1), - ) + fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) } #[inline(always)] - fn splat_mask64x4(self, val: bool) -> mask64x4 { - let half = self.splat_mask64x2(val); - self.combine_mask64x2(half, half) + fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) } #[inline(always)] - fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { - mask64x4 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) } #[inline(always)] - fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) } #[inline(always)] - fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { - let lo = self.from_bitmask_mask64x2(bits); - let hi = self.from_bitmask_mask64x2(bits >> 2usize); - self.combine_mask64x2(lo, hi) + fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) } #[inline(always)] - fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { - let (lo, hi) = self.split_mask64x4(a); - let lo = self.to_bitmask_mask64x2(lo); - let hi = self.to_bitmask_mask64x2(hi); - lo | (hi << 2usize) + fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, _) = self.split_i8x64(a); + let (b0, _) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) } #[inline(always)] - fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) + fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (_, a1) = self.split_i8x64(a); + let (_, b1) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) } #[inline(always)] - fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) + fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) } #[inline(always)] - fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) + fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) } #[inline(always)] - fn not_mask64x4(self, a: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) + fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_lo = self.zip_low_i8x32(a0, b0); + let lo_hi = self.zip_high_i8x32(a0, b0); + let hi_lo = self.zip_low_i8x32(a1, b1); + let hi_hi = self.zip_high_i8x32(a1, b1); + ( + self.combine_i8x32(lo_lo, lo_hi), + self.combine_i8x32(hi_lo, hi_hi), + ) } #[inline(always)] - fn select_mask64x4( - self, - a: mask64x4, - b: mask64x4, - c: mask64x4, - ) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - let (c0, c1) = self.split_mask64x4(c); - self.combine_mask64x2( - self.select_mask64x2(a0, b0, c0), - self.select_mask64x2(a1, b1, c1), + fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_even = self.unzip_low_i8x32(a0, a1); + let lo_odd = self.unzip_high_i8x32(a0, a1); + let hi_even = self.unzip_low_i8x32(b0, b1); + let hi_odd = self.unzip_high_i8x32(b0, b1); + ( + self.combine_i8x32(lo_even, hi_even), + self.combine_i8x32(lo_odd, hi_odd), ) } #[inline(always)] - fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) + fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_i8x64(b); + let (c0, c1) = self.split_i8x64(c); + self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) } #[inline(always)] - fn any_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) + fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) } #[inline(always)] - fn all_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) + fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) } #[inline(always)] - fn any_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) + fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + ( + i8x32 { + val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)), + simd: self, + }, + i8x32 { + val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)), + simd: self, + }, + ) } #[inline(always)] - fn all_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + fn neg_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] - fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { - mask64x8 { - val: crate::support::Aligned512(int64x2x4_t( - a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1, - )), - simd: self, - } + fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) } #[inline(always)] - fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { - ( - mask64x2 { - val: crate::support::Aligned128(a.val.0.0), - simd: self, - }, - mask64x2 { - val: crate::support::Aligned128(a.val.0.1), - simd: self, - }, + fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u32x8( + self.reinterpret_u32_i8x32(a0), + self.reinterpret_u32_i8x32(a1), ) } #[inline(always)] - fn splat_f32x16(self, val: f32) -> f32x16 { - let half = self.splat_f32x8(val); - self.combine_f32x8(half, half) + fn splat_u8x64(self, val: u8) -> u8x64 { + let half = self.splat_u8x32(val); + self.combine_u8x32(half, half) } #[inline(always)] - fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { + u8x64 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { + u8x64 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { - f32x16 { + fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { + fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - if SHIFT >= 16usize { + fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + if SHIFT >= 64usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_f32x16(a).val.0; - let b_bytes = self.cvt_to_bytes_f32x16(b).val.0; + let a_bytes = self.cvt_to_bytes_u8x64(a).val.0; + let b_bytes = self.cvt_to_bytes_u8x64(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT * 4usize; + let shift_bytes = SHIFT; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -6943,378 +9103,392 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_f32x16(u8x64 { + self.cvt_from_bytes_u8x64(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x16( + fn slide_within_blocks_u8x64( self, - a: f32x16, - b: f32x16, - ) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.slide_within_blocks_f32x8::(a0, b0), - self.slide_within_blocks_f32x8::(a1, b1), + a: u8x64, + b: u8x64, + ) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32( + self.slide_within_blocks_u8x32::(a0, b0), + self.slide_within_blocks_u8x32::(a1, b1), ) } #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) } #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) } #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) } #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), - ) + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) } #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) } #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) } #[inline(always)] - fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + fn not_u8x64(self, a: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) } #[inline(always)] - fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) } #[inline(always)] - fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) } #[inline(always)] - fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) } #[inline(always)] - fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) } #[inline(always)] - fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) } #[inline(always)] - fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) } #[inline(always)] - fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) } #[inline(always)] - fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, _) = self.split_f32x16(a); - let (b0, _) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) } #[inline(always)] - fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (_, a1) = self.split_f32x16(a); - let (_, b1) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) } #[inline(always)] - fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) + fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, _) = self.split_u8x64(a); + let (b0, _) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) } #[inline(always)] - fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) + fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (_, a1) = self.split_u8x64(a); + let (_, b1) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) } #[inline(always)] - fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_lo = self.zip_low_f32x8(a0, b0); - let lo_hi = self.zip_high_f32x8(a0, b0); - let hi_lo = self.zip_low_f32x8(a1, b1); - let hi_hi = self.zip_high_f32x8(a1, b1); + fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + } + #[inline(always)] + fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + } + #[inline(always)] + fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_lo = self.zip_low_u8x32(a0, b0); + let lo_hi = self.zip_high_u8x32(a0, b0); + let hi_lo = self.zip_low_u8x32(a1, b1); + let hi_hi = self.zip_high_u8x32(a1, b1); + ( + self.combine_u8x32(lo_lo, lo_hi), + self.combine_u8x32(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_even = self.unzip_low_u8x32(a0, a1); + let lo_odd = self.unzip_high_u8x32(a0, a1); + let hi_even = self.unzip_low_u8x32(b0, b1); + let hi_odd = self.unzip_high_u8x32(b0, b1); ( - self.combine_f32x8(lo_lo, lo_hi), - self.combine_f32x8(hi_lo, hi_hi), + self.combine_u8x32(lo_even, hi_even), + self.combine_u8x32(lo_odd, hi_odd), ) } #[inline(always)] - fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_even = self.unzip_low_f32x8(a0, a1); - let lo_odd = self.unzip_high_f32x8(a0, a1); - let hi_even = self.unzip_low_f32x8(b0, b1); - let hi_odd = self.unzip_high_f32x8(b0, b1); - ( - self.combine_f32x8(lo_even, hi_even), - self.combine_f32x8(lo_odd, hi_odd), - ) + fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_u8x64(b); + let (c0, c1) = self.split_u8x64(c); + self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) } #[inline(always)] - fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) + fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) } #[inline(always)] - fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) + fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) } #[inline(always)] - fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.max_precise_f32x8(a0, b0), - self.max_precise_f32x8(a1, b1), + fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + ( + u8x32 { + val: crate::support::Aligned256(uint8x16x2_t(a.val.0.0, a.val.0.1)), + simd: self, + }, + u8x32 { + val: crate::support::Aligned256(uint8x16x2_t(a.val.0.2, a.val.0.3)), + simd: self, + }, ) } #[inline(always)] - fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.min_precise_f32x8(a0, b0), - self.min_precise_f32x8(a1, b1), - ) + fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { + unsafe { vld4q_u8(src.as_ptr()).simd_into(self) } } #[inline(always)] - fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_add_f32x8(a0, b0, c0), - self.mul_add_f32x8(a1, b1, c1), - ) + fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + unsafe { vst4q_u8(dest.as_mut_ptr(), a.into()) } } #[inline(always)] - fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_sub_f32x8(a0, b0, c0), - self.mul_sub_f32x8(a1, b1, c1), + fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u32x8( + self.reinterpret_u32_u8x32(a0), + self.reinterpret_u32_u8x32(a1), ) } #[inline(always)] - fn floor_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) + fn splat_mask8x64(self, val: bool) -> mask8x64 { + let half = self.splat_mask8x32(val); + self.combine_mask8x32(half, half) } #[inline(always)] - fn ceil_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) + fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { + mask8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.round_ties_even_f32x8(a0), - self.round_ties_even_f32x8(a1), - ) + fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn fract_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) + fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { + let lo = self.from_bitmask_mask8x32(bits); + let hi = self.from_bitmask_mask8x32(bits >> 32usize); + self.combine_mask8x32(lo, hi) } #[inline(always)] - fn trunc_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) + fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { + let (lo, hi) = self.split_mask8x64(a); + let lo = self.to_bitmask_mask8x32(lo); + let hi = self.to_bitmask_mask8x32(hi); + lo | (hi << 32usize) } #[inline(always)] - fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let mut lanes = self.as_array_mask8x64(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x64(lanes); } #[inline(always)] - fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { - ( - f32x8 { - val: crate::support::Aligned256(float32x4x2_t(a.val.0.0, a.val.0.1)), - simd: self, - }, - f32x8 { - val: crate::support::Aligned256(float32x4x2_t(a.val.0.2, a.val.0.3)), - simd: self, - }, - ) + fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) } #[inline(always)] - fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f64x4( - self.reinterpret_f64_f32x8(a0), - self.reinterpret_f64_f32x8(a1), - ) + fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) } #[inline(always)] - fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.reinterpret_i32_f32x8(a0), - self.reinterpret_i32_f32x8(a1), - ) + fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - unsafe { vld4q_f32(src.as_ptr()).simd_into(self) } + fn not_mask8x64(self, a: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) } #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - unsafe { vst4q_f32(dest.as_mut_ptr(), a.into()) } + fn select_mask8x64( + self, + a: mask8x64, + b: mask8x64, + c: mask8x64, + ) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + let (c0, c1) = self.split_mask8x64(c); + self.combine_mask8x32( + self.select_mask8x32(a0, b0, c0), + self.select_mask8x32(a1, b1, c1), + ) } #[inline(always)] - fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) + fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) } #[inline(always)] - fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.reinterpret_u32_f32x8(a0), - self.reinterpret_u32_f32x8(a1), - ) + fn any_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) } #[inline(always)] - fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) + fn all_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) } #[inline(always)] - fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.cvt_u32_precise_f32x8(a0), - self.cvt_u32_precise_f32x8(a1), - ) + fn any_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) } #[inline(always)] - fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) + fn all_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) } #[inline(always)] - fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.cvt_i32_precise_f32x8(a0), - self.cvt_i32_precise_f32x8(a1), + fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + ( + mask8x32 { + val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)), + simd: self, + }, + mask8x32 { + val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)), + simd: self, + }, ) } #[inline(always)] - fn splat_i8x64(self, val: i8) -> i8x64 { - let half = self.splat_i8x32(val); - self.combine_i8x32(half, half) + fn splat_i16x32(self, val: i16) -> i16x32 { + let half = self.splat_i16x16(val); + self.combine_i16x16(half, half) } #[inline(always)] - fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { + i16x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { + i16x32 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { - i8x64 { + fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { + i16x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { + fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - if SHIFT >= 64usize { + fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + if SHIFT >= 32usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i8x64(a).val.0; - let b_bytes = self.cvt_to_bytes_i8x64(b).val.0; + let a_bytes = self.cvt_to_bytes_i16x32(a).val.0; + let b_bytes = self.cvt_to_bytes_i16x32(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT; + let shift_bytes = SHIFT * 2usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -7354,277 +9528,286 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i8x64(u8x64 { + self.cvt_from_bytes_i16x32(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x64( + fn slide_within_blocks_i16x32( self, - a: i8x64, - b: i8x64, - ) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32( - self.slide_within_blocks_i8x32::(a0, b0), - self.slide_within_blocks_i8x32::(a1, b1), + a: i16x32, + b: i16x32, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.slide_within_blocks_i16x16::(a0, b0), + self.slide_within_blocks_i16x16::(a1, b1), ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) } #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) } #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) } #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) } #[inline(always)] - fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) } #[inline(always)] - fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) } #[inline(always)] - fn not_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + fn not_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) } #[inline(always)] - fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) } #[inline(always)] - fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) } #[inline(always)] - fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) } #[inline(always)] - fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) } #[inline(always)] - fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) } #[inline(always)] - fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) } #[inline(always)] - fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) } #[inline(always)] - fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) } #[inline(always)] - fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) } #[inline(always)] - fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, _) = self.split_i8x64(a); - let (b0, _) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) + fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, _) = self.split_i16x32(a); + let (b0, _) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) } #[inline(always)] - fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (_, a1) = self.split_i8x64(a); - let (_, b1) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) + fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (_, a1) = self.split_i16x32(a); + let (_, b1) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) } #[inline(always)] - fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) + fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) } #[inline(always)] - fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) + fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.unzip_high_i16x16(a0, a1), + self.unzip_high_i16x16(b0, b1), + ) } #[inline(always)] - fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_lo = self.zip_low_i8x32(a0, b0); - let lo_hi = self.zip_high_i8x32(a0, b0); - let hi_lo = self.zip_low_i8x32(a1, b1); - let hi_hi = self.zip_high_i8x32(a1, b1); + fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_lo = self.zip_low_i16x16(a0, b0); + let lo_hi = self.zip_high_i16x16(a0, b0); + let hi_lo = self.zip_low_i16x16(a1, b1); + let hi_hi = self.zip_high_i16x16(a1, b1); ( - self.combine_i8x32(lo_lo, lo_hi), - self.combine_i8x32(hi_lo, hi_hi), + self.combine_i16x16(lo_lo, lo_hi), + self.combine_i16x16(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_even = self.unzip_low_i8x32(a0, a1); - let lo_odd = self.unzip_high_i8x32(a0, a1); - let hi_even = self.unzip_low_i8x32(b0, b1); - let hi_odd = self.unzip_high_i8x32(b0, b1); + fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_even = self.unzip_low_i16x16(a0, a1); + let lo_odd = self.unzip_high_i16x16(a0, a1); + let hi_even = self.unzip_low_i16x16(b0, b1); + let hi_odd = self.unzip_high_i16x16(b0, b1); ( - self.combine_i8x32(lo_even, hi_even), - self.combine_i8x32(lo_odd, hi_odd), + self.combine_i16x16(lo_even, hi_even), + self.combine_i16x16(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_i8x64(b); - let (c0, c1) = self.split_i8x64(c); - self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) + fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_i16x32(b); + let (c0, c1) = self.split_i16x32(c); + self.combine_i16x16( + self.select_i16x16(a0, b0, c0), + self.select_i16x16(a1, b1, c1), + ) } #[inline(always)] - fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) } #[inline(always)] - fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) } #[inline(always)] - fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( - i8x32 { - val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)), + i16x16 { + val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - i8x32 { - val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)), + i16x16 { + val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn neg_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) + fn neg_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) + fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { + let (a0, a1) = self.split_i16x32(a); + self.combine_u8x32( + self.reinterpret_u8_i16x16(a0), + self.reinterpret_u8_i16x16(a1), + ) } #[inline(always)] - fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { - let (a0, a1) = self.split_i8x64(a); + fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { + let (a0, a1) = self.split_i16x32(a); self.combine_u32x8( - self.reinterpret_u32_i8x32(a0), - self.reinterpret_u32_i8x32(a1), + self.reinterpret_u32_i16x16(a0), + self.reinterpret_u32_i16x16(a1), ) } #[inline(always)] - fn splat_u8x64(self, val: u8) -> u8x64 { - let half = self.splat_u8x32(val); - self.combine_u8x32(half, half) + fn splat_u16x32(self, val: u16) -> u16x32 { + let half = self.splat_u16x16(val); + self.combine_u16x16(half, half) } #[inline(always)] - fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { + u16x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { + u16x32 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { - u8x64 { + fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { + u16x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { + fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - if SHIFT >= 64usize { + fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + if SHIFT >= 32usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_u8x64(a).val.0; - let b_bytes = self.cvt_to_bytes_u8x64(b).val.0; + let a_bytes = self.cvt_to_bytes_u16x32(a).val.0; + let b_bytes = self.cvt_to_bytes_u16x32(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT; + let shift_bytes = SHIFT * 2usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -7664,381 +9847,414 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_u8x64(u8x64 { + self.cvt_from_bytes_u16x32(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x64( + fn slide_within_blocks_u16x32( self, - a: u8x64, - b: u8x64, - ) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32( - self.slide_within_blocks_u8x32::(a0, b0), - self.slide_within_blocks_u8x32::(a1, b1), + a: u16x32, + b: u16x32, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.slide_within_blocks_u16x16::(a0, b0), + self.slide_within_blocks_u16x16::(a1, b1), ) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) } #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) } #[inline(always)] - fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) } #[inline(always)] - fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) } #[inline(always)] - fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) } #[inline(always)] - fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) } #[inline(always)] - fn not_u8x64(self, a: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + fn not_u16x32(self, a: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) } #[inline(always)] - fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) } #[inline(always)] - fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) } #[inline(always)] - fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) } #[inline(always)] - fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) } #[inline(always)] - fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) } #[inline(always)] - fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) } #[inline(always)] - fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) } #[inline(always)] - fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) } #[inline(always)] - fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) + fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) } #[inline(always)] - fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, _) = self.split_u8x64(a); - let (b0, _) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) + fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, _) = self.split_u16x32(a); + let (b0, _) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) } #[inline(always)] - fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (_, a1) = self.split_u8x64(a); - let (_, b1) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) + fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (_, a1) = self.split_u16x32(a); + let (_, b1) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) } #[inline(always)] - fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) } #[inline(always)] - fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.unzip_high_u16x16(a0, a1), + self.unzip_high_u16x16(b0, b1), + ) } #[inline(always)] - fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_lo = self.zip_low_u8x32(a0, b0); - let lo_hi = self.zip_high_u8x32(a0, b0); - let hi_lo = self.zip_low_u8x32(a1, b1); - let hi_hi = self.zip_high_u8x32(a1, b1); + fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_lo = self.zip_low_u16x16(a0, b0); + let lo_hi = self.zip_high_u16x16(a0, b0); + let hi_lo = self.zip_low_u16x16(a1, b1); + let hi_hi = self.zip_high_u16x16(a1, b1); ( - self.combine_u8x32(lo_lo, lo_hi), - self.combine_u8x32(hi_lo, hi_hi), + self.combine_u16x16(lo_lo, lo_hi), + self.combine_u16x16(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_even = self.unzip_low_u8x32(a0, a1); - let lo_odd = self.unzip_high_u8x32(a0, a1); - let hi_even = self.unzip_low_u8x32(b0, b1); - let hi_odd = self.unzip_high_u8x32(b0, b1); + fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_even = self.unzip_low_u16x16(a0, a1); + let lo_odd = self.unzip_high_u16x16(a0, a1); + let hi_even = self.unzip_low_u16x16(b0, b1); + let hi_odd = self.unzip_high_u16x16(b0, b1); ( - self.combine_u8x32(lo_even, hi_even), - self.combine_u8x32(lo_odd, hi_odd), + self.combine_u16x16(lo_even, hi_even), + self.combine_u16x16(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_u8x64(b); - let (c0, c1) = self.split_u8x64(c); - self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) + fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_u16x32(b); + let (c0, c1) = self.split_u16x32(c); + self.combine_u16x16( + self.select_u16x16(a0, b0, c0), + self.select_u16x16(a1, b1, c1), + ) } #[inline(always)] - fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) + fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) } #[inline(always)] - fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) + fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) } #[inline(always)] - fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( - u8x32 { - val: crate::support::Aligned256(uint8x16x2_t(a.val.0.0, a.val.0.1)), + u16x16 { + val: crate::support::Aligned256(uint16x8x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - u8x32 { - val: crate::support::Aligned256(uint8x16x2_t(a.val.0.2, a.val.0.3)), + u16x16 { + val: crate::support::Aligned256(uint16x8x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - unsafe { vld4q_u8(src.as_ptr()).simd_into(self) } + fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { + unsafe { vld4q_u16(src.as_ptr()).simd_into(self) } } #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - unsafe { vst4q_u8(dest.as_mut_ptr(), a.into()) } + fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + unsafe { vst4q_u16(dest.as_mut_ptr(), a.into()) } } #[inline(always)] - fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { - let (a0, a1) = self.split_u8x64(a); + fn narrow_u16x32(self, a: u16x32) -> u8x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x32( + self.reinterpret_u8_u16x16(a0), + self.reinterpret_u8_u16x16(a1), + ) + } + #[inline(always)] + fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { + let (a0, a1) = self.split_u16x32(a); self.combine_u32x8( - self.reinterpret_u32_u8x32(a0), - self.reinterpret_u32_u8x32(a1), + self.reinterpret_u32_u16x16(a0), + self.reinterpret_u32_u16x16(a1), ) } #[inline(always)] - fn splat_mask8x64(self, val: bool) -> mask8x64 { - let half = self.splat_mask8x32(val); - self.combine_mask8x32(half, half) + fn splat_mask16x32(self, val: bool) -> mask16x32 { + let half = self.splat_mask16x16(val); + self.combine_mask16x16(half, half) } #[inline(always)] - fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { - mask8x64 { + fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { + mask16x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { - let lo = self.from_bitmask_mask8x32(bits); - let hi = self.from_bitmask_mask8x32(bits >> 32usize); - self.combine_mask8x32(lo, hi) + fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { + let lo = self.from_bitmask_mask16x16(bits); + let hi = self.from_bitmask_mask16x16(bits >> 16usize); + self.combine_mask16x16(lo, hi) } #[inline(always)] - fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { - let (lo, hi) = self.split_mask8x64(a); - let lo = self.to_bitmask_mask8x32(lo); - let hi = self.to_bitmask_mask8x32(hi); - lo | (hi << 32usize) + fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { + let (lo, hi) = self.split_mask16x32(a); + let lo = self.to_bitmask_mask16x16(lo); + let hi = self.to_bitmask_mask16x16(hi); + lo | (hi << 16usize) } #[inline(always)] - fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask16x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x32(lanes); } #[inline(always)] - fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) + fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) } #[inline(always)] - fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) + fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) } #[inline(always)] - fn not_mask8x64(self, a: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) + fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) } #[inline(always)] - fn select_mask8x64( + fn not_mask16x32(self, a: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + } + #[inline(always)] + fn select_mask16x32( self, - a: mask8x64, - b: mask8x64, - c: mask8x64, - ) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - let (c0, c1) = self.split_mask8x64(c); - self.combine_mask8x32( - self.select_mask8x32(a0, b0, c0), - self.select_mask8x32(a1, b1, c1), + a: mask16x32, + b: mask16x32, + c: mask16x32, + ) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + let (c0, c1) = self.split_mask16x32(c); + self.combine_mask16x16( + self.select_mask16x16(a0, b0, c0), + self.select_mask16x16(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) + fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16( + self.simd_eq_mask16x16(a0, b0), + self.simd_eq_mask16x16(a1, b1), + ) } #[inline(always)] - fn any_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) + fn any_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) } #[inline(always)] - fn all_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) + fn all_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) } #[inline(always)] - fn any_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) + fn any_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) } #[inline(always)] - fn all_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) + fn all_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) } #[inline(always)] - fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { ( - mask8x32 { - val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)), + mask16x16 { + val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - mask8x32 { - val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)), + mask16x16 { + val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn splat_i16x32(self, val: i16) -> i16x32 { - let half = self.splat_i16x16(val); - self.combine_i16x16(half, half) + fn splat_i32x16(self, val: i32) -> i32x16 { + let half = self.splat_i32x8(val); + self.combine_i32x8(half, half) } #[inline(always)] - fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { - i16x32 { + fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } - #[inline(always)] - fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { - i16x32 { + #[inline(always)] + fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { - i16x32 { + fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - if SHIFT >= 32usize { + fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + if SHIFT >= 16usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i16x32(a).val.0; - let b_bytes = self.cvt_to_bytes_i16x32(b).val.0; + let a_bytes = self.cvt_to_bytes_i32x16(a).val.0; + let b_bytes = self.cvt_to_bytes_i32x16(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT * 2usize; + let shift_bytes = SHIFT * 4usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -8078,286 +10294,282 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i16x32(u8x64 { + self.cvt_from_bytes_i32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i16x32( + fn slide_within_blocks_i32x16( self, - a: i16x32, - b: i16x32, - ) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.slide_within_blocks_i16x16::(a0, b0), - self.slide_within_blocks_i16x16::(a1, b1), + a: i32x16, + b: i32x16, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8( + self.slide_within_blocks_i32x8::(a0, b0), + self.slide_within_blocks_i32x8::(a1, b1), ) } #[inline(always)] - fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) } #[inline(always)] - fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) } #[inline(always)] - fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) } #[inline(always)] - fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) + fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) } #[inline(always)] - fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) + fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) } #[inline(always)] - fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) + fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) } #[inline(always)] - fn not_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) + fn not_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) } #[inline(always)] - fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) + fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) } #[inline(always)] - fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) + fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) } #[inline(always)] - fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) + fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) } #[inline(always)] - fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) + fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) } #[inline(always)] - fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) + fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) } #[inline(always)] - fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) + fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) } #[inline(always)] - fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) + fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) } #[inline(always)] - fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) + fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) } #[inline(always)] - fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) + fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) } #[inline(always)] - fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, _) = self.split_i16x32(a); - let (b0, _) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) + fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, _) = self.split_i32x16(a); + let (b0, _) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) } #[inline(always)] - fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (_, a1) = self.split_i16x32(a); - let (_, b1) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) + fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (_, a1) = self.split_i32x16(a); + let (_, b1) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) } #[inline(always)] - fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) + fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) } #[inline(always)] - fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.unzip_high_i16x16(a0, a1), - self.unzip_high_i16x16(b0, b1), - ) + fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) } #[inline(always)] - fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_lo = self.zip_low_i16x16(a0, b0); - let lo_hi = self.zip_high_i16x16(a0, b0); - let hi_lo = self.zip_low_i16x16(a1, b1); - let hi_hi = self.zip_high_i16x16(a1, b1); + fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_lo = self.zip_low_i32x8(a0, b0); + let lo_hi = self.zip_high_i32x8(a0, b0); + let hi_lo = self.zip_low_i32x8(a1, b1); + let hi_hi = self.zip_high_i32x8(a1, b1); ( - self.combine_i16x16(lo_lo, lo_hi), - self.combine_i16x16(hi_lo, hi_hi), + self.combine_i32x8(lo_lo, lo_hi), + self.combine_i32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_even = self.unzip_low_i16x16(a0, a1); - let lo_odd = self.unzip_high_i16x16(a0, a1); - let hi_even = self.unzip_low_i16x16(b0, b1); - let hi_odd = self.unzip_high_i16x16(b0, b1); + fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_even = self.unzip_low_i32x8(a0, a1); + let lo_odd = self.unzip_high_i32x8(a0, a1); + let hi_even = self.unzip_low_i32x8(b0, b1); + let hi_odd = self.unzip_high_i32x8(b0, b1); ( - self.combine_i16x16(lo_even, hi_even), - self.combine_i16x16(lo_odd, hi_odd), + self.combine_i32x8(lo_even, hi_even), + self.combine_i32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_i16x32(b); - let (c0, c1) = self.split_i16x32(c); - self.combine_i16x16( - self.select_i16x16(a0, b0, c0), - self.select_i16x16(a1, b1, c1), - ) + fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_i32x16(b); + let (c0, c1) = self.split_i32x16(c); + self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) } #[inline(always)] - fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) + fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) } #[inline(always)] - fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) + fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) } #[inline(always)] - fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { + fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( - i16x16 { - val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)), + i32x8 { + val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - i16x16 { - val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)), + i32x8 { + val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn neg_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) + fn neg_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { - let (a0, a1) = self.split_i16x32(a); - self.combine_u8x32( - self.reinterpret_u8_i16x16(a0), - self.reinterpret_u8_i16x16(a1), - ) + fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { + let (a0, a1) = self.split_i32x16(a); + self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) } #[inline(always)] - fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { - let (a0, a1) = self.split_i16x32(a); + fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { + let (a0, a1) = self.split_i32x16(a); self.combine_u32x8( - self.reinterpret_u32_i16x16(a0), - self.reinterpret_u32_i16x16(a1), + self.reinterpret_u32_i32x8(a0), + self.reinterpret_u32_i32x8(a1), ) } #[inline(always)] - fn splat_u16x32(self, val: u16) -> u16x32 { - let half = self.splat_u16x16(val); - self.combine_u16x16(half, half) + fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) } #[inline(always)] - fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { - u16x32 { + fn splat_u32x16(self, val: u32) -> u32x16 { + let half = self.splat_u32x8(val); + self.combine_u32x8(half, half) + } + #[inline(always)] + fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { - u16x32 { + fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { - u16x32 { + fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { + fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - if SHIFT >= 32usize { + fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + if SHIFT >= 16usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_u16x32(a).val.0; - let b_bytes = self.cvt_to_bytes_u16x32(b).val.0; + let a_bytes = self.cvt_to_bytes_u32x16(a).val.0; + let b_bytes = self.cvt_to_bytes_u32x16(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT * 2usize; + let shift_bytes = SHIFT * 4usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -8397,403 +10609,394 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_u16x32(u8x64 { + self.cvt_from_bytes_u32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u16x32( + fn slide_within_blocks_u32x16( self, - a: u16x32, - b: u16x32, - ) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.slide_within_blocks_u16x16::(a0, b0), - self.slide_within_blocks_u16x16::(a1, b1), + a: u32x16, + b: u32x16, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8( + self.slide_within_blocks_u32x8::(a0, b0), + self.slide_within_blocks_u32x8::(a1, b1), ) } #[inline(always)] - fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) + fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) } #[inline(always)] - fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) + fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) } #[inline(always)] - fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) + fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) } #[inline(always)] - fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) + fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) } #[inline(always)] - fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) + fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) } #[inline(always)] - fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) + fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) } #[inline(always)] - fn not_u16x32(self, a: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) + fn not_u32x16(self, a: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) } #[inline(always)] - fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) + fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) } #[inline(always)] - fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) + fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) } #[inline(always)] - fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) + fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) } #[inline(always)] - fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) + fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) } #[inline(always)] - fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) + fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) } #[inline(always)] - fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) + fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) } #[inline(always)] - fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) + fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) } #[inline(always)] - fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) + fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) } #[inline(always)] - fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) + fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) } #[inline(always)] - fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, _) = self.split_u16x32(a); - let (b0, _) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) + fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, _) = self.split_u32x16(a); + let (b0, _) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) } - #[inline(always)] - fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (_, a1) = self.split_u16x32(a); - let (_, b1) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) + #[inline(always)] + fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (_, a1) = self.split_u32x16(a); + let (_, b1) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) } #[inline(always)] - fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) + fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) } #[inline(always)] - fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.unzip_high_u16x16(a0, a1), - self.unzip_high_u16x16(b0, b1), - ) + fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) } #[inline(always)] - fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_lo = self.zip_low_u16x16(a0, b0); - let lo_hi = self.zip_high_u16x16(a0, b0); - let hi_lo = self.zip_low_u16x16(a1, b1); - let hi_hi = self.zip_high_u16x16(a1, b1); + fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_lo = self.zip_low_u32x8(a0, b0); + let lo_hi = self.zip_high_u32x8(a0, b0); + let hi_lo = self.zip_low_u32x8(a1, b1); + let hi_hi = self.zip_high_u32x8(a1, b1); ( - self.combine_u16x16(lo_lo, lo_hi), - self.combine_u16x16(hi_lo, hi_hi), + self.combine_u32x8(lo_lo, lo_hi), + self.combine_u32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_even = self.unzip_low_u16x16(a0, a1); - let lo_odd = self.unzip_high_u16x16(a0, a1); - let hi_even = self.unzip_low_u16x16(b0, b1); - let hi_odd = self.unzip_high_u16x16(b0, b1); + fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_even = self.unzip_low_u32x8(a0, a1); + let lo_odd = self.unzip_high_u32x8(a0, a1); + let hi_even = self.unzip_low_u32x8(b0, b1); + let hi_odd = self.unzip_high_u32x8(b0, b1); ( - self.combine_u16x16(lo_even, hi_even), - self.combine_u16x16(lo_odd, hi_odd), + self.combine_u32x8(lo_even, hi_even), + self.combine_u32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_u16x32(b); - let (c0, c1) = self.split_u16x32(c); - self.combine_u16x16( - self.select_u16x16(a0, b0, c0), - self.select_u16x16(a1, b1, c1), - ) + fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_u32x16(b); + let (c0, c1) = self.split_u32x16(c); + self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) } #[inline(always)] - fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) + fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) } #[inline(always)] - fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) } #[inline(always)] - fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( - u16x16 { - val: crate::support::Aligned256(uint16x8x2_t(a.val.0.0, a.val.0.1)), + u32x8 { + val: crate::support::Aligned256(uint32x4x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - u16x16 { - val: crate::support::Aligned256(uint16x8x2_t(a.val.0.2, a.val.0.3)), + u32x8 { + val: crate::support::Aligned256(uint32x4x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - unsafe { vld4q_u16(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - unsafe { vst4q_u16(dest.as_mut_ptr(), a.into()) } + fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { + unsafe { vld4q_u32(src.as_ptr()).simd_into(self) } } #[inline(always)] - fn narrow_u16x32(self, a: u16x32) -> u8x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + unsafe { vst4q_u32(dest.as_mut_ptr(), a.into()) } } #[inline(always)] - fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x32( - self.reinterpret_u8_u16x16(a0), - self.reinterpret_u8_u16x16(a1), - ) + fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) } #[inline(always)] - fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u32x8( - self.reinterpret_u32_u16x16(a0), - self.reinterpret_u32_u16x16(a1), - ) + fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) } #[inline(always)] - fn splat_mask16x32(self, val: bool) -> mask16x32 { - let half = self.splat_mask16x16(val); - self.combine_mask16x16(half, half) + fn splat_mask32x16(self, val: bool) -> mask32x16 { + let half = self.splat_mask32x8(val); + self.combine_mask32x8(half, half) } #[inline(always)] - fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { - mask16x32 { + fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { + mask32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { - let lo = self.from_bitmask_mask16x16(bits); - let hi = self.from_bitmask_mask16x16(bits >> 16usize); - self.combine_mask16x16(lo, hi) + fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { + let lo = self.from_bitmask_mask32x8(bits); + let hi = self.from_bitmask_mask32x8(bits >> 8usize); + self.combine_mask32x8(lo, hi) } #[inline(always)] - fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { - let (lo, hi) = self.split_mask16x32(a); - let lo = self.to_bitmask_mask16x16(lo); - let hi = self.to_bitmask_mask16x16(hi); - lo | (hi << 16usize) + fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { + let (lo, hi) = self.split_mask32x16(a); + let lo = self.to_bitmask_mask32x8(lo); + let hi = self.to_bitmask_mask32x8(hi); + lo | (hi << 8usize) } #[inline(always)] - fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask32x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x16(lanes); } #[inline(always)] - fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) + fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) } #[inline(always)] - fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) + fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) } #[inline(always)] - fn not_mask16x32(self, a: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) } #[inline(always)] - fn select_mask16x32( + fn not_mask32x16(self, a: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + } + #[inline(always)] + fn select_mask32x16( self, - a: mask16x32, - b: mask16x32, - c: mask16x32, - ) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - let (c0, c1) = self.split_mask16x32(c); - self.combine_mask16x16( - self.select_mask16x16(a0, b0, c0), - self.select_mask16x16(a1, b1, c1), + a: mask32x16, + b: mask32x16, + c: mask32x16, + ) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + let (c0, c1) = self.split_mask32x16(c); + self.combine_mask32x8( + self.select_mask32x8(a0, b0, c0), + self.select_mask32x8(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16( - self.simd_eq_mask16x16(a0, b0), - self.simd_eq_mask16x16(a1, b1), - ) + fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) } #[inline(always)] - fn any_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + fn any_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) } #[inline(always)] - fn all_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + fn all_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) } #[inline(always)] - fn any_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + fn any_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) } #[inline(always)] - fn all_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + fn all_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) } #[inline(always)] - fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { ( - mask16x16 { - val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)), + mask32x8 { + val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - mask16x16 { - val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)), + mask32x8 { + val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn splat_i32x16(self, val: i32) -> i32x16 { - let half = self.splat_i32x8(val); - self.combine_i32x8(half, half) + fn splat_f64x8(self, val: f64) -> f64x8 { + let half = self.splat_f64x4(val); + self.combine_f64x4(half, half) } #[inline(always)] - fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { - i32x16 { + fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { - i32x16 { + fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { - i32x16 { + fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - if SHIFT >= 16usize { + fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + if SHIFT >= 8usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_i32x16(a).val.0; - let b_bytes = self.cvt_to_bytes_i32x16(b).val.0; + let a_bytes = self.cvt_to_bytes_f64x8(a).val.0; + let b_bytes = self.cvt_to_bytes_f64x8(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT * 4usize; + let shift_bytes = SHIFT * 8usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -8833,282 +11036,323 @@ impl Simd for Neon { }, ) }; - self.cvt_from_bytes_i32x16(u8x64 { + self.cvt_from_bytes_f64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x16( + fn slide_within_blocks_f64x8( self, - a: i32x16, - b: i32x16, - ) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8( - self.slide_within_blocks_i32x8::(a0, b0), - self.slide_within_blocks_i32x8::(a1, b1), + a: f64x8, + b: f64x8, + ) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.slide_within_blocks_f64x4::(a0, b0), + self.slide_within_blocks_f64x4::(a1, b1), ) } #[inline(always)] - fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) + fn abs_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) } #[inline(always)] - fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) + fn neg_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) } #[inline(always)] - fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) + fn sqrt_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) } #[inline(always)] - fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) + fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.approximate_recip_f64x4(a0), + self.approximate_recip_f64x4(a1), + ) } #[inline(always)] - fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) + fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) } #[inline(always)] - fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) + fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) } #[inline(always)] - fn not_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) + fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) } #[inline(always)] - fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) + fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) } #[inline(always)] - fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) + fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) } #[inline(always)] - fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) + fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) } #[inline(always)] - fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) + fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) } #[inline(always)] - fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) + fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) } #[inline(always)] - fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) + fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) } #[inline(always)] - fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) + fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) } #[inline(always)] - fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) + fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, _) = self.split_f64x8(a); + let (b0, _) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) } #[inline(always)] - fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) + fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (_, a1) = self.split_f64x8(a); + let (_, b1) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) } #[inline(always)] - fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, _) = self.split_i32x16(a); - let (b0, _) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) + fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) + } + #[inline(always)] + fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_lo = self.zip_low_f64x4(a0, b0); + let lo_hi = self.zip_high_f64x4(a0, b0); + let hi_lo = self.zip_low_f64x4(a1, b1); + let hi_hi = self.zip_high_f64x4(a1, b1); + ( + self.combine_f64x4(lo_lo, lo_hi), + self.combine_f64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_even = self.unzip_low_f64x4(a0, a1); + let lo_odd = self.unzip_high_f64x4(a0, a1); + let hi_even = self.unzip_low_f64x4(b0, b1); + let hi_odd = self.unzip_high_f64x4(b0, b1); + ( + self.combine_f64x4(lo_even, hi_even), + self.combine_f64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) + } + #[inline(always)] + fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) + } + #[inline(always)] + fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.max_precise_f64x4(a0, b0), + self.max_precise_f64x4(a1, b1), + ) + } + #[inline(always)] + fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.min_precise_f64x4(a0, b0), + self.min_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (_, a1) = self.split_i32x16(a); - let (_, b1) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_add_f64x4(a0, b0, c0), + self.mul_add_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_sub_f64x4(a0, b0, c0), + self.mul_sub_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + fn floor_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) } #[inline(always)] - fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_lo = self.zip_low_i32x8(a0, b0); - let lo_hi = self.zip_high_i32x8(a0, b0); - let hi_lo = self.zip_low_i32x8(a1, b1); - let hi_hi = self.zip_high_i32x8(a1, b1); - ( - self.combine_i32x8(lo_lo, lo_hi), - self.combine_i32x8(hi_lo, hi_hi), - ) + fn ceil_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) } #[inline(always)] - fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_even = self.unzip_low_i32x8(a0, a1); - let lo_odd = self.unzip_high_i32x8(a0, a1); - let hi_even = self.unzip_low_i32x8(b0, b1); - let hi_odd = self.unzip_high_i32x8(b0, b1); - ( - self.combine_i32x8(lo_even, hi_even), - self.combine_i32x8(lo_odd, hi_odd), + fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.round_ties_even_f64x4(a0), + self.round_ties_even_f64x4(a1), ) } #[inline(always)] - fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_i32x16(b); - let (c0, c1) = self.split_i32x16(c); - self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) + fn fract_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) } #[inline(always)] - fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) + fn trunc_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) } #[inline(always)] - fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) } #[inline(always)] - fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { + fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { ( - i32x8 { - val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)), + f64x4 { + val: crate::support::Aligned256(float64x2x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - i32x8 { - val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)), + f64x4 { + val: crate::support::Aligned256(float64x2x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn neg_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u32x8( - self.reinterpret_u32_i32x8(a0), - self.reinterpret_u32_i32x8(a1), + fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f32x8( + self.reinterpret_f32_f64x4(a0), + self.reinterpret_f32_f64x4(a1), ) } #[inline(always)] - fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) - } - #[inline(always)] - fn splat_u32x16(self, val: u32) -> u32x16 { - let half = self.splat_u32x8(val); - self.combine_u32x8(half, half) + fn splat_i64x8(self, val: i64) -> i64x8 { + let half = self.splat_i64x4(val); + self.combine_i64x4(half, half) } #[inline(always)] - fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { - u32x16 { + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - if SHIFT >= 16usize { + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + if SHIFT >= 8usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_u32x16(a).val.0; - let b_bytes = self.cvt_to_bytes_u32x16(b).val.0; + let a_bytes = self.cvt_to_bytes_i64x8(a).val.0; + let b_bytes = self.cvt_to_bytes_i64x8(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; - let shift_bytes = SHIFT * 4usize; + let shift_bytes = SHIFT * 8usize; uint8x16x4_t( { let [lo, hi] = crate::support::cross_block_slide_blocks_at( @@ -9136,392 +11380,286 @@ impl Simd for Neon { shift_bytes, ); dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 3, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - ) - }; - self.cvt_from_bytes_u32x16(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) - } - #[inline(always)] - fn slide_within_blocks_u32x16( - self, - a: u32x16, - b: u32x16, - ) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8( - self.slide_within_blocks_u32x8::(a0, b0), - self.slide_within_blocks_u32x8::(a1, b1), - ) - } - #[inline(always)] - fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) - } - #[inline(always)] - fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) - } - #[inline(always)] - fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) - } - #[inline(always)] - fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) - } - #[inline(always)] - fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) - } - #[inline(always)] - fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) - } - #[inline(always)] - fn not_u32x16(self, a: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) - } - #[inline(always)] - fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) - } - #[inline(always)] - fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) - } - #[inline(always)] - fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) - } - #[inline(always)] - fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 3, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_i64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) + fn slide_within_blocks_i64x8( + self, + a: i64x8, + b: i64x8, + ) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4( + self.slide_within_blocks_i64x4::(a0, b0), + self.slide_within_blocks_i64x4::(a1, b1), + ) } #[inline(always)] - fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) } #[inline(always)] - fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) } #[inline(always)] - fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, _) = self.split_u32x16(a); - let (b0, _) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) } #[inline(always)] - fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (_, a1) = self.split_u32x16(a); - let (_, b1) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) } #[inline(always)] - fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) } #[inline(always)] - fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) } #[inline(always)] - fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_lo = self.zip_low_u32x8(a0, b0); - let lo_hi = self.zip_high_u32x8(a0, b0); - let hi_lo = self.zip_low_u32x8(a1, b1); - let hi_hi = self.zip_high_u32x8(a1, b1); - ( - self.combine_u32x8(lo_lo, lo_hi), - self.combine_u32x8(hi_lo, hi_hi), - ) + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) } #[inline(always)] - fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_even = self.unzip_low_u32x8(a0, a1); - let lo_odd = self.unzip_high_u32x8(a0, a1); - let hi_even = self.unzip_low_u32x8(b0, b1); - let hi_odd = self.unzip_high_u32x8(b0, b1); - ( - self.combine_u32x8(lo_even, hi_even), - self.combine_u32x8(lo_odd, hi_odd), - ) + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) } #[inline(always)] - fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_u32x16(b); - let (c0, c1) = self.split_u32x16(c); - self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) } #[inline(always)] - fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) } #[inline(always)] - fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) } #[inline(always)] - fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { - ( - u32x8 { - val: crate::support::Aligned256(uint32x4x2_t(a.val.0.0, a.val.0.1)), - simd: self, - }, - u32x8 { - val: crate::support::Aligned256(uint32x4x2_t(a.val.0.2, a.val.0.3)), - simd: self, - }, - ) + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - unsafe { vld4q_u32(src.as_ptr()).simd_into(self) } + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) } #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - unsafe { vst4q_u32(dest.as_mut_ptr(), a.into()) } + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) } #[inline(always)] - fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) } #[inline(always)] - fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) } #[inline(always)] - fn splat_mask32x16(self, val: bool) -> mask32x16 { - let half = self.splat_mask32x8(val); - self.combine_mask32x8(half, half) + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) } #[inline(always)] - fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { - mask32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) } #[inline(always)] - fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) } #[inline(always)] - fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { - let lo = self.from_bitmask_mask32x8(bits); - let hi = self.from_bitmask_mask32x8(bits >> 8usize); - self.combine_mask32x8(lo, hi) + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) } #[inline(always)] - fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { - let (lo, hi) = self.split_mask32x16(a); - let lo = self.to_bitmask_mask32x8(lo); - let hi = self.to_bitmask_mask32x8(hi); - lo | (hi << 8usize) + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) } #[inline(always)] - fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) } #[inline(always)] - fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) } #[inline(always)] - fn not_mask32x16(self, a: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) } #[inline(always)] - fn select_mask32x16( - self, - a: mask32x16, - b: mask32x16, - c: mask32x16, - ) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - let (c0, c1) = self.split_mask32x16(c); - self.combine_mask32x8( - self.select_mask32x8(a0, b0, c0), - self.select_mask32x8(a1, b1, c1), + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256(int64x2x2_t(a.val.0.0, a.val.0.1)), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256(int64x2x2_t(a.val.0.2, a.val.0.3)), + simd: self, + }, ) } #[inline(always)] - fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) - } - #[inline(always)] - fn any_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) - } - #[inline(always)] - fn all_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) - } - #[inline(always)] - fn any_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn all_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) } #[inline(always)] - fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { - ( - mask32x8 { - val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)), - simd: self, - }, - mask32x8 { - val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)), - simd: self, - }, + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), ) } #[inline(always)] - fn splat_f64x8(self, val: f64) -> f64x8 { - let half = self.splat_f64x4(val); - self.combine_f64x4(half, half) + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) } #[inline(always)] - fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { - f64x8 { + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { - f64x8 { + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { - crate::transmute::checked_transmute_copy::(&a.val.0) + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { - crate::transmute::checked_cast_ref::(&a.val.0) + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { - crate::transmute::checked_cast_mut::(&mut a.val.0) + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { - f64x8 { + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { if SHIFT >= 8usize { return b; } let result = { - let a_bytes = self.cvt_to_bytes_f64x8(a).val.0; - let b_bytes = self.cvt_to_bytes_f64x8(b).val.0; + let a_bytes = self.cvt_to_bytes_u64x8(a).val.0; + let b_bytes = self.cvt_to_bytes_u64x8(b).val.0; let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3]; let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3]; let shift_bytes = SHIFT * 8usize; @@ -9552,273 +11690,230 @@ impl Simd for Neon { shift_bytes, ); dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - { - let [lo, hi] = crate::support::cross_block_slide_blocks_at( - &a_blocks, - &b_blocks, - 3, - shift_bytes, - ); - dyn_vext_128(self, lo, hi, shift_bytes % 16) - }, - ) - }; - self.cvt_from_bytes_f64x8(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) - } - #[inline(always)] - fn slide_within_blocks_f64x8( - self, - a: f64x8, - b: f64x8, - ) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.slide_within_blocks_f64x4::(a0, b0), - self.slide_within_blocks_f64x4::(a1, b1), - ) - } - #[inline(always)] - fn abs_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) - } - #[inline(always)] - fn neg_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) - } - #[inline(always)] - fn sqrt_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) + }, + { + let [lo, hi] = crate::support::cross_block_slide_blocks_at( + &a_blocks, + &b_blocks, + 3, + shift_bytes, + ); + dyn_vext_128(self, lo, hi, shift_bytes % 16) + }, + ) + }; + self.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.approximate_recip_f64x4(a0), - self.approximate_recip_f64x4(a1), + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), ) } #[inline(always)] - fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) - } - #[inline(always)] - fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.add_u64x4(a0, b0), self.add_u64x4(a1, b1)) } #[inline(always)] - fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.sub_u64x4(a0, b0), self.sub_u64x4(a1, b1)) } #[inline(always)] - fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1)) } #[inline(always)] - fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.and_u64x4(a0, b0), self.and_u64x4(a1, b1)) } #[inline(always)] - fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.or_u64x4(a0, b0), self.or_u64x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.xor_u64x4(a0, b0), self.xor_u64x4(a1, b1)) } #[inline(always)] - fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) + fn not_u64x8(self, a: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.not_u64x4(a0), self.not_u64x4(a1)) } #[inline(always)] - fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shl_u64x4(a0, shift), self.shl_u64x4(a1, shift)) } #[inline(always)] - fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shlv_u64x4(a0, b0), self.shlv_u64x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, _) = self.split_f64x8(a); - let (b0, _) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shr_u64x4(a0, shift), self.shr_u64x4(a1, shift)) } #[inline(always)] - fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (_, a1) = self.split_f64x8(a); - let (_, b1) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shrv_u64x4(a0, b0), self.shrv_u64x4(a1, b1)) } #[inline(always)] - fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_eq_u64x4(a0, b0), self.simd_eq_u64x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_lt_u64x4(a0, b0), self.simd_lt_u64x4(a1, b1)) } #[inline(always)] - fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_lo = self.zip_low_f64x4(a0, b0); - let lo_hi = self.zip_high_f64x4(a0, b0); - let hi_lo = self.zip_low_f64x4(a1, b1); - let hi_hi = self.zip_high_f64x4(a1, b1); - ( - self.combine_f64x4(lo_lo, lo_hi), - self.combine_f64x4(hi_lo, hi_hi), - ) + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_le_u64x4(a0, b0), self.simd_le_u64x4(a1, b1)) } #[inline(always)] - fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_even = self.unzip_low_f64x4(a0, a1); - let lo_odd = self.unzip_high_f64x4(a0, a1); - let hi_even = self.unzip_low_f64x4(b0, b1); - let hi_odd = self.unzip_high_f64x4(b0, b1); - ( - self.combine_f64x4(lo_even, hi_even), - self.combine_f64x4(lo_odd, hi_odd), - ) + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_ge_u64x4(a0, b0), self.simd_ge_u64x4(a1, b1)) } #[inline(always)] - fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_gt_u64x4(a0, b0), self.simd_gt_u64x4(a1, b1)) } #[inline(always)] - fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, _) = self.split_u64x8(a); + let (b0, _) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a0, b0), self.zip_high_u64x4(a0, b0)) } #[inline(always)] - fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.max_precise_f64x4(a0, b0), - self.max_precise_f64x4(a1, b1), - ) + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (_, a1) = self.split_u64x8(a); + let (_, b1) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a1, b1), self.zip_high_u64x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.min_precise_f64x4(a0, b0), - self.min_precise_f64x4(a1, b1), - ) + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_low_u64x4(a0, a1), self.unzip_low_u64x4(b0, b1)) } #[inline(always)] - fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_add_f64x4(a0, b0, c0), - self.mul_add_f64x4(a1, b1, c1), - ) + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_high_u64x4(a0, a1), self.unzip_high_u64x4(b0, b1)) } #[inline(always)] - fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_sub_f64x4(a0, b0, c0), - self.mul_sub_f64x4(a1, b1, c1), + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_lo = self.zip_low_u64x4(a0, b0); + let lo_hi = self.zip_high_u64x4(a0, b0); + let hi_lo = self.zip_low_u64x4(a1, b1); + let hi_hi = self.zip_high_u64x4(a1, b1); + ( + self.combine_u64x4(lo_lo, lo_hi), + self.combine_u64x4(hi_lo, hi_hi), ) } #[inline(always)] - fn floor_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) - } - #[inline(always)] - fn ceil_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) - } - #[inline(always)] - fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.round_ties_even_f64x4(a0), - self.round_ties_even_f64x4(a1), + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_even = self.unzip_low_u64x4(a0, a1); + let lo_odd = self.unzip_high_u64x4(a0, a1); + let hi_even = self.unzip_low_u64x4(b0, b1); + let hi_odd = self.unzip_high_u64x4(b0, b1); + ( + self.combine_u64x4(lo_even, hi_even), + self.combine_u64x4(lo_odd, hi_odd), ) } #[inline(always)] - fn fract_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_u64x8(b); + let (c0, c1) = self.split_u64x8(c); + self.combine_u64x4(self.select_u64x4(a0, b0, c0), self.select_u64x4(a1, b1, c1)) } #[inline(always)] - fn trunc_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.min_u64x4(a0, b0), self.min_u64x4(a1, b1)) } #[inline(always)] - fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.max_u64x4(a0, b0), self.max_u64x4(a1, b1)) } #[inline(always)] - fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( - f64x4 { - val: crate::support::Aligned256(float64x2x2_t(a.val.0.0, a.val.0.1)), + u64x4 { + val: crate::support::Aligned256(uint64x2x2_t(a.val.0.0, a.val.0.1)), simd: self, }, - f64x4 { - val: crate::support::Aligned256(float64x2x2_t(a.val.0.2, a.val.0.3)), + u64x4 { + val: crate::support::Aligned256(uint64x2x2_t(a.val.0.2, a.val.0.3)), simd: self, }, ) } #[inline(always)] - fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f32x8( - self.reinterpret_f32_f64x4(a0), - self.reinterpret_f32_f64x4(a1), + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + unsafe { vld4q_u64(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + unsafe { vst4q_u64(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u8x32(self.reinterpret_u8_u64x4(a0), self.reinterpret_u8_u64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u32x8( + self.reinterpret_u32_u64x4(a0), + self.reinterpret_u32_u64x4(a1), ) } #[inline(always)] @@ -9851,6 +11946,17 @@ impl Simd for Neon { lo | (hi << 4usize) } #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask64x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x8(lanes); + } + #[inline(always)] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { let (a0, a1) = self.split_mask64x8(a); let (b0, b1) = self.split_mask64x8(b); @@ -10093,6 +12199,36 @@ impl From> for float64x2_t { crate::transmute::checked_transmute_copy(&value.val) } } +impl SimdFrom for i64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: int64x2_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for int64x2_t { + #[inline(always)] + fn from(value: i64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom for u64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: uint64x2_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for uint64x2_t { + #[inline(always)] + fn from(value: u64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} impl SimdFrom for mask64x2 { #[inline(always)] fn simd_from(simd: S, arch: int64x2_t) -> Self { @@ -10273,6 +12409,36 @@ impl From> for float64x2x2_t { crate::transmute::checked_transmute_copy(&value.val) } } +impl SimdFrom for i64x4 { + #[inline(always)] + fn simd_from(simd: S, arch: int64x2x2_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for int64x2x2_t { + #[inline(always)] + fn from(value: i64x4) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom for u64x4 { + #[inline(always)] + fn simd_from(simd: S, arch: uint64x2x2_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for uint64x2x2_t { + #[inline(always)] + fn from(value: u64x4) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} impl SimdFrom for mask64x4 { #[inline(always)] fn simd_from(simd: S, arch: int64x2x2_t) -> Self { @@ -10453,6 +12619,36 @@ impl From> for float64x2x4_t { crate::transmute::checked_transmute_copy(&value.val) } } +impl SimdFrom for i64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: int64x2x4_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for int64x2x4_t { + #[inline(always)] + fn from(value: i64x8) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom for u64x8 { + #[inline(always)] + fn simd_from(simd: S, arch: uint64x2x4_t) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for uint64x2x4_t { + #[inline(always)] + fn from(value: u64x8) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} impl SimdFrom for mask64x8 { #[inline(always)] fn simd_from(simd: S, arch: int64x2x4_t) -> Self { diff --git a/fearless_simd/src/generated/ops.rs b/fearless_simd/src/generated/ops.rs index b05d99186..53c6f9e92 100644 --- a/fearless_simd/src/generated/ops.rs +++ b/fearless_simd/src/generated/ops.rs @@ -6,9 +6,9 @@ use crate::{Simd, SimdInto}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; impl core::ops::Neg for f32x4 { type Output = Self; @@ -2145,4380 +2145,6060 @@ impl core::ops::Div> for f64 { rhs.simd.div_f64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for mask64x2 { +impl core::ops::Neg for i64x2 { type Output = Self; - #[doc = "Compute the logical AND of two masks."] + #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask64x2(self, rhs) + fn neg(self) -> Self::Output { + self.simd.neg_i64x2(self) } } -impl core::ops::BitAndAssign for mask64x2 { - #[doc = "Compute the logical AND of two masks."] +impl core::ops::Add for i64x2 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask64x2(*self, rhs); + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_i64x2(self, rhs) } } -impl core::ops::BitOr for mask64x2 { +impl core::ops::AddAssign for i64x2 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_i64x2(*self, rhs); + } +} +impl core::ops::Add for i64x2 { type Output = Self; - #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask64x2(self, rhs) + fn add(self, rhs: i64) -> Self::Output { + self.simd.add_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for mask64x2 { - #[doc = "Compute the logical OR of two masks."] +impl core::ops::AddAssign for i64x2 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask64x2(*self, rhs); + fn add_assign(&mut self, rhs: i64) { + *self = self.simd.add_i64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for mask64x2 { +impl core::ops::Add> for i64 { + type Output = i64x2; + #[inline(always)] + fn add(self, rhs: i64x2) -> Self::Output { + rhs.simd.add_i64x2(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for i64x2 { type Output = Self; - #[doc = "Compute the logical XOR of two masks."] + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask64x2(self, rhs) + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_i64x2(self, rhs) } } -impl core::ops::BitXorAssign for mask64x2 { - #[doc = "Compute the logical XOR of two masks."] +impl core::ops::SubAssign for i64x2 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask64x2(*self, rhs); + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_i64x2(*self, rhs); } } -impl core::ops::Not for mask64x2 { +impl core::ops::Sub for i64x2 { type Output = Self; - #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask64x2(self) + fn sub(self, rhs: i64) -> Self::Output { + self.simd.sub_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::Neg for f32x8 { +impl core::ops::SubAssign for i64x2 { + #[inline(always)] + fn sub_assign(&mut self, rhs: i64) { + *self = self.simd.sub_i64x2(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for i64 { + type Output = i64x2; + #[inline(always)] + fn sub(self, rhs: i64x2) -> Self::Output { + rhs.simd.sub_i64x2(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for i64x2 { type Output = Self; - #[doc = "Negate each element of the vector."] + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_f32x8(self) + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_i64x2(self, rhs) } } -impl core::ops::Add for f32x8 { +impl core::ops::MulAssign for i64x2 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_i64x2(*self, rhs); + } +} +impl core::ops::Mul for i64x2 { type Output = Self; - #[doc = "Add two vectors element-wise."] #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_f32x8(self, rhs) + fn mul(self, rhs: i64) -> Self::Output { + self.simd.mul_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for f32x8 { - #[doc = "Add two vectors element-wise."] +impl core::ops::MulAssign for i64x2 { #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_f32x8(*self, rhs); + fn mul_assign(&mut self, rhs: i64) { + *self = self.simd.mul_i64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add for f32x8 { +impl core::ops::Mul> for i64 { + type Output = i64x2; + #[inline(always)] + fn mul(self, rhs: i64x2) -> Self::Output { + rhs.simd.mul_i64x2(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for i64x2 { type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn add(self, rhs: f32) -> Self::Output { - self.simd.add_f32x8(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_i64x2(self, rhs) } } -impl core::ops::AddAssign for f32x8 { +impl core::ops::BitAndAssign for i64x2 { + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn add_assign(&mut self, rhs: f32) { - *self = self.simd.add_f32x8(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_i64x2(*self, rhs); } } -impl core::ops::Add> for f32 { - type Output = f32x8; +impl core::ops::BitAnd for i64x2 { + type Output = Self; #[inline(always)] - fn add(self, rhs: f32x8) -> Self::Output { - rhs.simd.add_f32x8(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i64) -> Self::Output { + self.simd.and_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::Sub for f32x8 { +impl core::ops::BitAndAssign for i64x2 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: i64) { + *self = self.simd.and_i64x2(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for i64 { + type Output = i64x2; + #[inline(always)] + fn bitand(self, rhs: i64x2) -> Self::Output { + rhs.simd.and_i64x2(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for i64x2 { type Output = Self; - #[doc = "Subtract two vectors element-wise."] + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_f32x8(self, rhs) + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_i64x2(self, rhs) } } -impl core::ops::SubAssign for f32x8 { - #[doc = "Subtract two vectors element-wise."] +impl core::ops::BitOrAssign for i64x2 { + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_f32x8(*self, rhs); + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_i64x2(*self, rhs); } } -impl core::ops::Sub for f32x8 { +impl core::ops::BitOr for i64x2 { type Output = Self; #[inline(always)] - fn sub(self, rhs: f32) -> Self::Output { - self.simd.sub_f32x8(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i64) -> Self::Output { + self.simd.or_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for f32x8 { +impl core::ops::BitOrAssign for i64x2 { #[inline(always)] - fn sub_assign(&mut self, rhs: f32) { - *self = self.simd.sub_f32x8(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i64) { + *self = self.simd.or_i64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for f32 { - type Output = f32x8; +impl core::ops::BitOr> for i64 { + type Output = i64x2; #[inline(always)] - fn sub(self, rhs: f32x8) -> Self::Output { - rhs.simd.sub_f32x8(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i64x2) -> Self::Output { + rhs.simd.or_i64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for f32x8 { +impl core::ops::BitXor for i64x2 { type Output = Self; - #[doc = "Multiply two vectors element-wise."] + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_f32x8(self, rhs) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_i64x2(self, rhs) } } -impl core::ops::MulAssign for f32x8 { - #[doc = "Multiply two vectors element-wise."] +impl core::ops::BitXorAssign for i64x2 { + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_f32x8(*self, rhs); + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_i64x2(*self, rhs); } } -impl core::ops::Mul for f32x8 { +impl core::ops::BitXor for i64x2 { type Output = Self; #[inline(always)] - fn mul(self, rhs: f32) -> Self::Output { - self.simd.mul_f32x8(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i64) -> Self::Output { + self.simd.xor_i64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for f32x8 { +impl core::ops::BitXorAssign for i64x2 { #[inline(always)] - fn mul_assign(&mut self, rhs: f32) { - *self = self.simd.mul_f32x8(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i64) { + *self = self.simd.xor_i64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for f32 { - type Output = f32x8; +impl core::ops::BitXor> for i64 { + type Output = i64x2; #[inline(always)] - fn mul(self, rhs: f32x8) -> Self::Output { - rhs.simd.mul_f32x8(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i64x2) -> Self::Output { + rhs.simd.xor_i64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Div for f32x8 { +impl core::ops::Not for i64x2 { type Output = Self; - #[doc = "Divide two vectors element-wise."] + #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] - fn div(self, rhs: Self) -> Self::Output { - self.simd.div_f32x8(self, rhs) + fn not(self) -> Self::Output { + self.simd.not_i64x2(self) } } -impl core::ops::DivAssign for f32x8 { - #[doc = "Divide two vectors element-wise."] +impl core::ops::Shl for i64x2 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] - fn div_assign(&mut self, rhs: Self) { - *self = self.simd.div_f32x8(*self, rhs); + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_i64x2(self, rhs) } } -impl core::ops::Div for f32x8 { +impl core::ops::ShlAssign for i64x2 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_i64x2(*self, rhs); + } +} +impl core::ops::Shl for i64x2 { type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn div(self, rhs: f32) -> Self::Output { - self.simd.div_f32x8(self, rhs.simd_into(self.simd)) + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_i64x2(self, rhs) } } -impl core::ops::DivAssign for f32x8 { +impl core::ops::ShlAssign for i64x2 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn div_assign(&mut self, rhs: f32) { - *self = self.simd.div_f32x8(*self, rhs.simd_into(self.simd)); + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_i64x2(*self, rhs); } } -impl core::ops::Div> for f32 { - type Output = f32x8; +impl core::ops::Shr for i64x2 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] - fn div(self, rhs: f32x8) -> Self::Output { - rhs.simd.div_f32x8(self.simd_into(rhs.simd), rhs) + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_i64x2(self, rhs) } } -impl core::ops::Neg for i8x32 { +impl core::ops::ShrAssign for i64x2 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_i64x2(*self, rhs); + } +} +impl core::ops::Shr for i64x2 { type Output = Self; - #[doc = "Negate each element of the vector, wrapping on overflow."] + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_i8x32(self) + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_i64x2(self, rhs) } } -impl core::ops::Add for i8x32 { +impl core::ops::ShrAssign for i64x2 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_i64x2(*self, rhs); + } +} +impl core::ops::Add for u64x2 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i8x32(self, rhs) + self.simd.add_u64x2(self, rhs) } } -impl core::ops::AddAssign for i8x32 { +impl core::ops::AddAssign for u64x2 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i8x32(*self, rhs); + *self = self.simd.add_u64x2(*self, rhs); } } -impl core::ops::Add for i8x32 { +impl core::ops::Add for u64x2 { type Output = Self; #[inline(always)] - fn add(self, rhs: i8) -> Self::Output { - self.simd.add_i8x32(self, rhs.simd_into(self.simd)) + fn add(self, rhs: u64) -> Self::Output { + self.simd.add_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for i8x32 { +impl core::ops::AddAssign for u64x2 { #[inline(always)] - fn add_assign(&mut self, rhs: i8) { - *self = self.simd.add_i8x32(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: u64) { + *self = self.simd.add_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for i8 { - type Output = i8x32; +impl core::ops::Add> for u64 { + type Output = u64x2; #[inline(always)] - fn add(self, rhs: i8x32) -> Self::Output { - rhs.simd.add_i8x32(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: u64x2) -> Self::Output { + rhs.simd.add_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for i8x32 { +impl core::ops::Sub for u64x2 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i8x32(self, rhs) + self.simd.sub_u64x2(self, rhs) } } -impl core::ops::SubAssign for i8x32 { +impl core::ops::SubAssign for u64x2 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i8x32(*self, rhs); + *self = self.simd.sub_u64x2(*self, rhs); } } -impl core::ops::Sub for i8x32 { +impl core::ops::Sub for u64x2 { type Output = Self; #[inline(always)] - fn sub(self, rhs: i8) -> Self::Output { - self.simd.sub_i8x32(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: u64) -> Self::Output { + self.simd.sub_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for i8x32 { +impl core::ops::SubAssign for u64x2 { #[inline(always)] - fn sub_assign(&mut self, rhs: i8) { - *self = self.simd.sub_i8x32(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: u64) { + *self = self.simd.sub_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for i8 { - type Output = i8x32; +impl core::ops::Sub> for u64 { + type Output = u64x2; #[inline(always)] - fn sub(self, rhs: i8x32) -> Self::Output { - rhs.simd.sub_i8x32(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: u64x2) -> Self::Output { + rhs.simd.sub_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for i8x32 { +impl core::ops::Mul for u64x2 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i8x32(self, rhs) + self.simd.mul_u64x2(self, rhs) } } -impl core::ops::MulAssign for i8x32 { +impl core::ops::MulAssign for u64x2 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i8x32(*self, rhs); + *self = self.simd.mul_u64x2(*self, rhs); } } -impl core::ops::Mul for i8x32 { +impl core::ops::Mul for u64x2 { type Output = Self; #[inline(always)] - fn mul(self, rhs: i8) -> Self::Output { - self.simd.mul_i8x32(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: u64) -> Self::Output { + self.simd.mul_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for i8x32 { +impl core::ops::MulAssign for u64x2 { #[inline(always)] - fn mul_assign(&mut self, rhs: i8) { - *self = self.simd.mul_i8x32(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: u64) { + *self = self.simd.mul_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for i8 { - type Output = i8x32; +impl core::ops::Mul> for u64 { + type Output = u64x2; #[inline(always)] - fn mul(self, rhs: i8x32) -> Self::Output { - rhs.simd.mul_i8x32(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: u64x2) -> Self::Output { + rhs.simd.mul_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for i8x32 { +impl core::ops::BitAnd for u64x2 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i8x32(self, rhs) + self.simd.and_u64x2(self, rhs) } } -impl core::ops::BitAndAssign for i8x32 { +impl core::ops::BitAndAssign for u64x2 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i8x32(*self, rhs); + *self = self.simd.and_u64x2(*self, rhs); } } -impl core::ops::BitAnd for i8x32 { +impl core::ops::BitAnd for u64x2 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: i8) -> Self::Output { - self.simd.and_i8x32(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: u64) -> Self::Output { + self.simd.and_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for i8x32 { +impl core::ops::BitAndAssign for u64x2 { #[inline(always)] - fn bitand_assign(&mut self, rhs: i8) { - *self = self.simd.and_i8x32(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: u64) { + *self = self.simd.and_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for i8 { - type Output = i8x32; +impl core::ops::BitAnd> for u64 { + type Output = u64x2; #[inline(always)] - fn bitand(self, rhs: i8x32) -> Self::Output { - rhs.simd.and_i8x32(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u64x2) -> Self::Output { + rhs.simd.and_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for i8x32 { +impl core::ops::BitOr for u64x2 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i8x32(self, rhs) + self.simd.or_u64x2(self, rhs) } } -impl core::ops::BitOrAssign for i8x32 { +impl core::ops::BitOrAssign for u64x2 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i8x32(*self, rhs); + *self = self.simd.or_u64x2(*self, rhs); } } -impl core::ops::BitOr for i8x32 { +impl core::ops::BitOr for u64x2 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: i8) -> Self::Output { - self.simd.or_i8x32(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: u64) -> Self::Output { + self.simd.or_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i8x32 { +impl core::ops::BitOrAssign for u64x2 { #[inline(always)] - fn bitor_assign(&mut self, rhs: i8) { - *self = self.simd.or_i8x32(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: u64) { + *self = self.simd.or_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for i8 { - type Output = i8x32; +impl core::ops::BitOr> for u64 { + type Output = u64x2; #[inline(always)] - fn bitor(self, rhs: i8x32) -> Self::Output { - rhs.simd.or_i8x32(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: u64x2) -> Self::Output { + rhs.simd.or_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for i8x32 { +impl core::ops::BitXor for u64x2 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i8x32(self, rhs) + self.simd.xor_u64x2(self, rhs) } } -impl core::ops::BitXorAssign for i8x32 { +impl core::ops::BitXorAssign for u64x2 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i8x32(*self, rhs); + *self = self.simd.xor_u64x2(*self, rhs); } } -impl core::ops::BitXor for i8x32 { +impl core::ops::BitXor for u64x2 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: i8) -> Self::Output { - self.simd.xor_i8x32(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: u64) -> Self::Output { + self.simd.xor_u64x2(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i8x32 { +impl core::ops::BitXorAssign for u64x2 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: i8) { - *self = self.simd.xor_i8x32(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: u64) { + *self = self.simd.xor_u64x2(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for i8 { - type Output = i8x32; +impl core::ops::BitXor> for u64 { + type Output = u64x2; #[inline(always)] - fn bitxor(self, rhs: i8x32) -> Self::Output { - rhs.simd.xor_i8x32(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: u64x2) -> Self::Output { + rhs.simd.xor_u64x2(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for i8x32 { +impl core::ops::Not for u64x2 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_i8x32(self) + self.simd.not_u64x2(self) } } -impl core::ops::Shl for i8x32 { +impl core::ops::Shl for u64x2 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i8x32(self, rhs) + self.simd.shl_u64x2(self, rhs) } } -impl core::ops::ShlAssign for i8x32 { +impl core::ops::ShlAssign for u64x2 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i8x32(*self, rhs); + *self = self.simd.shl_u64x2(*self, rhs); } } -impl core::ops::Shl for i8x32 { +impl core::ops::Shl for u64x2 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i8x32(self, rhs) + self.simd.shlv_u64x2(self, rhs) } } -impl core::ops::ShlAssign for i8x32 { +impl core::ops::ShlAssign for u64x2 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i8x32(*self, rhs); + *self = self.simd.shlv_u64x2(*self, rhs); } } -impl core::ops::Shr for i8x32 { +impl core::ops::Shr for u64x2 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i8x32(self, rhs) + self.simd.shr_u64x2(self, rhs) } } -impl core::ops::ShrAssign for i8x32 { +impl core::ops::ShrAssign for u64x2 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i8x32(*self, rhs); + *self = self.simd.shr_u64x2(*self, rhs); } } -impl core::ops::Shr for i8x32 { +impl core::ops::Shr for u64x2 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i8x32(self, rhs) + self.simd.shrv_u64x2(self, rhs) } } -impl core::ops::ShrAssign for i8x32 { +impl core::ops::ShrAssign for u64x2 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i8x32(*self, rhs); + *self = self.simd.shrv_u64x2(*self, rhs); } } -impl core::ops::Add for u8x32 { +impl core::ops::BitAnd for mask64x2 { type Output = Self; - #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u8x32(self, rhs) + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask64x2(self, rhs) } } -impl core::ops::AddAssign for u8x32 { - #[doc = "Add two vectors element-wise, wrapping on overflow."] +impl core::ops::BitAndAssign for mask64x2 { + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u8x32(*self, rhs); + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask64x2(*self, rhs); } } -impl core::ops::Add for u8x32 { +impl core::ops::BitOr for mask64x2 { type Output = Self; + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn add(self, rhs: u8) -> Self::Output { - self.simd.add_u8x32(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::AddAssign for u8x32 { - #[inline(always)] - fn add_assign(&mut self, rhs: u8) { - *self = self.simd.add_u8x32(*self, rhs.simd_into(self.simd)); + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask64x2(self, rhs) } } -impl core::ops::Add> for u8 { - type Output = u8x32; +impl core::ops::BitOrAssign for mask64x2 { + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn add(self, rhs: u8x32) -> Self::Output { - rhs.simd.add_u8x32(self.simd_into(rhs.simd), rhs) + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask64x2(*self, rhs); } } -impl core::ops::Sub for u8x32 { +impl core::ops::BitXor for mask64x2 { type Output = Self; - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u8x32(self, rhs) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask64x2(self, rhs) } } -impl core::ops::SubAssign for u8x32 { - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] +impl core::ops::BitXorAssign for mask64x2 { + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u8x32(*self, rhs); + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask64x2(*self, rhs); } } -impl core::ops::Sub for u8x32 { +impl core::ops::Not for mask64x2 { type Output = Self; + #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn sub(self, rhs: u8) -> Self::Output { - self.simd.sub_u8x32(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::SubAssign for u8x32 { - #[inline(always)] - fn sub_assign(&mut self, rhs: u8) { - *self = self.simd.sub_u8x32(*self, rhs.simd_into(self.simd)); + fn not(self) -> Self::Output { + self.simd.not_mask64x2(self) } } -impl core::ops::Sub> for u8 { - type Output = u8x32; +impl core::ops::Neg for f32x8 { + type Output = Self; + #[doc = "Negate each element of the vector."] #[inline(always)] - fn sub(self, rhs: u8x32) -> Self::Output { - rhs.simd.sub_u8x32(self.simd_into(rhs.simd), rhs) + fn neg(self) -> Self::Output { + self.simd.neg_f32x8(self) } } -impl core::ops::Mul for u8x32 { +impl core::ops::Add for f32x8 { type Output = Self; - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u8x32(self, rhs) + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_f32x8(self, rhs) } } -impl core::ops::MulAssign for u8x32 { - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] +impl core::ops::AddAssign for f32x8 { + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u8x32(*self, rhs); + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_f32x8(*self, rhs); } } -impl core::ops::Mul for u8x32 { +impl core::ops::Add for f32x8 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u8) -> Self::Output { - self.simd.mul_u8x32(self, rhs.simd_into(self.simd)) + fn add(self, rhs: f32) -> Self::Output { + self.simd.add_f32x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u8x32 { +impl core::ops::AddAssign for f32x8 { #[inline(always)] - fn mul_assign(&mut self, rhs: u8) { - *self = self.simd.mul_u8x32(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: f32) { + *self = self.simd.add_f32x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for u8 { - type Output = u8x32; +impl core::ops::Add> for f32 { + type Output = f32x8; #[inline(always)] - fn mul(self, rhs: u8x32) -> Self::Output { - rhs.simd.mul_u8x32(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: f32x8) -> Self::Output { + rhs.simd.add_f32x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u8x32 { +impl core::ops::Sub for f32x8 { type Output = Self; - #[doc = "Compute the bitwise AND of two vectors."] + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u8x32(self, rhs) + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_f32x8(self, rhs) } } -impl core::ops::BitAndAssign for u8x32 { - #[doc = "Compute the bitwise AND of two vectors."] +impl core::ops::SubAssign for f32x8 { + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u8x32(*self, rhs); + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_f32x8(*self, rhs); } } -impl core::ops::BitAnd for u8x32 { +impl core::ops::Sub for f32x8 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u8) -> Self::Output { - self.simd.and_u8x32(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: f32) -> Self::Output { + self.simd.sub_f32x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u8x32 { +impl core::ops::SubAssign for f32x8 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u8) { - *self = self.simd.and_u8x32(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: f32) { + *self = self.simd.sub_f32x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u8 { - type Output = u8x32; +impl core::ops::Sub> for f32 { + type Output = f32x8; #[inline(always)] - fn bitand(self, rhs: u8x32) -> Self::Output { - rhs.simd.and_u8x32(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: f32x8) -> Self::Output { + rhs.simd.sub_f32x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u8x32 { +impl core::ops::Mul for f32x8 { type Output = Self; - #[doc = "Compute the bitwise OR of two vectors."] + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u8x32(self, rhs) + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_f32x8(self, rhs) } } -impl core::ops::BitOrAssign for u8x32 { - #[doc = "Compute the bitwise OR of two vectors."] +impl core::ops::MulAssign for f32x8 { + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u8x32(*self, rhs); + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_f32x8(*self, rhs); } } -impl core::ops::BitOr for u8x32 { +impl core::ops::Mul for f32x8 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u8) -> Self::Output { - self.simd.or_u8x32(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: f32) -> Self::Output { + self.simd.mul_f32x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u8x32 { +impl core::ops::MulAssign for f32x8 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u8) { - *self = self.simd.or_u8x32(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: f32) { + *self = self.simd.mul_f32x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u8 { - type Output = u8x32; +impl core::ops::Mul> for f32 { + type Output = f32x8; #[inline(always)] - fn bitor(self, rhs: u8x32) -> Self::Output { - rhs.simd.or_u8x32(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: f32x8) -> Self::Output { + rhs.simd.mul_f32x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u8x32 { +impl core::ops::Div for f32x8 { type Output = Self; - #[doc = "Compute the bitwise XOR of two vectors."] + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u8x32(self, rhs) + fn div(self, rhs: Self) -> Self::Output { + self.simd.div_f32x8(self, rhs) } } -impl core::ops::BitXorAssign for u8x32 { - #[doc = "Compute the bitwise XOR of two vectors."] +impl core::ops::DivAssign for f32x8 { + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u8x32(*self, rhs); + fn div_assign(&mut self, rhs: Self) { + *self = self.simd.div_f32x8(*self, rhs); } } -impl core::ops::BitXor for u8x32 { +impl core::ops::Div for f32x8 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u8) -> Self::Output { - self.simd.xor_u8x32(self, rhs.simd_into(self.simd)) + fn div(self, rhs: f32) -> Self::Output { + self.simd.div_f32x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u8x32 { +impl core::ops::DivAssign for f32x8 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u8) { - *self = self.simd.xor_u8x32(*self, rhs.simd_into(self.simd)); + fn div_assign(&mut self, rhs: f32) { + *self = self.simd.div_f32x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u8 { - type Output = u8x32; +impl core::ops::Div> for f32 { + type Output = f32x8; #[inline(always)] - fn bitxor(self, rhs: u8x32) -> Self::Output { - rhs.simd.xor_u8x32(self.simd_into(rhs.simd), rhs) + fn div(self, rhs: f32x8) -> Self::Output { + rhs.simd.div_f32x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u8x32 { +impl core::ops::Neg for i8x32 { type Output = Self; - #[doc = "Compute the bitwise NOT of the vector."] + #[doc = "Negate each element of the vector, wrapping on overflow."] + #[inline(always)] + fn neg(self) -> Self::Output { + self.simd.neg_i8x32(self) + } +} +impl core::ops::Add for i8x32 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_i8x32(self, rhs) + } +} +impl core::ops::AddAssign for i8x32 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_i8x32(*self, rhs); + } +} +impl core::ops::Add for i8x32 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: i8) -> Self::Output { + self.simd.add_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for i8x32 { + #[inline(always)] + fn add_assign(&mut self, rhs: i8) { + *self = self.simd.add_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for i8 { + type Output = i8x32; + #[inline(always)] + fn add(self, rhs: i8x32) -> Self::Output { + rhs.simd.add_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for i8x32 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_i8x32(self, rhs) + } +} +impl core::ops::SubAssign for i8x32 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_i8x32(*self, rhs); + } +} +impl core::ops::Sub for i8x32 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: i8) -> Self::Output { + self.simd.sub_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for i8x32 { + #[inline(always)] + fn sub_assign(&mut self, rhs: i8) { + *self = self.simd.sub_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for i8 { + type Output = i8x32; + #[inline(always)] + fn sub(self, rhs: i8x32) -> Self::Output { + rhs.simd.sub_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for i8x32 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_i8x32(self, rhs) + } +} +impl core::ops::MulAssign for i8x32 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_i8x32(*self, rhs); + } +} +impl core::ops::Mul for i8x32 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: i8) -> Self::Output { + self.simd.mul_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for i8x32 { + #[inline(always)] + fn mul_assign(&mut self, rhs: i8) { + *self = self.simd.mul_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for i8 { + type Output = i8x32; + #[inline(always)] + fn mul(self, rhs: i8x32) -> Self::Output { + rhs.simd.mul_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for i8x32 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_i8x32(self, rhs) + } +} +impl core::ops::BitAndAssign for i8x32 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_i8x32(*self, rhs); + } +} +impl core::ops::BitAnd for i8x32 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: i8) -> Self::Output { + self.simd.and_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for i8x32 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: i8) { + *self = self.simd.and_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for i8 { + type Output = i8x32; + #[inline(always)] + fn bitand(self, rhs: i8x32) -> Self::Output { + rhs.simd.and_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for i8x32 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_i8x32(self, rhs) + } +} +impl core::ops::BitOrAssign for i8x32 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_i8x32(*self, rhs); + } +} +impl core::ops::BitOr for i8x32 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: i8) -> Self::Output { + self.simd.or_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for i8x32 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: i8) { + *self = self.simd.or_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for i8 { + type Output = i8x32; + #[inline(always)] + fn bitor(self, rhs: i8x32) -> Self::Output { + rhs.simd.or_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for i8x32 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_i8x32(self, rhs) + } +} +impl core::ops::BitXorAssign for i8x32 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_i8x32(*self, rhs); + } +} +impl core::ops::BitXor for i8x32 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: i8) -> Self::Output { + self.simd.xor_i8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for i8x32 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: i8) { + *self = self.simd.xor_i8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for i8 { + type Output = i8x32; + #[inline(always)] + fn bitxor(self, rhs: i8x32) -> Self::Output { + rhs.simd.xor_i8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for i8x32 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_i8x32(self) + } +} +impl core::ops::Shl for i8x32 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_i8x32(self, rhs) + } +} +impl core::ops::ShlAssign for i8x32 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_i8x32(*self, rhs); + } +} +impl core::ops::Shl for i8x32 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_i8x32(self, rhs) + } +} +impl core::ops::ShlAssign for i8x32 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_i8x32(*self, rhs); + } +} +impl core::ops::Shr for i8x32 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_i8x32(self, rhs) + } +} +impl core::ops::ShrAssign for i8x32 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_i8x32(*self, rhs); + } +} +impl core::ops::Shr for i8x32 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_i8x32(self, rhs) + } +} +impl core::ops::ShrAssign for i8x32 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_i8x32(*self, rhs); + } +} +impl core::ops::Add for u8x32 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_u8x32(self, rhs) + } +} +impl core::ops::AddAssign for u8x32 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_u8x32(*self, rhs); + } +} +impl core::ops::Add for u8x32 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: u8) -> Self::Output { + self.simd.add_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for u8x32 { + #[inline(always)] + fn add_assign(&mut self, rhs: u8) { + *self = self.simd.add_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for u8 { + type Output = u8x32; + #[inline(always)] + fn add(self, rhs: u8x32) -> Self::Output { + rhs.simd.add_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for u8x32 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_u8x32(self, rhs) + } +} +impl core::ops::SubAssign for u8x32 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_u8x32(*self, rhs); + } +} +impl core::ops::Sub for u8x32 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: u8) -> Self::Output { + self.simd.sub_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for u8x32 { + #[inline(always)] + fn sub_assign(&mut self, rhs: u8) { + *self = self.simd.sub_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for u8 { + type Output = u8x32; + #[inline(always)] + fn sub(self, rhs: u8x32) -> Self::Output { + rhs.simd.sub_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for u8x32 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_u8x32(self, rhs) + } +} +impl core::ops::MulAssign for u8x32 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_u8x32(*self, rhs); + } +} +impl core::ops::Mul for u8x32 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: u8) -> Self::Output { + self.simd.mul_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for u8x32 { + #[inline(always)] + fn mul_assign(&mut self, rhs: u8) { + *self = self.simd.mul_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for u8 { + type Output = u8x32; + #[inline(always)] + fn mul(self, rhs: u8x32) -> Self::Output { + rhs.simd.mul_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for u8x32 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_u8x32(self, rhs) + } +} +impl core::ops::BitAndAssign for u8x32 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_u8x32(*self, rhs); + } +} +impl core::ops::BitAnd for u8x32 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: u8) -> Self::Output { + self.simd.and_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for u8x32 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: u8) { + *self = self.simd.and_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for u8 { + type Output = u8x32; + #[inline(always)] + fn bitand(self, rhs: u8x32) -> Self::Output { + rhs.simd.and_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for u8x32 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_u8x32(self, rhs) + } +} +impl core::ops::BitOrAssign for u8x32 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_u8x32(*self, rhs); + } +} +impl core::ops::BitOr for u8x32 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: u8) -> Self::Output { + self.simd.or_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for u8x32 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: u8) { + *self = self.simd.or_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for u8 { + type Output = u8x32; + #[inline(always)] + fn bitor(self, rhs: u8x32) -> Self::Output { + rhs.simd.or_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for u8x32 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_u8x32(self, rhs) + } +} +impl core::ops::BitXorAssign for u8x32 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_u8x32(*self, rhs); + } +} +impl core::ops::BitXor for u8x32 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: u8) -> Self::Output { + self.simd.xor_u8x32(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for u8x32 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: u8) { + *self = self.simd.xor_u8x32(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for u8 { + type Output = u8x32; + #[inline(always)] + fn bitxor(self, rhs: u8x32) -> Self::Output { + rhs.simd.xor_u8x32(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for u8x32 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_u8x32(self) + } +} +impl core::ops::Shl for u8x32 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_u8x32(self, rhs) + } +} +impl core::ops::ShlAssign for u8x32 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_u8x32(*self, rhs); + } +} +impl core::ops::Shl for u8x32 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_u8x32(self, rhs) + } +} +impl core::ops::ShlAssign for u8x32 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_u8x32(*self, rhs); + } +} +impl core::ops::Shr for u8x32 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_u8x32(self, rhs) + } +} +impl core::ops::ShrAssign for u8x32 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_u8x32(*self, rhs); + } +} +impl core::ops::Shr for u8x32 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_u8x32(self, rhs) + } +} +impl core::ops::ShrAssign for u8x32 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u8x32(*self, rhs); + } +} +impl core::ops::BitAnd for mask8x32 { + type Output = Self; + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask8x32(self, rhs) + } +} +impl core::ops::BitAndAssign for mask8x32 { + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask8x32(*self, rhs); + } +} +impl core::ops::BitOr for mask8x32 { + type Output = Self; + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask8x32(self, rhs) + } +} +impl core::ops::BitOrAssign for mask8x32 { + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask8x32(*self, rhs); + } +} +impl core::ops::BitXor for mask8x32 { + type Output = Self; + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask8x32(self, rhs) + } +} +impl core::ops::BitXorAssign for mask8x32 { + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask8x32(*self, rhs); + } +} +impl core::ops::Not for mask8x32 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_mask8x32(self) + } +} +impl core::ops::Neg for i16x16 { + type Output = Self; + #[doc = "Negate each element of the vector, wrapping on overflow."] + #[inline(always)] + fn neg(self) -> Self::Output { + self.simd.neg_i16x16(self) + } +} +impl core::ops::Add for i16x16 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_i16x16(self, rhs) + } +} +impl core::ops::AddAssign for i16x16 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_i16x16(*self, rhs); + } +} +impl core::ops::Add for i16x16 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: i16) -> Self::Output { + self.simd.add_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for i16x16 { + #[inline(always)] + fn add_assign(&mut self, rhs: i16) { + *self = self.simd.add_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for i16 { + type Output = i16x16; + #[inline(always)] + fn add(self, rhs: i16x16) -> Self::Output { + rhs.simd.add_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for i16x16 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_i16x16(self, rhs) + } +} +impl core::ops::SubAssign for i16x16 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_i16x16(*self, rhs); + } +} +impl core::ops::Sub for i16x16 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: i16) -> Self::Output { + self.simd.sub_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for i16x16 { + #[inline(always)] + fn sub_assign(&mut self, rhs: i16) { + *self = self.simd.sub_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for i16 { + type Output = i16x16; + #[inline(always)] + fn sub(self, rhs: i16x16) -> Self::Output { + rhs.simd.sub_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for i16x16 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_i16x16(self, rhs) + } +} +impl core::ops::MulAssign for i16x16 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_i16x16(*self, rhs); + } +} +impl core::ops::Mul for i16x16 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: i16) -> Self::Output { + self.simd.mul_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for i16x16 { + #[inline(always)] + fn mul_assign(&mut self, rhs: i16) { + *self = self.simd.mul_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for i16 { + type Output = i16x16; + #[inline(always)] + fn mul(self, rhs: i16x16) -> Self::Output { + rhs.simd.mul_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for i16x16 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_i16x16(self, rhs) + } +} +impl core::ops::BitAndAssign for i16x16 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_i16x16(*self, rhs); + } +} +impl core::ops::BitAnd for i16x16 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: i16) -> Self::Output { + self.simd.and_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for i16x16 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: i16) { + *self = self.simd.and_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for i16 { + type Output = i16x16; + #[inline(always)] + fn bitand(self, rhs: i16x16) -> Self::Output { + rhs.simd.and_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for i16x16 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_i16x16(self, rhs) + } +} +impl core::ops::BitOrAssign for i16x16 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_i16x16(*self, rhs); + } +} +impl core::ops::BitOr for i16x16 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: i16) -> Self::Output { + self.simd.or_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for i16x16 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: i16) { + *self = self.simd.or_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for i16 { + type Output = i16x16; + #[inline(always)] + fn bitor(self, rhs: i16x16) -> Self::Output { + rhs.simd.or_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for i16x16 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_i16x16(self, rhs) + } +} +impl core::ops::BitXorAssign for i16x16 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_i16x16(*self, rhs); + } +} +impl core::ops::BitXor for i16x16 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: i16) -> Self::Output { + self.simd.xor_i16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for i16x16 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: i16) { + *self = self.simd.xor_i16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for i16 { + type Output = i16x16; + #[inline(always)] + fn bitxor(self, rhs: i16x16) -> Self::Output { + rhs.simd.xor_i16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for i16x16 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_i16x16(self) + } +} +impl core::ops::Shl for i16x16 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_i16x16(self, rhs) + } +} +impl core::ops::ShlAssign for i16x16 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_i16x16(*self, rhs); + } +} +impl core::ops::Shl for i16x16 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_i16x16(self, rhs) + } +} +impl core::ops::ShlAssign for i16x16 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_i16x16(*self, rhs); + } +} +impl core::ops::Shr for i16x16 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_i16x16(self, rhs) + } +} +impl core::ops::ShrAssign for i16x16 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_i16x16(*self, rhs); + } +} +impl core::ops::Shr for i16x16 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_i16x16(self, rhs) + } +} +impl core::ops::ShrAssign for i16x16 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_i16x16(*self, rhs); + } +} +impl core::ops::Add for u16x16 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_u16x16(self, rhs) + } +} +impl core::ops::AddAssign for u16x16 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_u16x16(*self, rhs); + } +} +impl core::ops::Add for u16x16 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: u16) -> Self::Output { + self.simd.add_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for u16x16 { + #[inline(always)] + fn add_assign(&mut self, rhs: u16) { + *self = self.simd.add_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for u16 { + type Output = u16x16; + #[inline(always)] + fn add(self, rhs: u16x16) -> Self::Output { + rhs.simd.add_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for u16x16 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_u16x16(self, rhs) + } +} +impl core::ops::SubAssign for u16x16 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_u16x16(*self, rhs); + } +} +impl core::ops::Sub for u16x16 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: u16) -> Self::Output { + self.simd.sub_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for u16x16 { + #[inline(always)] + fn sub_assign(&mut self, rhs: u16) { + *self = self.simd.sub_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for u16 { + type Output = u16x16; + #[inline(always)] + fn sub(self, rhs: u16x16) -> Self::Output { + rhs.simd.sub_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for u16x16 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_u16x16(self, rhs) + } +} +impl core::ops::MulAssign for u16x16 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_u16x16(*self, rhs); + } +} +impl core::ops::Mul for u16x16 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: u16) -> Self::Output { + self.simd.mul_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for u16x16 { + #[inline(always)] + fn mul_assign(&mut self, rhs: u16) { + *self = self.simd.mul_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for u16 { + type Output = u16x16; + #[inline(always)] + fn mul(self, rhs: u16x16) -> Self::Output { + rhs.simd.mul_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for u16x16 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_u16x16(self, rhs) + } +} +impl core::ops::BitAndAssign for u16x16 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_u16x16(*self, rhs); + } +} +impl core::ops::BitAnd for u16x16 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: u16) -> Self::Output { + self.simd.and_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for u16x16 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: u16) { + *self = self.simd.and_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for u16 { + type Output = u16x16; + #[inline(always)] + fn bitand(self, rhs: u16x16) -> Self::Output { + rhs.simd.and_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for u16x16 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_u16x16(self, rhs) + } +} +impl core::ops::BitOrAssign for u16x16 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_u16x16(*self, rhs); + } +} +impl core::ops::BitOr for u16x16 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: u16) -> Self::Output { + self.simd.or_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for u16x16 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: u16) { + *self = self.simd.or_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for u16 { + type Output = u16x16; + #[inline(always)] + fn bitor(self, rhs: u16x16) -> Self::Output { + rhs.simd.or_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for u16x16 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_u16x16(self, rhs) + } +} +impl core::ops::BitXorAssign for u16x16 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_u16x16(*self, rhs); + } +} +impl core::ops::BitXor for u16x16 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: u16) -> Self::Output { + self.simd.xor_u16x16(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for u16x16 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: u16) { + *self = self.simd.xor_u16x16(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for u16 { + type Output = u16x16; + #[inline(always)] + fn bitxor(self, rhs: u16x16) -> Self::Output { + rhs.simd.xor_u16x16(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for u16x16 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_u16x16(self) + } +} +impl core::ops::Shl for u16x16 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_u16x16(self, rhs) + } +} +impl core::ops::ShlAssign for u16x16 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_u16x16(*self, rhs); + } +} +impl core::ops::Shl for u16x16 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_u16x16(self, rhs) + } +} +impl core::ops::ShlAssign for u16x16 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_u16x16(*self, rhs); + } +} +impl core::ops::Shr for u16x16 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_u16x16(self, rhs) + } +} +impl core::ops::ShrAssign for u16x16 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_u16x16(*self, rhs); + } +} +impl core::ops::Shr for u16x16 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_u16x16(self, rhs) + } +} +impl core::ops::ShrAssign for u16x16 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u16x16(*self, rhs); + } +} +impl core::ops::BitAnd for mask16x16 { + type Output = Self; + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask16x16(self, rhs) + } +} +impl core::ops::BitAndAssign for mask16x16 { + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask16x16(*self, rhs); + } +} +impl core::ops::BitOr for mask16x16 { + type Output = Self; + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask16x16(self, rhs) + } +} +impl core::ops::BitOrAssign for mask16x16 { + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask16x16(*self, rhs); + } +} +impl core::ops::BitXor for mask16x16 { + type Output = Self; + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask16x16(self, rhs) + } +} +impl core::ops::BitXorAssign for mask16x16 { + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask16x16(*self, rhs); + } +} +impl core::ops::Not for mask16x16 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_mask16x16(self) + } +} +impl core::ops::Neg for i32x8 { + type Output = Self; + #[doc = "Negate each element of the vector, wrapping on overflow."] + #[inline(always)] + fn neg(self) -> Self::Output { + self.simd.neg_i32x8(self) + } +} +impl core::ops::Add for i32x8 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_i32x8(self, rhs) + } +} +impl core::ops::AddAssign for i32x8 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_i32x8(*self, rhs); + } +} +impl core::ops::Add for i32x8 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: i32) -> Self::Output { + self.simd.add_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for i32x8 { + #[inline(always)] + fn add_assign(&mut self, rhs: i32) { + *self = self.simd.add_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for i32 { + type Output = i32x8; + #[inline(always)] + fn add(self, rhs: i32x8) -> Self::Output { + rhs.simd.add_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for i32x8 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_i32x8(self, rhs) + } +} +impl core::ops::SubAssign for i32x8 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_i32x8(*self, rhs); + } +} +impl core::ops::Sub for i32x8 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: i32) -> Self::Output { + self.simd.sub_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for i32x8 { + #[inline(always)] + fn sub_assign(&mut self, rhs: i32) { + *self = self.simd.sub_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for i32 { + type Output = i32x8; + #[inline(always)] + fn sub(self, rhs: i32x8) -> Self::Output { + rhs.simd.sub_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for i32x8 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_i32x8(self, rhs) + } +} +impl core::ops::MulAssign for i32x8 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_i32x8(*self, rhs); + } +} +impl core::ops::Mul for i32x8 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: i32) -> Self::Output { + self.simd.mul_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for i32x8 { + #[inline(always)] + fn mul_assign(&mut self, rhs: i32) { + *self = self.simd.mul_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for i32 { + type Output = i32x8; + #[inline(always)] + fn mul(self, rhs: i32x8) -> Self::Output { + rhs.simd.mul_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for i32x8 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_i32x8(self, rhs) + } +} +impl core::ops::BitAndAssign for i32x8 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_i32x8(*self, rhs); + } +} +impl core::ops::BitAnd for i32x8 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: i32) -> Self::Output { + self.simd.and_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for i32x8 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: i32) { + *self = self.simd.and_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for i32 { + type Output = i32x8; + #[inline(always)] + fn bitand(self, rhs: i32x8) -> Self::Output { + rhs.simd.and_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for i32x8 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_i32x8(self, rhs) + } +} +impl core::ops::BitOrAssign for i32x8 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_i32x8(*self, rhs); + } +} +impl core::ops::BitOr for i32x8 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: i32) -> Self::Output { + self.simd.or_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for i32x8 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: i32) { + *self = self.simd.or_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for i32 { + type Output = i32x8; + #[inline(always)] + fn bitor(self, rhs: i32x8) -> Self::Output { + rhs.simd.or_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for i32x8 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_i32x8(self, rhs) + } +} +impl core::ops::BitXorAssign for i32x8 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_i32x8(*self, rhs); + } +} +impl core::ops::BitXor for i32x8 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: i32) -> Self::Output { + self.simd.xor_i32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for i32x8 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: i32) { + *self = self.simd.xor_i32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for i32 { + type Output = i32x8; + #[inline(always)] + fn bitxor(self, rhs: i32x8) -> Self::Output { + rhs.simd.xor_i32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for i32x8 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_i32x8(self) + } +} +impl core::ops::Shl for i32x8 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_i32x8(self, rhs) + } +} +impl core::ops::ShlAssign for i32x8 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_i32x8(*self, rhs); + } +} +impl core::ops::Shl for i32x8 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_i32x8(self, rhs) + } +} +impl core::ops::ShlAssign for i32x8 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_i32x8(*self, rhs); + } +} +impl core::ops::Shr for i32x8 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_i32x8(self, rhs) + } +} +impl core::ops::ShrAssign for i32x8 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_i32x8(*self, rhs); + } +} +impl core::ops::Shr for i32x8 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_i32x8(self, rhs) + } +} +impl core::ops::ShrAssign for i32x8 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_i32x8(*self, rhs); + } +} +impl core::ops::Add for u32x8 { + type Output = Self; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_u32x8(self, rhs) + } +} +impl core::ops::AddAssign for u32x8 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_u32x8(*self, rhs); + } +} +impl core::ops::Add for u32x8 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: u32) -> Self::Output { + self.simd.add_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::AddAssign for u32x8 { + #[inline(always)] + fn add_assign(&mut self, rhs: u32) { + *self = self.simd.add_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Add> for u32 { + type Output = u32x8; + #[inline(always)] + fn add(self, rhs: u32x8) -> Self::Output { + rhs.simd.add_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for u32x8 { + type Output = Self; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_u32x8(self, rhs) + } +} +impl core::ops::SubAssign for u32x8 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_u32x8(*self, rhs); + } +} +impl core::ops::Sub for u32x8 { + type Output = Self; + #[inline(always)] + fn sub(self, rhs: u32) -> Self::Output { + self.simd.sub_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::SubAssign for u32x8 { + #[inline(always)] + fn sub_assign(&mut self, rhs: u32) { + *self = self.simd.sub_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for u32 { + type Output = u32x8; + #[inline(always)] + fn sub(self, rhs: u32x8) -> Self::Output { + rhs.simd.sub_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for u32x8 { + type Output = Self; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_u32x8(self, rhs) + } +} +impl core::ops::MulAssign for u32x8 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_u32x8(*self, rhs); + } +} +impl core::ops::Mul for u32x8 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: u32) -> Self::Output { + self.simd.mul_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::MulAssign for u32x8 { + #[inline(always)] + fn mul_assign(&mut self, rhs: u32) { + *self = self.simd.mul_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Mul> for u32 { + type Output = u32x8; + #[inline(always)] + fn mul(self, rhs: u32x8) -> Self::Output { + rhs.simd.mul_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for u32x8 { + type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_u32x8(self, rhs) + } +} +impl core::ops::BitAndAssign for u32x8 { + #[doc = "Compute the bitwise AND of two vectors."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_u32x8(*self, rhs); + } +} +impl core::ops::BitAnd for u32x8 { + type Output = Self; + #[inline(always)] + fn bitand(self, rhs: u32) -> Self::Output { + self.simd.and_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitAndAssign for u32x8 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: u32) { + *self = self.simd.and_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for u32 { + type Output = u32x8; + #[inline(always)] + fn bitand(self, rhs: u32x8) -> Self::Output { + rhs.simd.and_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for u32x8 { + type Output = Self; + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_u32x8(self, rhs) + } +} +impl core::ops::BitOrAssign for u32x8 { + #[doc = "Compute the bitwise OR of two vectors."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_u32x8(*self, rhs); + } +} +impl core::ops::BitOr for u32x8 { + type Output = Self; + #[inline(always)] + fn bitor(self, rhs: u32) -> Self::Output { + self.simd.or_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitOrAssign for u32x8 { + #[inline(always)] + fn bitor_assign(&mut self, rhs: u32) { + *self = self.simd.or_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitOr> for u32 { + type Output = u32x8; + #[inline(always)] + fn bitor(self, rhs: u32x8) -> Self::Output { + rhs.simd.or_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitXor for u32x8 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_u32x8(self, rhs) + } +} +impl core::ops::BitXorAssign for u32x8 { + #[doc = "Compute the bitwise XOR of two vectors."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_u32x8(*self, rhs); + } +} +impl core::ops::BitXor for u32x8 { + type Output = Self; + #[inline(always)] + fn bitxor(self, rhs: u32) -> Self::Output { + self.simd.xor_u32x8(self, rhs.simd_into(self.simd)) + } +} +impl core::ops::BitXorAssign for u32x8 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: u32) { + *self = self.simd.xor_u32x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitXor> for u32 { + type Output = u32x8; + #[inline(always)] + fn bitxor(self, rhs: u32x8) -> Self::Output { + rhs.simd.xor_u32x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Not for u32x8 { + type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] + #[inline(always)] + fn not(self) -> Self::Output { + self.simd.not_u32x8(self) + } +} +impl core::ops::Shl for u32x8 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[inline(always)] + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_u32x8(self, rhs) + } +} +impl core::ops::ShlAssign for u32x8 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_u32x8(*self, rhs); + } +} +impl core::ops::Shl for u32x8 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_u32x8(self, rhs) + } +} +impl core::ops::ShlAssign for u32x8 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_u32x8(*self, rhs); + } +} +impl core::ops::Shr for u32x8 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[inline(always)] + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_u32x8(self, rhs) + } +} +impl core::ops::ShrAssign for u32x8 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_u32x8(*self, rhs); + } +} +impl core::ops::Shr for u32x8 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_u32x8(self, rhs) + } +} +impl core::ops::ShrAssign for u32x8 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u32x8(*self, rhs); + } +} +impl core::ops::BitAnd for mask32x8 { + type Output = Self; + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask32x8(self, rhs) + } +} +impl core::ops::BitAndAssign for mask32x8 { + #[doc = "Compute the logical AND of two masks."] + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask32x8(*self, rhs); + } +} +impl core::ops::BitOr for mask32x8 { + type Output = Self; + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask32x8(self, rhs) + } +} +impl core::ops::BitOrAssign for mask32x8 { + #[doc = "Compute the logical OR of two masks."] + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask32x8(*self, rhs); + } +} +impl core::ops::BitXor for mask32x8 { + type Output = Self; + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask32x8(self, rhs) + } +} +impl core::ops::BitXorAssign for mask32x8 { + #[doc = "Compute the logical XOR of two masks."] + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask32x8(*self, rhs); + } +} +impl core::ops::Not for mask32x8 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u8x32(self) + self.simd.not_mask32x8(self) } } -impl core::ops::Shl for u8x32 { +impl core::ops::Neg for f64x4 { type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + #[doc = "Negate each element of the vector."] #[inline(always)] - fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u8x32(self, rhs) + fn neg(self) -> Self::Output { + self.simd.neg_f64x4(self) } } -impl core::ops::ShlAssign for u8x32 { +impl core::ops::Add for f64x4 { + type Output = Self; + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u8x32(*self, rhs); + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_f64x4(self, rhs) } } -impl core::ops::Shl for u8x32 { +impl core::ops::AddAssign for f64x4 { + #[doc = "Add two vectors element-wise."] + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_f64x4(*self, rhs); + } +} +impl core::ops::Add for f64x4 { type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u8x32(self, rhs) + fn add(self, rhs: f64) -> Self::Output { + self.simd.add_f64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::ShlAssign for u8x32 { - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::AddAssign for f64x4 { #[inline(always)] - fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u8x32(*self, rhs); + fn add_assign(&mut self, rhs: f64) { + *self = self.simd.add_f64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Shr for u8x32 { +impl core::ops::Add> for f64 { + type Output = f64x4; + #[inline(always)] + fn add(self, rhs: f64x4) -> Self::Output { + rhs.simd.add_f64x4(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for f64x4 { type Output = Self; - #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u8x32(self, rhs) + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_f64x4(self, rhs) } } -impl core::ops::ShrAssign for u8x32 { +impl core::ops::SubAssign for f64x4 { + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u8x32(*self, rhs); + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_f64x4(*self, rhs); } } -impl core::ops::Shr for u8x32 { +impl core::ops::Sub for f64x4 { type Output = Self; - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u8x32(self, rhs) + fn sub(self, rhs: f64) -> Self::Output { + self.simd.sub_f64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::ShrAssign for u8x32 { - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::SubAssign for f64x4 { #[inline(always)] - fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u8x32(*self, rhs); + fn sub_assign(&mut self, rhs: f64) { + *self = self.simd.sub_f64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd for mask8x32 { +impl core::ops::Sub> for f64 { + type Output = f64x4; + #[inline(always)] + fn sub(self, rhs: f64x4) -> Self::Output { + rhs.simd.sub_f64x4(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for f64x4 { type Output = Self; - #[doc = "Compute the logical AND of two masks."] + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask8x32(self, rhs) + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_f64x4(self, rhs) } } -impl core::ops::BitAndAssign for mask8x32 { - #[doc = "Compute the logical AND of two masks."] +impl core::ops::MulAssign for f64x4 { + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask8x32(*self, rhs); + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_f64x4(*self, rhs); } } -impl core::ops::BitOr for mask8x32 { +impl core::ops::Mul for f64x4 { type Output = Self; - #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask8x32(self, rhs) + fn mul(self, rhs: f64) -> Self::Output { + self.simd.mul_f64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for mask8x32 { - #[doc = "Compute the logical OR of two masks."] +impl core::ops::MulAssign for f64x4 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask8x32(*self, rhs); + fn mul_assign(&mut self, rhs: f64) { + *self = self.simd.mul_f64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for mask8x32 { +impl core::ops::Mul> for f64 { + type Output = f64x4; + #[inline(always)] + fn mul(self, rhs: f64x4) -> Self::Output { + rhs.simd.mul_f64x4(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Div for f64x4 { type Output = Self; - #[doc = "Compute the logical XOR of two masks."] + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask8x32(self, rhs) + fn div(self, rhs: Self) -> Self::Output { + self.simd.div_f64x4(self, rhs) } } -impl core::ops::BitXorAssign for mask8x32 { - #[doc = "Compute the logical XOR of two masks."] +impl core::ops::DivAssign for f64x4 { + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask8x32(*self, rhs); + fn div_assign(&mut self, rhs: Self) { + *self = self.simd.div_f64x4(*self, rhs); } } -impl core::ops::Not for mask8x32 { +impl core::ops::Div for f64x4 { type Output = Self; - #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask8x32(self) + fn div(self, rhs: f64) -> Self::Output { + self.simd.div_f64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::Neg for i16x16 { +impl core::ops::DivAssign for f64x4 { + #[inline(always)] + fn div_assign(&mut self, rhs: f64) { + *self = self.simd.div_f64x4(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Div> for f64 { + type Output = f64x4; + #[inline(always)] + fn div(self, rhs: f64x4) -> Self::Output { + rhs.simd.div_f64x4(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Neg for i64x4 { type Output = Self; #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] fn neg(self) -> Self::Output { - self.simd.neg_i16x16(self) + self.simd.neg_i64x4(self) } } -impl core::ops::Add for i16x16 { +impl core::ops::Add for i64x4 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i16x16(self, rhs) + self.simd.add_i64x4(self, rhs) } } -impl core::ops::AddAssign for i16x16 { +impl core::ops::AddAssign for i64x4 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i16x16(*self, rhs); + *self = self.simd.add_i64x4(*self, rhs); } } -impl core::ops::Add for i16x16 { +impl core::ops::Add for i64x4 { type Output = Self; #[inline(always)] - fn add(self, rhs: i16) -> Self::Output { - self.simd.add_i16x16(self, rhs.simd_into(self.simd)) + fn add(self, rhs: i64) -> Self::Output { + self.simd.add_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for i16x16 { +impl core::ops::AddAssign for i64x4 { #[inline(always)] - fn add_assign(&mut self, rhs: i16) { - *self = self.simd.add_i16x16(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: i64) { + *self = self.simd.add_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for i16 { - type Output = i16x16; +impl core::ops::Add> for i64 { + type Output = i64x4; #[inline(always)] - fn add(self, rhs: i16x16) -> Self::Output { - rhs.simd.add_i16x16(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: i64x4) -> Self::Output { + rhs.simd.add_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for i16x16 { +impl core::ops::Sub for i64x4 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i16x16(self, rhs) + self.simd.sub_i64x4(self, rhs) } } -impl core::ops::SubAssign for i16x16 { +impl core::ops::SubAssign for i64x4 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i16x16(*self, rhs); + *self = self.simd.sub_i64x4(*self, rhs); } } -impl core::ops::Sub for i16x16 { +impl core::ops::Sub for i64x4 { type Output = Self; #[inline(always)] - fn sub(self, rhs: i16) -> Self::Output { - self.simd.sub_i16x16(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: i64) -> Self::Output { + self.simd.sub_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for i16x16 { +impl core::ops::SubAssign for i64x4 { #[inline(always)] - fn sub_assign(&mut self, rhs: i16) { - *self = self.simd.sub_i16x16(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: i64) { + *self = self.simd.sub_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for i16 { - type Output = i16x16; +impl core::ops::Sub> for i64 { + type Output = i64x4; #[inline(always)] - fn sub(self, rhs: i16x16) -> Self::Output { - rhs.simd.sub_i16x16(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: i64x4) -> Self::Output { + rhs.simd.sub_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for i16x16 { +impl core::ops::Mul for i64x4 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i16x16(self, rhs) + self.simd.mul_i64x4(self, rhs) } } -impl core::ops::MulAssign for i16x16 { +impl core::ops::MulAssign for i64x4 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i16x16(*self, rhs); + *self = self.simd.mul_i64x4(*self, rhs); } } -impl core::ops::Mul for i16x16 { +impl core::ops::Mul for i64x4 { type Output = Self; #[inline(always)] - fn mul(self, rhs: i16) -> Self::Output { - self.simd.mul_i16x16(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: i64) -> Self::Output { + self.simd.mul_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for i16x16 { +impl core::ops::MulAssign for i64x4 { #[inline(always)] - fn mul_assign(&mut self, rhs: i16) { - *self = self.simd.mul_i16x16(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: i64) { + *self = self.simd.mul_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for i16 { - type Output = i16x16; +impl core::ops::Mul> for i64 { + type Output = i64x4; #[inline(always)] - fn mul(self, rhs: i16x16) -> Self::Output { - rhs.simd.mul_i16x16(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: i64x4) -> Self::Output { + rhs.simd.mul_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for i16x16 { +impl core::ops::BitAnd for i64x4 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i16x16(self, rhs) + self.simd.and_i64x4(self, rhs) } } -impl core::ops::BitAndAssign for i16x16 { +impl core::ops::BitAndAssign for i64x4 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i16x16(*self, rhs); + *self = self.simd.and_i64x4(*self, rhs); } } -impl core::ops::BitAnd for i16x16 { +impl core::ops::BitAnd for i64x4 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: i16) -> Self::Output { - self.simd.and_i16x16(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: i64) -> Self::Output { + self.simd.and_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for i16x16 { +impl core::ops::BitAndAssign for i64x4 { #[inline(always)] - fn bitand_assign(&mut self, rhs: i16) { - *self = self.simd.and_i16x16(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: i64) { + *self = self.simd.and_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for i16 { - type Output = i16x16; +impl core::ops::BitAnd> for i64 { + type Output = i64x4; #[inline(always)] - fn bitand(self, rhs: i16x16) -> Self::Output { - rhs.simd.and_i16x16(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i64x4) -> Self::Output { + rhs.simd.and_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for i16x16 { +impl core::ops::BitOr for i64x4 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i16x16(self, rhs) + self.simd.or_i64x4(self, rhs) } } -impl core::ops::BitOrAssign for i16x16 { +impl core::ops::BitOrAssign for i64x4 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i16x16(*self, rhs); + *self = self.simd.or_i64x4(*self, rhs); } } -impl core::ops::BitOr for i16x16 { +impl core::ops::BitOr for i64x4 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: i16) -> Self::Output { - self.simd.or_i16x16(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i64) -> Self::Output { + self.simd.or_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i16x16 { +impl core::ops::BitOrAssign for i64x4 { #[inline(always)] - fn bitor_assign(&mut self, rhs: i16) { - *self = self.simd.or_i16x16(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i64) { + *self = self.simd.or_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for i16 { - type Output = i16x16; +impl core::ops::BitOr> for i64 { + type Output = i64x4; #[inline(always)] - fn bitor(self, rhs: i16x16) -> Self::Output { - rhs.simd.or_i16x16(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i64x4) -> Self::Output { + rhs.simd.or_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for i16x16 { +impl core::ops::BitXor for i64x4 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i16x16(self, rhs) + self.simd.xor_i64x4(self, rhs) } } -impl core::ops::BitXorAssign for i16x16 { +impl core::ops::BitXorAssign for i64x4 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i16x16(*self, rhs); + *self = self.simd.xor_i64x4(*self, rhs); } } -impl core::ops::BitXor for i16x16 { +impl core::ops::BitXor for i64x4 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: i16) -> Self::Output { - self.simd.xor_i16x16(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i64) -> Self::Output { + self.simd.xor_i64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i16x16 { +impl core::ops::BitXorAssign for i64x4 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: i16) { - *self = self.simd.xor_i16x16(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i64) { + *self = self.simd.xor_i64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for i16 { - type Output = i16x16; +impl core::ops::BitXor> for i64 { + type Output = i64x4; #[inline(always)] - fn bitxor(self, rhs: i16x16) -> Self::Output { - rhs.simd.xor_i16x16(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i64x4) -> Self::Output { + rhs.simd.xor_i64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for i16x16 { +impl core::ops::Not for i64x4 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_i16x16(self) + self.simd.not_i64x4(self) } } -impl core::ops::Shl for i16x16 { +impl core::ops::Shl for i64x4 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i16x16(self, rhs) + self.simd.shl_i64x4(self, rhs) } } -impl core::ops::ShlAssign for i16x16 { +impl core::ops::ShlAssign for i64x4 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i16x16(*self, rhs); + *self = self.simd.shl_i64x4(*self, rhs); } } -impl core::ops::Shl for i16x16 { +impl core::ops::Shl for i64x4 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i16x16(self, rhs) + self.simd.shlv_i64x4(self, rhs) } } -impl core::ops::ShlAssign for i16x16 { +impl core::ops::ShlAssign for i64x4 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i16x16(*self, rhs); + *self = self.simd.shlv_i64x4(*self, rhs); } } -impl core::ops::Shr for i16x16 { +impl core::ops::Shr for i64x4 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i16x16(self, rhs) + self.simd.shr_i64x4(self, rhs) } } -impl core::ops::ShrAssign for i16x16 { +impl core::ops::ShrAssign for i64x4 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i16x16(*self, rhs); + *self = self.simd.shr_i64x4(*self, rhs); } } -impl core::ops::Shr for i16x16 { +impl core::ops::Shr for i64x4 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i16x16(self, rhs) + self.simd.shrv_i64x4(self, rhs) } } -impl core::ops::ShrAssign for i16x16 { +impl core::ops::ShrAssign for i64x4 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i16x16(*self, rhs); + *self = self.simd.shrv_i64x4(*self, rhs); } } -impl core::ops::Add for u16x16 { +impl core::ops::Add for u64x4 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u16x16(self, rhs) + self.simd.add_u64x4(self, rhs) } } -impl core::ops::AddAssign for u16x16 { +impl core::ops::AddAssign for u64x4 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u16x16(*self, rhs); + *self = self.simd.add_u64x4(*self, rhs); } } -impl core::ops::Add for u16x16 { +impl core::ops::Add for u64x4 { type Output = Self; #[inline(always)] - fn add(self, rhs: u16) -> Self::Output { - self.simd.add_u16x16(self, rhs.simd_into(self.simd)) + fn add(self, rhs: u64) -> Self::Output { + self.simd.add_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for u16x16 { +impl core::ops::AddAssign for u64x4 { #[inline(always)] - fn add_assign(&mut self, rhs: u16) { - *self = self.simd.add_u16x16(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: u64) { + *self = self.simd.add_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for u16 { - type Output = u16x16; +impl core::ops::Add> for u64 { + type Output = u64x4; #[inline(always)] - fn add(self, rhs: u16x16) -> Self::Output { - rhs.simd.add_u16x16(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: u64x4) -> Self::Output { + rhs.simd.add_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for u16x16 { +impl core::ops::Sub for u64x4 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u16x16(self, rhs) + self.simd.sub_u64x4(self, rhs) } } -impl core::ops::SubAssign for u16x16 { +impl core::ops::SubAssign for u64x4 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u16x16(*self, rhs); + *self = self.simd.sub_u64x4(*self, rhs); } } -impl core::ops::Sub for u16x16 { +impl core::ops::Sub for u64x4 { type Output = Self; #[inline(always)] - fn sub(self, rhs: u16) -> Self::Output { - self.simd.sub_u16x16(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: u64) -> Self::Output { + self.simd.sub_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for u16x16 { +impl core::ops::SubAssign for u64x4 { #[inline(always)] - fn sub_assign(&mut self, rhs: u16) { - *self = self.simd.sub_u16x16(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: u64) { + *self = self.simd.sub_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for u16 { - type Output = u16x16; +impl core::ops::Sub> for u64 { + type Output = u64x4; #[inline(always)] - fn sub(self, rhs: u16x16) -> Self::Output { - rhs.simd.sub_u16x16(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: u64x4) -> Self::Output { + rhs.simd.sub_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for u16x16 { +impl core::ops::Mul for u64x4 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u16x16(self, rhs) + self.simd.mul_u64x4(self, rhs) } } -impl core::ops::MulAssign for u16x16 { +impl core::ops::MulAssign for u64x4 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u16x16(*self, rhs); + *self = self.simd.mul_u64x4(*self, rhs); } } -impl core::ops::Mul for u16x16 { +impl core::ops::Mul for u64x4 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u16) -> Self::Output { - self.simd.mul_u16x16(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: u64) -> Self::Output { + self.simd.mul_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u16x16 { +impl core::ops::MulAssign for u64x4 { #[inline(always)] - fn mul_assign(&mut self, rhs: u16) { - *self = self.simd.mul_u16x16(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: u64) { + *self = self.simd.mul_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for u16 { - type Output = u16x16; +impl core::ops::Mul> for u64 { + type Output = u64x4; #[inline(always)] - fn mul(self, rhs: u16x16) -> Self::Output { - rhs.simd.mul_u16x16(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: u64x4) -> Self::Output { + rhs.simd.mul_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u16x16 { +impl core::ops::BitAnd for u64x4 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u16x16(self, rhs) + self.simd.and_u64x4(self, rhs) } } -impl core::ops::BitAndAssign for u16x16 { +impl core::ops::BitAndAssign for u64x4 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u16x16(*self, rhs); + *self = self.simd.and_u64x4(*self, rhs); } } -impl core::ops::BitAnd for u16x16 { +impl core::ops::BitAnd for u64x4 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u16) -> Self::Output { - self.simd.and_u16x16(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: u64) -> Self::Output { + self.simd.and_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u16x16 { +impl core::ops::BitAndAssign for u64x4 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u16) { - *self = self.simd.and_u16x16(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: u64) { + *self = self.simd.and_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u16 { - type Output = u16x16; +impl core::ops::BitAnd> for u64 { + type Output = u64x4; #[inline(always)] - fn bitand(self, rhs: u16x16) -> Self::Output { - rhs.simd.and_u16x16(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u64x4) -> Self::Output { + rhs.simd.and_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u16x16 { +impl core::ops::BitOr for u64x4 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u16x16(self, rhs) + self.simd.or_u64x4(self, rhs) } } -impl core::ops::BitOrAssign for u16x16 { +impl core::ops::BitOrAssign for u64x4 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u16x16(*self, rhs); + *self = self.simd.or_u64x4(*self, rhs); } } -impl core::ops::BitOr for u16x16 { +impl core::ops::BitOr for u64x4 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u16) -> Self::Output { - self.simd.or_u16x16(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: u64) -> Self::Output { + self.simd.or_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u16x16 { +impl core::ops::BitOrAssign for u64x4 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u16) { - *self = self.simd.or_u16x16(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: u64) { + *self = self.simd.or_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u16 { - type Output = u16x16; +impl core::ops::BitOr> for u64 { + type Output = u64x4; #[inline(always)] - fn bitor(self, rhs: u16x16) -> Self::Output { - rhs.simd.or_u16x16(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: u64x4) -> Self::Output { + rhs.simd.or_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u16x16 { +impl core::ops::BitXor for u64x4 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u16x16(self, rhs) + self.simd.xor_u64x4(self, rhs) } } -impl core::ops::BitXorAssign for u16x16 { +impl core::ops::BitXorAssign for u64x4 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u16x16(*self, rhs); + *self = self.simd.xor_u64x4(*self, rhs); } } -impl core::ops::BitXor for u16x16 { +impl core::ops::BitXor for u64x4 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u16) -> Self::Output { - self.simd.xor_u16x16(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: u64) -> Self::Output { + self.simd.xor_u64x4(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u16x16 { +impl core::ops::BitXorAssign for u64x4 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u16) { - *self = self.simd.xor_u16x16(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: u64) { + *self = self.simd.xor_u64x4(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u16 { - type Output = u16x16; +impl core::ops::BitXor> for u64 { + type Output = u64x4; #[inline(always)] - fn bitxor(self, rhs: u16x16) -> Self::Output { - rhs.simd.xor_u16x16(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: u64x4) -> Self::Output { + rhs.simd.xor_u64x4(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u16x16 { +impl core::ops::Not for u64x4 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u16x16(self) + self.simd.not_u64x4(self) } } -impl core::ops::Shl for u16x16 { +impl core::ops::Shl for u64x4 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u16x16(self, rhs) + self.simd.shl_u64x4(self, rhs) } } -impl core::ops::ShlAssign for u16x16 { +impl core::ops::ShlAssign for u64x4 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u16x16(*self, rhs); + *self = self.simd.shl_u64x4(*self, rhs); } } -impl core::ops::Shl for u16x16 { +impl core::ops::Shl for u64x4 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u16x16(self, rhs) + self.simd.shlv_u64x4(self, rhs) } } -impl core::ops::ShlAssign for u16x16 { +impl core::ops::ShlAssign for u64x4 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u16x16(*self, rhs); + *self = self.simd.shlv_u64x4(*self, rhs); } } -impl core::ops::Shr for u16x16 { +impl core::ops::Shr for u64x4 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u16x16(self, rhs) + self.simd.shr_u64x4(self, rhs) } } -impl core::ops::ShrAssign for u16x16 { +impl core::ops::ShrAssign for u64x4 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u16x16(*self, rhs); + *self = self.simd.shr_u64x4(*self, rhs); } } -impl core::ops::Shr for u16x16 { +impl core::ops::Shr for u64x4 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u16x16(self, rhs) + self.simd.shrv_u64x4(self, rhs) } } -impl core::ops::ShrAssign for u16x16 { +impl core::ops::ShrAssign for u64x4 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u16x16(*self, rhs); + *self = self.simd.shrv_u64x4(*self, rhs); } } -impl core::ops::BitAnd for mask16x16 { +impl core::ops::BitAnd for mask64x4 { type Output = Self; #[doc = "Compute the logical AND of two masks."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask16x16(self, rhs) + self.simd.and_mask64x4(self, rhs) } } -impl core::ops::BitAndAssign for mask16x16 { +impl core::ops::BitAndAssign for mask64x4 { #[doc = "Compute the logical AND of two masks."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask16x16(*self, rhs); + *self = self.simd.and_mask64x4(*self, rhs); } } -impl core::ops::BitOr for mask16x16 { +impl core::ops::BitOr for mask64x4 { type Output = Self; #[doc = "Compute the logical OR of two masks."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask16x16(self, rhs) + self.simd.or_mask64x4(self, rhs) } } -impl core::ops::BitOrAssign for mask16x16 { +impl core::ops::BitOrAssign for mask64x4 { #[doc = "Compute the logical OR of two masks."] - #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask16x16(*self, rhs); - } -} -impl core::ops::BitXor for mask16x16 { - type Output = Self; - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask16x16(self, rhs) - } -} -impl core::ops::BitXorAssign for mask16x16 { - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask16x16(*self, rhs); - } -} -impl core::ops::Not for mask16x16 { - type Output = Self; - #[doc = "Compute the logical NOT of the mask."] - #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask16x16(self) - } -} -impl core::ops::Neg for i32x8 { - type Output = Self; - #[doc = "Negate each element of the vector, wrapping on overflow."] - #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_i32x8(self) - } -} -impl core::ops::Add for i32x8 { - type Output = Self; - #[doc = "Add two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i32x8(self, rhs) - } -} -impl core::ops::AddAssign for i32x8 { - #[doc = "Add two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i32x8(*self, rhs); - } -} -impl core::ops::Add for i32x8 { - type Output = Self; - #[inline(always)] - fn add(self, rhs: i32) -> Self::Output { - self.simd.add_i32x8(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::AddAssign for i32x8 { - #[inline(always)] - fn add_assign(&mut self, rhs: i32) { - *self = self.simd.add_i32x8(*self, rhs.simd_into(self.simd)); - } -} -impl core::ops::Add> for i32 { - type Output = i32x8; - #[inline(always)] - fn add(self, rhs: i32x8) -> Self::Output { - rhs.simd.add_i32x8(self.simd_into(rhs.simd), rhs) - } -} -impl core::ops::Sub for i32x8 { - type Output = Self; - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i32x8(self, rhs) - } -} -impl core::ops::SubAssign for i32x8 { - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i32x8(*self, rhs); - } -} -impl core::ops::Sub for i32x8 { - type Output = Self; - #[inline(always)] - fn sub(self, rhs: i32) -> Self::Output { - self.simd.sub_i32x8(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::SubAssign for i32x8 { - #[inline(always)] - fn sub_assign(&mut self, rhs: i32) { - *self = self.simd.sub_i32x8(*self, rhs.simd_into(self.simd)); - } -} -impl core::ops::Sub> for i32 { - type Output = i32x8; - #[inline(always)] - fn sub(self, rhs: i32x8) -> Self::Output { - rhs.simd.sub_i32x8(self.simd_into(rhs.simd), rhs) - } -} -impl core::ops::Mul for i32x8 { - type Output = Self; - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i32x8(self, rhs) - } -} -impl core::ops::MulAssign for i32x8 { - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i32x8(*self, rhs); - } -} -impl core::ops::Mul for i32x8 { - type Output = Self; - #[inline(always)] - fn mul(self, rhs: i32) -> Self::Output { - self.simd.mul_i32x8(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::MulAssign for i32x8 { - #[inline(always)] - fn mul_assign(&mut self, rhs: i32) { - *self = self.simd.mul_i32x8(*self, rhs.simd_into(self.simd)); + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask64x4(*self, rhs); } } -impl core::ops::Mul> for i32 { - type Output = i32x8; +impl core::ops::BitXor for mask64x4 { + type Output = Self; + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn mul(self, rhs: i32x8) -> Self::Output { - rhs.simd.mul_i32x8(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask64x4(self, rhs) } } -impl core::ops::BitAnd for i32x8 { - type Output = Self; - #[doc = "Compute the bitwise AND of two vectors."] +impl core::ops::BitXorAssign for mask64x4 { + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i32x8(self, rhs) + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask64x4(*self, rhs); } } -impl core::ops::BitAndAssign for i32x8 { - #[doc = "Compute the bitwise AND of two vectors."] +impl core::ops::Not for mask64x4 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i32x8(*self, rhs); + fn not(self) -> Self::Output { + self.simd.not_mask64x4(self) } } -impl core::ops::BitAnd for i32x8 { +impl core::ops::Neg for f32x16 { type Output = Self; + #[doc = "Negate each element of the vector."] #[inline(always)] - fn bitand(self, rhs: i32) -> Self::Output { - self.simd.and_i32x8(self, rhs.simd_into(self.simd)) + fn neg(self) -> Self::Output { + self.simd.neg_f32x16(self) } } -impl core::ops::BitAndAssign for i32x8 { +impl core::ops::Add for f32x16 { + type Output = Self; + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn bitand_assign(&mut self, rhs: i32) { - *self = self.simd.and_i32x8(*self, rhs.simd_into(self.simd)); + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_f32x16(self, rhs) } } -impl core::ops::BitAnd> for i32 { - type Output = i32x8; +impl core::ops::AddAssign for f32x16 { + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn bitand(self, rhs: i32x8) -> Self::Output { - rhs.simd.and_i32x8(self.simd_into(rhs.simd), rhs) + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_f32x16(*self, rhs); } } -impl core::ops::BitOr for i32x8 { +impl core::ops::Add for f32x16 { type Output = Self; - #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i32x8(self, rhs) + fn add(self, rhs: f32) -> Self::Output { + self.simd.add_f32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i32x8 { - #[doc = "Compute the bitwise OR of two vectors."] +impl core::ops::AddAssign for f32x16 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i32x8(*self, rhs); + fn add_assign(&mut self, rhs: f32) { + *self = self.simd.add_f32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr for i32x8 { - type Output = Self; +impl core::ops::Add> for f32 { + type Output = f32x16; #[inline(always)] - fn bitor(self, rhs: i32) -> Self::Output { - self.simd.or_i32x8(self, rhs.simd_into(self.simd)) + fn add(self, rhs: f32x16) -> Self::Output { + rhs.simd.add_f32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOrAssign for i32x8 { +impl core::ops::Sub for f32x16 { + type Output = Self; + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitor_assign(&mut self, rhs: i32) { - *self = self.simd.or_i32x8(*self, rhs.simd_into(self.simd)); + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_f32x16(self, rhs) } } -impl core::ops::BitOr> for i32 { - type Output = i32x8; +impl core::ops::SubAssign for f32x16 { + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitor(self, rhs: i32x8) -> Self::Output { - rhs.simd.or_i32x8(self.simd_into(rhs.simd), rhs) + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_f32x16(*self, rhs); } } -impl core::ops::BitXor for i32x8 { +impl core::ops::Sub for f32x16 { type Output = Self; - #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i32x8(self, rhs) + fn sub(self, rhs: f32) -> Self::Output { + self.simd.sub_f32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i32x8 { - #[doc = "Compute the bitwise XOR of two vectors."] +impl core::ops::SubAssign for f32x16 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i32x8(*self, rhs); + fn sub_assign(&mut self, rhs: f32) { + *self = self.simd.sub_f32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for i32x8 { - type Output = Self; +impl core::ops::Sub> for f32 { + type Output = f32x16; #[inline(always)] - fn bitxor(self, rhs: i32) -> Self::Output { - self.simd.xor_i32x8(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: f32x16) -> Self::Output { + rhs.simd.sub_f32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXorAssign for i32x8 { +impl core::ops::Mul for f32x16 { + type Output = Self; + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: i32) { - *self = self.simd.xor_i32x8(*self, rhs.simd_into(self.simd)); + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_f32x16(self, rhs) } } -impl core::ops::BitXor> for i32 { - type Output = i32x8; +impl core::ops::MulAssign for f32x16 { + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitxor(self, rhs: i32x8) -> Self::Output { - rhs.simd.xor_i32x8(self.simd_into(rhs.simd), rhs) + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_f32x16(*self, rhs); } } -impl core::ops::Not for i32x8 { +impl core::ops::Mul for f32x16 { type Output = Self; - #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_i32x8(self) + fn mul(self, rhs: f32) -> Self::Output { + self.simd.mul_f32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::Shl for i32x8 { - type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] +impl core::ops::MulAssign for f32x16 { #[inline(always)] - fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i32x8(self, rhs) + fn mul_assign(&mut self, rhs: f32) { + *self = self.simd.mul_f32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::ShlAssign for i32x8 { +impl core::ops::Mul> for f32 { + type Output = f32x16; #[inline(always)] - fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i32x8(*self, rhs); + fn mul(self, rhs: f32x16) -> Self::Output { + rhs.simd.mul_f32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Shl for i32x8 { +impl core::ops::Div for f32x16 { type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i32x8(self, rhs) + fn div(self, rhs: Self) -> Self::Output { + self.simd.div_f32x16(self, rhs) } } -impl core::ops::ShlAssign for i32x8 { - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::DivAssign for f32x16 { + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i32x8(*self, rhs); + fn div_assign(&mut self, rhs: Self) { + *self = self.simd.div_f32x16(*self, rhs); } } -impl core::ops::Shr for i32x8 { +impl core::ops::Div for f32x16 { type Output = Self; - #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] - fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i32x8(self, rhs) + fn div(self, rhs: f32) -> Self::Output { + self.simd.div_f32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::ShrAssign for i32x8 { +impl core::ops::DivAssign for f32x16 { #[inline(always)] - fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i32x8(*self, rhs); + fn div_assign(&mut self, rhs: f32) { + *self = self.simd.div_f32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Shr for i32x8 { - type Output = Self; - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::Div> for f32 { + type Output = f32x16; #[inline(always)] - fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i32x8(self, rhs) + fn div(self, rhs: f32x16) -> Self::Output { + rhs.simd.div_f32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::ShrAssign for i32x8 { - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::Neg for i8x64 { + type Output = Self; + #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] - fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i32x8(*self, rhs); + fn neg(self) -> Self::Output { + self.simd.neg_i8x64(self) } } -impl core::ops::Add for u32x8 { +impl core::ops::Add for i8x64 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u32x8(self, rhs) + self.simd.add_i8x64(self, rhs) } } -impl core::ops::AddAssign for u32x8 { +impl core::ops::AddAssign for i8x64 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u32x8(*self, rhs); + *self = self.simd.add_i8x64(*self, rhs); } } -impl core::ops::Add for u32x8 { +impl core::ops::Add for i8x64 { type Output = Self; #[inline(always)] - fn add(self, rhs: u32) -> Self::Output { - self.simd.add_u32x8(self, rhs.simd_into(self.simd)) + fn add(self, rhs: i8) -> Self::Output { + self.simd.add_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for u32x8 { +impl core::ops::AddAssign for i8x64 { #[inline(always)] - fn add_assign(&mut self, rhs: u32) { - *self = self.simd.add_u32x8(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: i8) { + *self = self.simd.add_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for u32 { - type Output = u32x8; +impl core::ops::Add> for i8 { + type Output = i8x64; #[inline(always)] - fn add(self, rhs: u32x8) -> Self::Output { - rhs.simd.add_u32x8(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: i8x64) -> Self::Output { + rhs.simd.add_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for u32x8 { +impl core::ops::Sub for i8x64 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u32x8(self, rhs) + self.simd.sub_i8x64(self, rhs) } } -impl core::ops::SubAssign for u32x8 { +impl core::ops::SubAssign for i8x64 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u32x8(*self, rhs); + *self = self.simd.sub_i8x64(*self, rhs); } } -impl core::ops::Sub for u32x8 { +impl core::ops::Sub for i8x64 { type Output = Self; #[inline(always)] - fn sub(self, rhs: u32) -> Self::Output { - self.simd.sub_u32x8(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: i8) -> Self::Output { + self.simd.sub_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for u32x8 { +impl core::ops::SubAssign for i8x64 { #[inline(always)] - fn sub_assign(&mut self, rhs: u32) { - *self = self.simd.sub_u32x8(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: i8) { + *self = self.simd.sub_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for u32 { - type Output = u32x8; +impl core::ops::Sub> for i8 { + type Output = i8x64; #[inline(always)] - fn sub(self, rhs: u32x8) -> Self::Output { - rhs.simd.sub_u32x8(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: i8x64) -> Self::Output { + rhs.simd.sub_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for u32x8 { +impl core::ops::Mul for i8x64 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u32x8(self, rhs) + self.simd.mul_i8x64(self, rhs) } } -impl core::ops::MulAssign for u32x8 { +impl core::ops::MulAssign for i8x64 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u32x8(*self, rhs); + *self = self.simd.mul_i8x64(*self, rhs); } } -impl core::ops::Mul for u32x8 { +impl core::ops::Mul for i8x64 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u32) -> Self::Output { - self.simd.mul_u32x8(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: i8) -> Self::Output { + self.simd.mul_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u32x8 { +impl core::ops::MulAssign for i8x64 { #[inline(always)] - fn mul_assign(&mut self, rhs: u32) { - *self = self.simd.mul_u32x8(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: i8) { + *self = self.simd.mul_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for u32 { - type Output = u32x8; +impl core::ops::Mul> for i8 { + type Output = i8x64; #[inline(always)] - fn mul(self, rhs: u32x8) -> Self::Output { - rhs.simd.mul_u32x8(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: i8x64) -> Self::Output { + rhs.simd.mul_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u32x8 { +impl core::ops::BitAnd for i8x64 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u32x8(self, rhs) + self.simd.and_i8x64(self, rhs) } } -impl core::ops::BitAndAssign for u32x8 { +impl core::ops::BitAndAssign for i8x64 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u32x8(*self, rhs); + *self = self.simd.and_i8x64(*self, rhs); } } -impl core::ops::BitAnd for u32x8 { +impl core::ops::BitAnd for i8x64 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u32) -> Self::Output { - self.simd.and_u32x8(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: i8) -> Self::Output { + self.simd.and_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u32x8 { +impl core::ops::BitAndAssign for i8x64 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u32) { - *self = self.simd.and_u32x8(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: i8) { + *self = self.simd.and_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u32 { - type Output = u32x8; +impl core::ops::BitAnd> for i8 { + type Output = i8x64; #[inline(always)] - fn bitand(self, rhs: u32x8) -> Self::Output { - rhs.simd.and_u32x8(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i8x64) -> Self::Output { + rhs.simd.and_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u32x8 { +impl core::ops::BitOr for i8x64 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u32x8(self, rhs) + self.simd.or_i8x64(self, rhs) } } -impl core::ops::BitOrAssign for u32x8 { +impl core::ops::BitOrAssign for i8x64 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u32x8(*self, rhs); + *self = self.simd.or_i8x64(*self, rhs); } } -impl core::ops::BitOr for u32x8 { +impl core::ops::BitOr for i8x64 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u32) -> Self::Output { - self.simd.or_u32x8(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i8) -> Self::Output { + self.simd.or_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u32x8 { +impl core::ops::BitOrAssign for i8x64 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u32) { - *self = self.simd.or_u32x8(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i8) { + *self = self.simd.or_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u32 { - type Output = u32x8; +impl core::ops::BitOr> for i8 { + type Output = i8x64; #[inline(always)] - fn bitor(self, rhs: u32x8) -> Self::Output { - rhs.simd.or_u32x8(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i8x64) -> Self::Output { + rhs.simd.or_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u32x8 { +impl core::ops::BitXor for i8x64 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u32x8(self, rhs) + self.simd.xor_i8x64(self, rhs) } } -impl core::ops::BitXorAssign for u32x8 { +impl core::ops::BitXorAssign for i8x64 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u32x8(*self, rhs); + *self = self.simd.xor_i8x64(*self, rhs); } } -impl core::ops::BitXor for u32x8 { +impl core::ops::BitXor for i8x64 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u32) -> Self::Output { - self.simd.xor_u32x8(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i8) -> Self::Output { + self.simd.xor_i8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u32x8 { +impl core::ops::BitXorAssign for i8x64 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u32) { - *self = self.simd.xor_u32x8(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i8) { + *self = self.simd.xor_i8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u32 { - type Output = u32x8; +impl core::ops::BitXor> for i8 { + type Output = i8x64; #[inline(always)] - fn bitxor(self, rhs: u32x8) -> Self::Output { - rhs.simd.xor_u32x8(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i8x64) -> Self::Output { + rhs.simd.xor_i8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u32x8 { +impl core::ops::Not for i8x64 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u32x8(self) + self.simd.not_i8x64(self) } } -impl core::ops::Shl for u32x8 { +impl core::ops::Shl for i8x64 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u32x8(self, rhs) + self.simd.shl_i8x64(self, rhs) } } -impl core::ops::ShlAssign for u32x8 { +impl core::ops::ShlAssign for i8x64 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u32x8(*self, rhs); + *self = self.simd.shl_i8x64(*self, rhs); } } -impl core::ops::Shl for u32x8 { +impl core::ops::Shl for i8x64 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u32x8(self, rhs) + self.simd.shlv_i8x64(self, rhs) } } -impl core::ops::ShlAssign for u32x8 { +impl core::ops::ShlAssign for i8x64 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u32x8(*self, rhs); + *self = self.simd.shlv_i8x64(*self, rhs); } } -impl core::ops::Shr for u32x8 { +impl core::ops::Shr for i8x64 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u32x8(self, rhs) + self.simd.shr_i8x64(self, rhs) } } -impl core::ops::ShrAssign for u32x8 { +impl core::ops::ShrAssign for i8x64 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u32x8(*self, rhs); + *self = self.simd.shr_i8x64(*self, rhs); } } -impl core::ops::Shr for u32x8 { +impl core::ops::Shr for i8x64 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u32x8(self, rhs) + self.simd.shrv_i8x64(self, rhs) } } -impl core::ops::ShrAssign for u32x8 { +impl core::ops::ShrAssign for i8x64 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u32x8(*self, rhs); - } -} -impl core::ops::BitAnd for mask32x8 { - type Output = Self; - #[doc = "Compute the logical AND of two masks."] - #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask32x8(self, rhs) - } -} -impl core::ops::BitAndAssign for mask32x8 { - #[doc = "Compute the logical AND of two masks."] - #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask32x8(*self, rhs); - } -} -impl core::ops::BitOr for mask32x8 { - type Output = Self; - #[doc = "Compute the logical OR of two masks."] - #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask32x8(self, rhs) - } -} -impl core::ops::BitOrAssign for mask32x8 { - #[doc = "Compute the logical OR of two masks."] - #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask32x8(*self, rhs); - } -} -impl core::ops::BitXor for mask32x8 { - type Output = Self; - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask32x8(self, rhs) - } -} -impl core::ops::BitXorAssign for mask32x8 { - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask32x8(*self, rhs); - } -} -impl core::ops::Not for mask32x8 { - type Output = Self; - #[doc = "Compute the logical NOT of the mask."] - #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask32x8(self) - } -} -impl core::ops::Neg for f64x4 { - type Output = Self; - #[doc = "Negate each element of the vector."] - #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_f64x4(self) + *self = self.simd.shrv_i8x64(*self, rhs); } } -impl core::ops::Add for f64x4 { +impl core::ops::Add for u8x64 { type Output = Self; - #[doc = "Add two vectors element-wise."] + #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_f64x4(self, rhs) + self.simd.add_u8x64(self, rhs) } } -impl core::ops::AddAssign for f64x4 { - #[doc = "Add two vectors element-wise."] +impl core::ops::AddAssign for u8x64 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_f64x4(*self, rhs); + *self = self.simd.add_u8x64(*self, rhs); } } -impl core::ops::Add for f64x4 { +impl core::ops::Add for u8x64 { type Output = Self; #[inline(always)] - fn add(self, rhs: f64) -> Self::Output { - self.simd.add_f64x4(self, rhs.simd_into(self.simd)) + fn add(self, rhs: u8) -> Self::Output { + self.simd.add_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for f64x4 { +impl core::ops::AddAssign for u8x64 { #[inline(always)] - fn add_assign(&mut self, rhs: f64) { - *self = self.simd.add_f64x4(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: u8) { + *self = self.simd.add_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for f64 { - type Output = f64x4; +impl core::ops::Add> for u8 { + type Output = u8x64; #[inline(always)] - fn add(self, rhs: f64x4) -> Self::Output { - rhs.simd.add_f64x4(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: u8x64) -> Self::Output { + rhs.simd.add_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for f64x4 { +impl core::ops::Sub for u8x64 { type Output = Self; - #[doc = "Subtract two vectors element-wise."] + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_f64x4(self, rhs) + self.simd.sub_u8x64(self, rhs) } } -impl core::ops::SubAssign for f64x4 { - #[doc = "Subtract two vectors element-wise."] +impl core::ops::SubAssign for u8x64 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_f64x4(*self, rhs); + *self = self.simd.sub_u8x64(*self, rhs); } } -impl core::ops::Sub for f64x4 { +impl core::ops::Sub for u8x64 { type Output = Self; #[inline(always)] - fn sub(self, rhs: f64) -> Self::Output { - self.simd.sub_f64x4(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: u8) -> Self::Output { + self.simd.sub_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for f64x4 { +impl core::ops::SubAssign for u8x64 { #[inline(always)] - fn sub_assign(&mut self, rhs: f64) { - *self = self.simd.sub_f64x4(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: u8) { + *self = self.simd.sub_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for f64 { - type Output = f64x4; +impl core::ops::Sub> for u8 { + type Output = u8x64; #[inline(always)] - fn sub(self, rhs: f64x4) -> Self::Output { - rhs.simd.sub_f64x4(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: u8x64) -> Self::Output { + rhs.simd.sub_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for f64x4 { +impl core::ops::Mul for u8x64 { type Output = Self; - #[doc = "Multiply two vectors element-wise."] + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_f64x4(self, rhs) + self.simd.mul_u8x64(self, rhs) } } -impl core::ops::MulAssign for f64x4 { - #[doc = "Multiply two vectors element-wise."] +impl core::ops::MulAssign for u8x64 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_f64x4(*self, rhs); + *self = self.simd.mul_u8x64(*self, rhs); } } -impl core::ops::Mul for f64x4 { +impl core::ops::Mul for u8x64 { type Output = Self; #[inline(always)] - fn mul(self, rhs: f64) -> Self::Output { - self.simd.mul_f64x4(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: u8) -> Self::Output { + self.simd.mul_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for f64x4 { +impl core::ops::MulAssign for u8x64 { #[inline(always)] - fn mul_assign(&mut self, rhs: f64) { - *self = self.simd.mul_f64x4(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: u8) { + *self = self.simd.mul_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for f64 { - type Output = f64x4; +impl core::ops::Mul> for u8 { + type Output = u8x64; #[inline(always)] - fn mul(self, rhs: f64x4) -> Self::Output { - rhs.simd.mul_f64x4(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: u8x64) -> Self::Output { + rhs.simd.mul_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Div for f64x4 { +impl core::ops::BitAnd for u8x64 { type Output = Self; - #[doc = "Divide two vectors element-wise."] + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn div(self, rhs: Self) -> Self::Output { - self.simd.div_f64x4(self, rhs) + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_u8x64(self, rhs) } } -impl core::ops::DivAssign for f64x4 { - #[doc = "Divide two vectors element-wise."] +impl core::ops::BitAndAssign for u8x64 { + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn div_assign(&mut self, rhs: Self) { - *self = self.simd.div_f64x4(*self, rhs); + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_u8x64(*self, rhs); } } -impl core::ops::Div for f64x4 { +impl core::ops::BitAnd for u8x64 { type Output = Self; #[inline(always)] - fn div(self, rhs: f64) -> Self::Output { - self.simd.div_f64x4(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: u8) -> Self::Output { + self.simd.and_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::DivAssign for f64x4 { +impl core::ops::BitAndAssign for u8x64 { #[inline(always)] - fn div_assign(&mut self, rhs: f64) { - *self = self.simd.div_f64x4(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: u8) { + *self = self.simd.and_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Div> for f64 { - type Output = f64x4; +impl core::ops::BitAnd> for u8 { + type Output = u8x64; #[inline(always)] - fn div(self, rhs: f64x4) -> Self::Output { - rhs.simd.div_f64x4(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u8x64) -> Self::Output { + rhs.simd.and_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for mask64x4 { +impl core::ops::BitOr for u8x64 { type Output = Self; - #[doc = "Compute the logical AND of two masks."] + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask64x4(self, rhs) + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_u8x64(self, rhs) } } -impl core::ops::BitAndAssign for mask64x4 { - #[doc = "Compute the logical AND of two masks."] +impl core::ops::BitOrAssign for u8x64 { + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask64x4(*self, rhs); + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_u8x64(*self, rhs); } } -impl core::ops::BitOr for mask64x4 { +impl core::ops::BitOr for u8x64 { type Output = Self; - #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask64x4(self, rhs) + fn bitor(self, rhs: u8) -> Self::Output { + self.simd.or_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for mask64x4 { - #[doc = "Compute the logical OR of two masks."] +impl core::ops::BitOrAssign for u8x64 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask64x4(*self, rhs); + fn bitor_assign(&mut self, rhs: u8) { + *self = self.simd.or_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for mask64x4 { - type Output = Self; - #[doc = "Compute the logical XOR of two masks."] +impl core::ops::BitOr> for u8 { + type Output = u8x64; #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask64x4(self, rhs) + fn bitor(self, rhs: u8x64) -> Self::Output { + rhs.simd.or_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXorAssign for mask64x4 { - #[doc = "Compute the logical XOR of two masks."] +impl core::ops::BitXor for u8x64 { + type Output = Self; + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask64x4(*self, rhs); + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_u8x64(self, rhs) } } -impl core::ops::Not for mask64x4 { - type Output = Self; - #[doc = "Compute the logical NOT of the mask."] +impl core::ops::BitXorAssign for u8x64 { + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask64x4(self) + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_u8x64(*self, rhs); } } -impl core::ops::Neg for f32x16 { +impl core::ops::BitXor for u8x64 { type Output = Self; - #[doc = "Negate each element of the vector."] #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_f32x16(self) + fn bitxor(self, rhs: u8) -> Self::Output { + self.simd.xor_u8x64(self, rhs.simd_into(self.simd)) } } -impl core::ops::Add for f32x16 { - type Output = Self; - #[doc = "Add two vectors element-wise."] +impl core::ops::BitXorAssign for u8x64 { #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_f32x16(self, rhs) + fn bitxor_assign(&mut self, rhs: u8) { + *self = self.simd.xor_u8x64(*self, rhs.simd_into(self.simd)); } } -impl core::ops::AddAssign for f32x16 { - #[doc = "Add two vectors element-wise."] +impl core::ops::BitXor> for u8 { + type Output = u8x64; #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_f32x16(*self, rhs); + fn bitxor(self, rhs: u8x64) -> Self::Output { + rhs.simd.xor_u8x64(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Add for f32x16 { +impl core::ops::Not for u8x64 { type Output = Self; + #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] - fn add(self, rhs: f32) -> Self::Output { - self.simd.add_f32x16(self, rhs.simd_into(self.simd)) + fn not(self) -> Self::Output { + self.simd.not_u8x64(self) } } -impl core::ops::AddAssign for f32x16 { +impl core::ops::Shl for u8x64 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] - fn add_assign(&mut self, rhs: f32) { - *self = self.simd.add_f32x16(*self, rhs.simd_into(self.simd)); + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_u8x64(self, rhs) } } -impl core::ops::Add> for f32 { - type Output = f32x16; +impl core::ops::ShlAssign for u8x64 { #[inline(always)] - fn add(self, rhs: f32x16) -> Self::Output { - rhs.simd.add_f32x16(self.simd_into(rhs.simd), rhs) + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_u8x64(*self, rhs); } } -impl core::ops::Sub for f32x16 { +impl core::ops::Shl for u8x64 { type Output = Self; - #[doc = "Subtract two vectors element-wise."] + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_f32x16(self, rhs) + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_u8x64(self, rhs) } } -impl core::ops::SubAssign for f32x16 { - #[doc = "Subtract two vectors element-wise."] +impl core::ops::ShlAssign for u8x64 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_f32x16(*self, rhs); + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_u8x64(*self, rhs); } } -impl core::ops::Sub for f32x16 { +impl core::ops::Shr for u8x64 { type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] - fn sub(self, rhs: f32) -> Self::Output { - self.simd.sub_f32x16(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::SubAssign for f32x16 { - #[inline(always)] - fn sub_assign(&mut self, rhs: f32) { - *self = self.simd.sub_f32x16(*self, rhs.simd_into(self.simd)); + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_u8x64(self, rhs) } } -impl core::ops::Sub> for f32 { - type Output = f32x16; +impl core::ops::ShrAssign for u8x64 { #[inline(always)] - fn sub(self, rhs: f32x16) -> Self::Output { - rhs.simd.sub_f32x16(self.simd_into(rhs.simd), rhs) + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_u8x64(*self, rhs); } } -impl core::ops::Mul for f32x16 { +impl core::ops::Shr for u8x64 { type Output = Self; - #[doc = "Multiply two vectors element-wise."] + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_f32x16(self, rhs) + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_u8x64(self, rhs) } } -impl core::ops::MulAssign for f32x16 { - #[doc = "Multiply two vectors element-wise."] +impl core::ops::ShrAssign for u8x64 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_f32x16(*self, rhs); + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u8x64(*self, rhs); } } -impl core::ops::Mul for f32x16 { +impl core::ops::BitAnd for mask8x64 { type Output = Self; + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn mul(self, rhs: f32) -> Self::Output { - self.simd.mul_f32x16(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::MulAssign for f32x16 { - #[inline(always)] - fn mul_assign(&mut self, rhs: f32) { - *self = self.simd.mul_f32x16(*self, rhs.simd_into(self.simd)); + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask8x64(self, rhs) } } -impl core::ops::Mul> for f32 { - type Output = f32x16; +impl core::ops::BitAndAssign for mask8x64 { + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn mul(self, rhs: f32x16) -> Self::Output { - rhs.simd.mul_f32x16(self.simd_into(rhs.simd), rhs) + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask8x64(*self, rhs); } } -impl core::ops::Div for f32x16 { +impl core::ops::BitOr for mask8x64 { type Output = Self; - #[doc = "Divide two vectors element-wise."] + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn div(self, rhs: Self) -> Self::Output { - self.simd.div_f32x16(self, rhs) + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask8x64(self, rhs) } } -impl core::ops::DivAssign for f32x16 { - #[doc = "Divide two vectors element-wise."] +impl core::ops::BitOrAssign for mask8x64 { + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn div_assign(&mut self, rhs: Self) { - *self = self.simd.div_f32x16(*self, rhs); + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask8x64(*self, rhs); } } -impl core::ops::Div for f32x16 { +impl core::ops::BitXor for mask8x64 { type Output = Self; + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn div(self, rhs: f32) -> Self::Output { - self.simd.div_f32x16(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask8x64(self, rhs) } } -impl core::ops::DivAssign for f32x16 { +impl core::ops::BitXorAssign for mask8x64 { + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn div_assign(&mut self, rhs: f32) { - *self = self.simd.div_f32x16(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask8x64(*self, rhs); } } -impl core::ops::Div> for f32 { - type Output = f32x16; +impl core::ops::Not for mask8x64 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn div(self, rhs: f32x16) -> Self::Output { - rhs.simd.div_f32x16(self.simd_into(rhs.simd), rhs) + fn not(self) -> Self::Output { + self.simd.not_mask8x64(self) } } -impl core::ops::Neg for i8x64 { +impl core::ops::Neg for i16x32 { type Output = Self; #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] fn neg(self) -> Self::Output { - self.simd.neg_i8x64(self) + self.simd.neg_i16x32(self) } } -impl core::ops::Add for i8x64 { +impl core::ops::Add for i16x32 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i8x64(self, rhs) + self.simd.add_i16x32(self, rhs) } } -impl core::ops::AddAssign for i8x64 { +impl core::ops::AddAssign for i16x32 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i8x64(*self, rhs); + *self = self.simd.add_i16x32(*self, rhs); } } -impl core::ops::Add for i8x64 { +impl core::ops::Add for i16x32 { type Output = Self; #[inline(always)] - fn add(self, rhs: i8) -> Self::Output { - self.simd.add_i8x64(self, rhs.simd_into(self.simd)) + fn add(self, rhs: i16) -> Self::Output { + self.simd.add_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for i8x64 { +impl core::ops::AddAssign for i16x32 { #[inline(always)] - fn add_assign(&mut self, rhs: i8) { - *self = self.simd.add_i8x64(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: i16) { + *self = self.simd.add_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for i8 { - type Output = i8x64; +impl core::ops::Add> for i16 { + type Output = i16x32; #[inline(always)] - fn add(self, rhs: i8x64) -> Self::Output { - rhs.simd.add_i8x64(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: i16x32) -> Self::Output { + rhs.simd.add_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for i8x64 { +impl core::ops::Sub for i16x32 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i8x64(self, rhs) + self.simd.sub_i16x32(self, rhs) } } -impl core::ops::SubAssign for i8x64 { +impl core::ops::SubAssign for i16x32 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i8x64(*self, rhs); + *self = self.simd.sub_i16x32(*self, rhs); } } -impl core::ops::Sub for i8x64 { +impl core::ops::Sub for i16x32 { type Output = Self; #[inline(always)] - fn sub(self, rhs: i8) -> Self::Output { - self.simd.sub_i8x64(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: i16) -> Self::Output { + self.simd.sub_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for i8x64 { +impl core::ops::SubAssign for i16x32 { #[inline(always)] - fn sub_assign(&mut self, rhs: i8) { - *self = self.simd.sub_i8x64(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: i16) { + *self = self.simd.sub_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for i8 { - type Output = i8x64; +impl core::ops::Sub> for i16 { + type Output = i16x32; #[inline(always)] - fn sub(self, rhs: i8x64) -> Self::Output { - rhs.simd.sub_i8x64(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: i16x32) -> Self::Output { + rhs.simd.sub_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for i8x64 { +impl core::ops::Mul for i16x32 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i8x64(self, rhs) + self.simd.mul_i16x32(self, rhs) } } -impl core::ops::MulAssign for i8x64 { +impl core::ops::MulAssign for i16x32 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i8x64(*self, rhs); + *self = self.simd.mul_i16x32(*self, rhs); } } -impl core::ops::Mul for i8x64 { +impl core::ops::Mul for i16x32 { type Output = Self; #[inline(always)] - fn mul(self, rhs: i8) -> Self::Output { - self.simd.mul_i8x64(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: i16) -> Self::Output { + self.simd.mul_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for i8x64 { +impl core::ops::MulAssign for i16x32 { #[inline(always)] - fn mul_assign(&mut self, rhs: i8) { - *self = self.simd.mul_i8x64(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: i16) { + *self = self.simd.mul_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for i8 { - type Output = i8x64; +impl core::ops::Mul> for i16 { + type Output = i16x32; #[inline(always)] - fn mul(self, rhs: i8x64) -> Self::Output { - rhs.simd.mul_i8x64(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: i16x32) -> Self::Output { + rhs.simd.mul_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for i8x64 { +impl core::ops::BitAnd for i16x32 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i8x64(self, rhs) + self.simd.and_i16x32(self, rhs) } } -impl core::ops::BitAndAssign for i8x64 { +impl core::ops::BitAndAssign for i16x32 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i8x64(*self, rhs); + *self = self.simd.and_i16x32(*self, rhs); } } -impl core::ops::BitAnd for i8x64 { +impl core::ops::BitAnd for i16x32 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: i8) -> Self::Output { - self.simd.and_i8x64(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: i16) -> Self::Output { + self.simd.and_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for i8x64 { +impl core::ops::BitAndAssign for i16x32 { #[inline(always)] - fn bitand_assign(&mut self, rhs: i8) { - *self = self.simd.and_i8x64(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: i16) { + *self = self.simd.and_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for i8 { - type Output = i8x64; +impl core::ops::BitAnd> for i16 { + type Output = i16x32; #[inline(always)] - fn bitand(self, rhs: i8x64) -> Self::Output { - rhs.simd.and_i8x64(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i16x32) -> Self::Output { + rhs.simd.and_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for i8x64 { +impl core::ops::BitOr for i16x32 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i8x64(self, rhs) + self.simd.or_i16x32(self, rhs) } } -impl core::ops::BitOrAssign for i8x64 { +impl core::ops::BitOrAssign for i16x32 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i8x64(*self, rhs); + *self = self.simd.or_i16x32(*self, rhs); } } -impl core::ops::BitOr for i8x64 { +impl core::ops::BitOr for i16x32 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: i8) -> Self::Output { - self.simd.or_i8x64(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i16) -> Self::Output { + self.simd.or_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i8x64 { +impl core::ops::BitOrAssign for i16x32 { #[inline(always)] - fn bitor_assign(&mut self, rhs: i8) { - *self = self.simd.or_i8x64(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i16) { + *self = self.simd.or_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for i8 { - type Output = i8x64; +impl core::ops::BitOr> for i16 { + type Output = i16x32; #[inline(always)] - fn bitor(self, rhs: i8x64) -> Self::Output { - rhs.simd.or_i8x64(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i16x32) -> Self::Output { + rhs.simd.or_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for i8x64 { +impl core::ops::BitXor for i16x32 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i8x64(self, rhs) + self.simd.xor_i16x32(self, rhs) } } -impl core::ops::BitXorAssign for i8x64 { +impl core::ops::BitXorAssign for i16x32 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i8x64(*self, rhs); + *self = self.simd.xor_i16x32(*self, rhs); } } -impl core::ops::BitXor for i8x64 { +impl core::ops::BitXor for i16x32 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: i8) -> Self::Output { - self.simd.xor_i8x64(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i16) -> Self::Output { + self.simd.xor_i16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i8x64 { +impl core::ops::BitXorAssign for i16x32 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: i8) { - *self = self.simd.xor_i8x64(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i16) { + *self = self.simd.xor_i16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for i8 { - type Output = i8x64; +impl core::ops::BitXor> for i16 { + type Output = i16x32; #[inline(always)] - fn bitxor(self, rhs: i8x64) -> Self::Output { - rhs.simd.xor_i8x64(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i16x32) -> Self::Output { + rhs.simd.xor_i16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for i8x64 { +impl core::ops::Not for i16x32 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_i8x64(self) + self.simd.not_i16x32(self) } } -impl core::ops::Shl for i8x64 { +impl core::ops::Shl for i16x32 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i8x64(self, rhs) + self.simd.shl_i16x32(self, rhs) } } -impl core::ops::ShlAssign for i8x64 { +impl core::ops::ShlAssign for i16x32 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i8x64(*self, rhs); + *self = self.simd.shl_i16x32(*self, rhs); } } -impl core::ops::Shl for i8x64 { +impl core::ops::Shl for i16x32 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i8x64(self, rhs) + self.simd.shlv_i16x32(self, rhs) } } -impl core::ops::ShlAssign for i8x64 { +impl core::ops::ShlAssign for i16x32 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i8x64(*self, rhs); + *self = self.simd.shlv_i16x32(*self, rhs); } } -impl core::ops::Shr for i8x64 { +impl core::ops::Shr for i16x32 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i8x64(self, rhs) + self.simd.shr_i16x32(self, rhs) } } -impl core::ops::ShrAssign for i8x64 { +impl core::ops::ShrAssign for i16x32 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i8x64(*self, rhs); + *self = self.simd.shr_i16x32(*self, rhs); } } -impl core::ops::Shr for i8x64 { +impl core::ops::Shr for i16x32 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i8x64(self, rhs) + self.simd.shrv_i16x32(self, rhs) } } -impl core::ops::ShrAssign for i8x64 { +impl core::ops::ShrAssign for i16x32 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i8x64(*self, rhs); + *self = self.simd.shrv_i16x32(*self, rhs); } } -impl core::ops::Add for u8x64 { +impl core::ops::Add for u16x32 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u8x64(self, rhs) + self.simd.add_u16x32(self, rhs) } } -impl core::ops::AddAssign for u8x64 { +impl core::ops::AddAssign for u16x32 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u8x64(*self, rhs); + *self = self.simd.add_u16x32(*self, rhs); } } -impl core::ops::Add for u8x64 { +impl core::ops::Add for u16x32 { type Output = Self; #[inline(always)] - fn add(self, rhs: u8) -> Self::Output { - self.simd.add_u8x64(self, rhs.simd_into(self.simd)) + fn add(self, rhs: u16) -> Self::Output { + self.simd.add_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for u8x64 { +impl core::ops::AddAssign for u16x32 { #[inline(always)] - fn add_assign(&mut self, rhs: u8) { - *self = self.simd.add_u8x64(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: u16) { + *self = self.simd.add_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for u8 { - type Output = u8x64; +impl core::ops::Add> for u16 { + type Output = u16x32; #[inline(always)] - fn add(self, rhs: u8x64) -> Self::Output { - rhs.simd.add_u8x64(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: u16x32) -> Self::Output { + rhs.simd.add_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for u8x64 { +impl core::ops::Sub for u16x32 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u8x64(self, rhs) + self.simd.sub_u16x32(self, rhs) } } -impl core::ops::SubAssign for u8x64 { +impl core::ops::SubAssign for u16x32 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u8x64(*self, rhs); + *self = self.simd.sub_u16x32(*self, rhs); } } -impl core::ops::Sub for u8x64 { +impl core::ops::Sub for u16x32 { type Output = Self; #[inline(always)] - fn sub(self, rhs: u8) -> Self::Output { - self.simd.sub_u8x64(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: u16) -> Self::Output { + self.simd.sub_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for u8x64 { +impl core::ops::SubAssign for u16x32 { #[inline(always)] - fn sub_assign(&mut self, rhs: u8) { - *self = self.simd.sub_u8x64(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: u16) { + *self = self.simd.sub_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for u8 { - type Output = u8x64; +impl core::ops::Sub> for u16 { + type Output = u16x32; #[inline(always)] - fn sub(self, rhs: u8x64) -> Self::Output { - rhs.simd.sub_u8x64(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: u16x32) -> Self::Output { + rhs.simd.sub_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for u8x64 { +impl core::ops::Mul for u16x32 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u8x64(self, rhs) + self.simd.mul_u16x32(self, rhs) } } -impl core::ops::MulAssign for u8x64 { +impl core::ops::MulAssign for u16x32 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u8x64(*self, rhs); + *self = self.simd.mul_u16x32(*self, rhs); } } -impl core::ops::Mul for u8x64 { +impl core::ops::Mul for u16x32 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u8) -> Self::Output { - self.simd.mul_u8x64(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: u16) -> Self::Output { + self.simd.mul_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u8x64 { +impl core::ops::MulAssign for u16x32 { #[inline(always)] - fn mul_assign(&mut self, rhs: u8) { - *self = self.simd.mul_u8x64(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: u16) { + *self = self.simd.mul_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for u8 { - type Output = u8x64; +impl core::ops::Mul> for u16 { + type Output = u16x32; #[inline(always)] - fn mul(self, rhs: u8x64) -> Self::Output { - rhs.simd.mul_u8x64(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: u16x32) -> Self::Output { + rhs.simd.mul_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u8x64 { +impl core::ops::BitAnd for u16x32 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u8x64(self, rhs) + self.simd.and_u16x32(self, rhs) } } -impl core::ops::BitAndAssign for u8x64 { +impl core::ops::BitAndAssign for u16x32 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u8x64(*self, rhs); + *self = self.simd.and_u16x32(*self, rhs); } } -impl core::ops::BitAnd for u8x64 { +impl core::ops::BitAnd for u16x32 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u8) -> Self::Output { - self.simd.and_u8x64(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: u16) -> Self::Output { + self.simd.and_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u8x64 { +impl core::ops::BitAndAssign for u16x32 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u8) { - *self = self.simd.and_u8x64(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: u16) { + *self = self.simd.and_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u8 { - type Output = u8x64; +impl core::ops::BitAnd> for u16 { + type Output = u16x32; #[inline(always)] - fn bitand(self, rhs: u8x64) -> Self::Output { - rhs.simd.and_u8x64(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u16x32) -> Self::Output { + rhs.simd.and_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u8x64 { +impl core::ops::BitOr for u16x32 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u8x64(self, rhs) + self.simd.or_u16x32(self, rhs) } } -impl core::ops::BitOrAssign for u8x64 { +impl core::ops::BitOrAssign for u16x32 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u8x64(*self, rhs); + *self = self.simd.or_u16x32(*self, rhs); } } -impl core::ops::BitOr for u8x64 { +impl core::ops::BitOr for u16x32 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u8) -> Self::Output { - self.simd.or_u8x64(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: u16) -> Self::Output { + self.simd.or_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u8x64 { +impl core::ops::BitOrAssign for u16x32 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u8) { - *self = self.simd.or_u8x64(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: u16) { + *self = self.simd.or_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u8 { - type Output = u8x64; +impl core::ops::BitOr> for u16 { + type Output = u16x32; #[inline(always)] - fn bitor(self, rhs: u8x64) -> Self::Output { - rhs.simd.or_u8x64(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: u16x32) -> Self::Output { + rhs.simd.or_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u8x64 { +impl core::ops::BitXor for u16x32 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u8x64(self, rhs) + self.simd.xor_u16x32(self, rhs) } } -impl core::ops::BitXorAssign for u8x64 { +impl core::ops::BitXorAssign for u16x32 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u8x64(*self, rhs); + *self = self.simd.xor_u16x32(*self, rhs); } } -impl core::ops::BitXor for u8x64 { +impl core::ops::BitXor for u16x32 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u8) -> Self::Output { - self.simd.xor_u8x64(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: u16) -> Self::Output { + self.simd.xor_u16x32(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u8x64 { +impl core::ops::BitXorAssign for u16x32 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u8) { - *self = self.simd.xor_u8x64(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: u16) { + *self = self.simd.xor_u16x32(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u8 { - type Output = u8x64; +impl core::ops::BitXor> for u16 { + type Output = u16x32; #[inline(always)] - fn bitxor(self, rhs: u8x64) -> Self::Output { - rhs.simd.xor_u8x64(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: u16x32) -> Self::Output { + rhs.simd.xor_u16x32(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u8x64 { +impl core::ops::Not for u16x32 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u8x64(self) + self.simd.not_u16x32(self) } } -impl core::ops::Shl for u8x64 { +impl core::ops::Shl for u16x32 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u8x64(self, rhs) + self.simd.shl_u16x32(self, rhs) } } -impl core::ops::ShlAssign for u8x64 { +impl core::ops::ShlAssign for u16x32 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u8x64(*self, rhs); + *self = self.simd.shl_u16x32(*self, rhs); } } -impl core::ops::Shl for u8x64 { +impl core::ops::Shl for u16x32 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u8x64(self, rhs) + self.simd.shlv_u16x32(self, rhs) } } -impl core::ops::ShlAssign for u8x64 { +impl core::ops::ShlAssign for u16x32 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u8x64(*self, rhs); + *self = self.simd.shlv_u16x32(*self, rhs); } } -impl core::ops::Shr for u8x64 { +impl core::ops::Shr for u16x32 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u8x64(self, rhs) + self.simd.shr_u16x32(self, rhs) } } -impl core::ops::ShrAssign for u8x64 { +impl core::ops::ShrAssign for u16x32 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u8x64(*self, rhs); + *self = self.simd.shr_u16x32(*self, rhs); } } -impl core::ops::Shr for u8x64 { +impl core::ops::Shr for u16x32 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u8x64(self, rhs) + self.simd.shrv_u16x32(self, rhs) } } -impl core::ops::ShrAssign for u8x64 { +impl core::ops::ShrAssign for u16x32 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u8x64(*self, rhs); + *self = self.simd.shrv_u16x32(*self, rhs); } } -impl core::ops::BitAnd for mask8x64 { +impl core::ops::BitAnd for mask16x32 { type Output = Self; #[doc = "Compute the logical AND of two masks."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask8x64(self, rhs) + self.simd.and_mask16x32(self, rhs) } } -impl core::ops::BitAndAssign for mask8x64 { +impl core::ops::BitAndAssign for mask16x32 { #[doc = "Compute the logical AND of two masks."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask8x64(*self, rhs); + *self = self.simd.and_mask16x32(*self, rhs); } } -impl core::ops::BitOr for mask8x64 { +impl core::ops::BitOr for mask16x32 { type Output = Self; #[doc = "Compute the logical OR of two masks."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask8x64(self, rhs) + self.simd.or_mask16x32(self, rhs) } } -impl core::ops::BitOrAssign for mask8x64 { +impl core::ops::BitOrAssign for mask16x32 { #[doc = "Compute the logical OR of two masks."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask8x64(*self, rhs); + *self = self.simd.or_mask16x32(*self, rhs); } } -impl core::ops::BitXor for mask8x64 { +impl core::ops::BitXor for mask16x32 { type Output = Self; #[doc = "Compute the logical XOR of two masks."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask8x64(self, rhs) + self.simd.xor_mask16x32(self, rhs) } } -impl core::ops::BitXorAssign for mask8x64 { +impl core::ops::BitXorAssign for mask16x32 { #[doc = "Compute the logical XOR of two masks."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask8x64(*self, rhs); + *self = self.simd.xor_mask16x32(*self, rhs); } } -impl core::ops::Not for mask8x64 { +impl core::ops::Not for mask16x32 { type Output = Self; #[doc = "Compute the logical NOT of the mask."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_mask8x64(self) + self.simd.not_mask16x32(self) } } -impl core::ops::Neg for i16x32 { +impl core::ops::Neg for i32x16 { type Output = Self; #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] fn neg(self) -> Self::Output { - self.simd.neg_i16x32(self) + self.simd.neg_i32x16(self) } } -impl core::ops::Add for i16x32 { +impl core::ops::Add for i32x16 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i16x32(self, rhs) + self.simd.add_i32x16(self, rhs) } } -impl core::ops::AddAssign for i16x32 { +impl core::ops::AddAssign for i32x16 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i16x32(*self, rhs); + *self = self.simd.add_i32x16(*self, rhs); } } -impl core::ops::Add for i16x32 { +impl core::ops::Add for i32x16 { type Output = Self; #[inline(always)] - fn add(self, rhs: i16) -> Self::Output { - self.simd.add_i16x32(self, rhs.simd_into(self.simd)) + fn add(self, rhs: i32) -> Self::Output { + self.simd.add_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for i16x32 { +impl core::ops::AddAssign for i32x16 { #[inline(always)] - fn add_assign(&mut self, rhs: i16) { - *self = self.simd.add_i16x32(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: i32) { + *self = self.simd.add_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for i16 { - type Output = i16x32; +impl core::ops::Add> for i32 { + type Output = i32x16; #[inline(always)] - fn add(self, rhs: i16x32) -> Self::Output { - rhs.simd.add_i16x32(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: i32x16) -> Self::Output { + rhs.simd.add_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for i16x32 { +impl core::ops::Sub for i32x16 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i16x32(self, rhs) + self.simd.sub_i32x16(self, rhs) } } -impl core::ops::SubAssign for i16x32 { +impl core::ops::SubAssign for i32x16 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i16x32(*self, rhs); + *self = self.simd.sub_i32x16(*self, rhs); } } -impl core::ops::Sub for i16x32 { +impl core::ops::Sub for i32x16 { type Output = Self; #[inline(always)] - fn sub(self, rhs: i16) -> Self::Output { - self.simd.sub_i16x32(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: i32) -> Self::Output { + self.simd.sub_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for i16x32 { +impl core::ops::SubAssign for i32x16 { #[inline(always)] - fn sub_assign(&mut self, rhs: i16) { - *self = self.simd.sub_i16x32(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: i32) { + *self = self.simd.sub_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for i16 { - type Output = i16x32; +impl core::ops::Sub> for i32 { + type Output = i32x16; #[inline(always)] - fn sub(self, rhs: i16x32) -> Self::Output { - rhs.simd.sub_i16x32(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: i32x16) -> Self::Output { + rhs.simd.sub_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for i16x32 { +impl core::ops::Mul for i32x16 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i16x32(self, rhs) + self.simd.mul_i32x16(self, rhs) } } -impl core::ops::MulAssign for i16x32 { +impl core::ops::MulAssign for i32x16 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i16x32(*self, rhs); + *self = self.simd.mul_i32x16(*self, rhs); } } -impl core::ops::Mul for i16x32 { +impl core::ops::Mul for i32x16 { type Output = Self; #[inline(always)] - fn mul(self, rhs: i16) -> Self::Output { - self.simd.mul_i16x32(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: i32) -> Self::Output { + self.simd.mul_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for i16x32 { +impl core::ops::MulAssign for i32x16 { #[inline(always)] - fn mul_assign(&mut self, rhs: i16) { - *self = self.simd.mul_i16x32(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: i32) { + *self = self.simd.mul_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for i16 { - type Output = i16x32; +impl core::ops::Mul> for i32 { + type Output = i32x16; #[inline(always)] - fn mul(self, rhs: i16x32) -> Self::Output { - rhs.simd.mul_i16x32(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: i32x16) -> Self::Output { + rhs.simd.mul_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for i16x32 { +impl core::ops::BitAnd for i32x16 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i16x32(self, rhs) + self.simd.and_i32x16(self, rhs) } } -impl core::ops::BitAndAssign for i16x32 { +impl core::ops::BitAndAssign for i32x16 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i16x32(*self, rhs); + *self = self.simd.and_i32x16(*self, rhs); } } -impl core::ops::BitAnd for i16x32 { +impl core::ops::BitAnd for i32x16 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: i16) -> Self::Output { - self.simd.and_i16x32(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: i32) -> Self::Output { + self.simd.and_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for i16x32 { +impl core::ops::BitAndAssign for i32x16 { #[inline(always)] - fn bitand_assign(&mut self, rhs: i16) { - *self = self.simd.and_i16x32(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: i32) { + *self = self.simd.and_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for i16 { - type Output = i16x32; +impl core::ops::BitAnd> for i32 { + type Output = i32x16; #[inline(always)] - fn bitand(self, rhs: i16x32) -> Self::Output { - rhs.simd.and_i16x32(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i32x16) -> Self::Output { + rhs.simd.and_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for i16x32 { +impl core::ops::BitOr for i32x16 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i16x32(self, rhs) + self.simd.or_i32x16(self, rhs) } } -impl core::ops::BitOrAssign for i16x32 { +impl core::ops::BitOrAssign for i32x16 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i16x32(*self, rhs); + *self = self.simd.or_i32x16(*self, rhs); } } -impl core::ops::BitOr for i16x32 { +impl core::ops::BitOr for i32x16 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: i16) -> Self::Output { - self.simd.or_i16x32(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i32) -> Self::Output { + self.simd.or_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i16x32 { +impl core::ops::BitOrAssign for i32x16 { #[inline(always)] - fn bitor_assign(&mut self, rhs: i16) { - *self = self.simd.or_i16x32(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i32) { + *self = self.simd.or_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for i16 { - type Output = i16x32; +impl core::ops::BitOr> for i32 { + type Output = i32x16; #[inline(always)] - fn bitor(self, rhs: i16x32) -> Self::Output { - rhs.simd.or_i16x32(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i32x16) -> Self::Output { + rhs.simd.or_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for i16x32 { +impl core::ops::BitXor for i32x16 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i16x32(self, rhs) + self.simd.xor_i32x16(self, rhs) } } -impl core::ops::BitXorAssign for i16x32 { +impl core::ops::BitXorAssign for i32x16 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i16x32(*self, rhs); + *self = self.simd.xor_i32x16(*self, rhs); } } -impl core::ops::BitXor for i16x32 { +impl core::ops::BitXor for i32x16 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: i16) -> Self::Output { - self.simd.xor_i16x32(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i32) -> Self::Output { + self.simd.xor_i32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i16x32 { +impl core::ops::BitXorAssign for i32x16 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: i16) { - *self = self.simd.xor_i16x32(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i32) { + *self = self.simd.xor_i32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for i16 { - type Output = i16x32; +impl core::ops::BitXor> for i32 { + type Output = i32x16; #[inline(always)] - fn bitxor(self, rhs: i16x32) -> Self::Output { - rhs.simd.xor_i16x32(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i32x16) -> Self::Output { + rhs.simd.xor_i32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for i16x32 { +impl core::ops::Not for i32x16 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_i16x32(self) + self.simd.not_i32x16(self) } } -impl core::ops::Shl for i16x32 { +impl core::ops::Shl for i32x16 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i16x32(self, rhs) + self.simd.shl_i32x16(self, rhs) } } -impl core::ops::ShlAssign for i16x32 { +impl core::ops::ShlAssign for i32x16 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i16x32(*self, rhs); + *self = self.simd.shl_i32x16(*self, rhs); } } -impl core::ops::Shl for i16x32 { +impl core::ops::Shl for i32x16 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i16x32(self, rhs) + self.simd.shlv_i32x16(self, rhs) } } -impl core::ops::ShlAssign for i16x32 { +impl core::ops::ShlAssign for i32x16 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i16x32(*self, rhs); + *self = self.simd.shlv_i32x16(*self, rhs); } } -impl core::ops::Shr for i16x32 { +impl core::ops::Shr for i32x16 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i16x32(self, rhs) + self.simd.shr_i32x16(self, rhs) } } -impl core::ops::ShrAssign for i16x32 { +impl core::ops::ShrAssign for i32x16 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i16x32(*self, rhs); + *self = self.simd.shr_i32x16(*self, rhs); } } -impl core::ops::Shr for i16x32 { +impl core::ops::Shr for i32x16 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i16x32(self, rhs) + self.simd.shrv_i32x16(self, rhs) } } -impl core::ops::ShrAssign for i16x32 { +impl core::ops::ShrAssign for i32x16 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i16x32(*self, rhs); + *self = self.simd.shrv_i32x16(*self, rhs); } } -impl core::ops::Add for u16x32 { +impl core::ops::Add for u32x16 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u16x32(self, rhs) + self.simd.add_u32x16(self, rhs) } } -impl core::ops::AddAssign for u16x32 { +impl core::ops::AddAssign for u32x16 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u16x32(*self, rhs); + *self = self.simd.add_u32x16(*self, rhs); } } -impl core::ops::Add for u16x32 { +impl core::ops::Add for u32x16 { type Output = Self; #[inline(always)] - fn add(self, rhs: u16) -> Self::Output { - self.simd.add_u16x32(self, rhs.simd_into(self.simd)) + fn add(self, rhs: u32) -> Self::Output { + self.simd.add_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for u16x32 { +impl core::ops::AddAssign for u32x16 { #[inline(always)] - fn add_assign(&mut self, rhs: u16) { - *self = self.simd.add_u16x32(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: u32) { + *self = self.simd.add_u32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for u16 { - type Output = u16x32; +impl core::ops::Add> for u32 { + type Output = u32x16; #[inline(always)] - fn add(self, rhs: u16x32) -> Self::Output { - rhs.simd.add_u16x32(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: u32x16) -> Self::Output { + rhs.simd.add_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for u16x32 { +impl core::ops::Sub for u32x16 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u16x32(self, rhs) + self.simd.sub_u32x16(self, rhs) } } -impl core::ops::SubAssign for u16x32 { +impl core::ops::SubAssign for u32x16 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u16x32(*self, rhs); + *self = self.simd.sub_u32x16(*self, rhs); } } -impl core::ops::Sub for u16x32 { +impl core::ops::Sub for u32x16 { type Output = Self; #[inline(always)] - fn sub(self, rhs: u16) -> Self::Output { - self.simd.sub_u16x32(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: u32) -> Self::Output { + self.simd.sub_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for u16x32 { +impl core::ops::SubAssign for u32x16 { #[inline(always)] - fn sub_assign(&mut self, rhs: u16) { - *self = self.simd.sub_u16x32(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: u32) { + *self = self.simd.sub_u32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for u16 { - type Output = u16x32; +impl core::ops::Sub> for u32 { + type Output = u32x16; #[inline(always)] - fn sub(self, rhs: u16x32) -> Self::Output { - rhs.simd.sub_u16x32(self.simd_into(rhs.simd), rhs) + fn sub(self, rhs: u32x16) -> Self::Output { + rhs.simd.sub_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for u16x32 { +impl core::ops::Mul for u32x16 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u16x32(self, rhs) + self.simd.mul_u32x16(self, rhs) } } -impl core::ops::MulAssign for u16x32 { +impl core::ops::MulAssign for u32x16 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u16x32(*self, rhs); + *self = self.simd.mul_u32x16(*self, rhs); } } -impl core::ops::Mul for u16x32 { +impl core::ops::Mul for u32x16 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u16) -> Self::Output { - self.simd.mul_u16x32(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: u32) -> Self::Output { + self.simd.mul_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u16x32 { +impl core::ops::MulAssign for u32x16 { #[inline(always)] - fn mul_assign(&mut self, rhs: u16) { - *self = self.simd.mul_u16x32(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: u32) { + *self = self.simd.mul_u32x16(*self, rhs.simd_into(self.simd)); } -} -impl core::ops::Mul> for u16 { - type Output = u16x32; +} +impl core::ops::Mul> for u32 { + type Output = u32x16; #[inline(always)] - fn mul(self, rhs: u16x32) -> Self::Output { - rhs.simd.mul_u16x32(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: u32x16) -> Self::Output { + rhs.simd.mul_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u16x32 { +impl core::ops::BitAnd for u32x16 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u16x32(self, rhs) + self.simd.and_u32x16(self, rhs) } } -impl core::ops::BitAndAssign for u16x32 { +impl core::ops::BitAndAssign for u32x16 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u16x32(*self, rhs); + *self = self.simd.and_u32x16(*self, rhs); } } -impl core::ops::BitAnd for u16x32 { +impl core::ops::BitAnd for u32x16 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u16) -> Self::Output { - self.simd.and_u16x32(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: u32) -> Self::Output { + self.simd.and_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u16x32 { +impl core::ops::BitAndAssign for u32x16 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u16) { - *self = self.simd.and_u16x32(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: u32) { + *self = self.simd.and_u32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u16 { - type Output = u16x32; +impl core::ops::BitAnd> for u32 { + type Output = u32x16; #[inline(always)] - fn bitand(self, rhs: u16x32) -> Self::Output { - rhs.simd.and_u16x32(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u32x16) -> Self::Output { + rhs.simd.and_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u16x32 { +impl core::ops::BitOr for u32x16 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u16x32(self, rhs) + self.simd.or_u32x16(self, rhs) } } -impl core::ops::BitOrAssign for u16x32 { +impl core::ops::BitOrAssign for u32x16 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u16x32(*self, rhs); + *self = self.simd.or_u32x16(*self, rhs); } } -impl core::ops::BitOr for u16x32 { +impl core::ops::BitOr for u32x16 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u16) -> Self::Output { - self.simd.or_u16x32(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: u32) -> Self::Output { + self.simd.or_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u16x32 { +impl core::ops::BitOrAssign for u32x16 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u16) { - *self = self.simd.or_u16x32(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: u32) { + *self = self.simd.or_u32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u16 { - type Output = u16x32; +impl core::ops::BitOr> for u32 { + type Output = u32x16; #[inline(always)] - fn bitor(self, rhs: u16x32) -> Self::Output { - rhs.simd.or_u16x32(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: u32x16) -> Self::Output { + rhs.simd.or_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u16x32 { +impl core::ops::BitXor for u32x16 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u16x32(self, rhs) + self.simd.xor_u32x16(self, rhs) } } -impl core::ops::BitXorAssign for u16x32 { +impl core::ops::BitXorAssign for u32x16 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u16x32(*self, rhs); + *self = self.simd.xor_u32x16(*self, rhs); } } -impl core::ops::BitXor for u16x32 { +impl core::ops::BitXor for u32x16 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u16) -> Self::Output { - self.simd.xor_u16x32(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: u32) -> Self::Output { + self.simd.xor_u32x16(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u16x32 { +impl core::ops::BitXorAssign for u32x16 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u16) { - *self = self.simd.xor_u16x32(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: u32) { + *self = self.simd.xor_u32x16(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u16 { - type Output = u16x32; +impl core::ops::BitXor> for u32 { + type Output = u32x16; #[inline(always)] - fn bitxor(self, rhs: u16x32) -> Self::Output { - rhs.simd.xor_u16x32(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: u32x16) -> Self::Output { + rhs.simd.xor_u32x16(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u16x32 { +impl core::ops::Not for u32x16 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u16x32(self) + self.simd.not_u32x16(self) } } -impl core::ops::Shl for u16x32 { +impl core::ops::Shl for u32x16 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u16x32(self, rhs) + self.simd.shl_u32x16(self, rhs) } } -impl core::ops::ShlAssign for u16x32 { +impl core::ops::ShlAssign for u32x16 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u16x32(*self, rhs); + *self = self.simd.shl_u32x16(*self, rhs); } } -impl core::ops::Shl for u16x32 { +impl core::ops::Shl for u32x16 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u16x32(self, rhs) + self.simd.shlv_u32x16(self, rhs) } } -impl core::ops::ShlAssign for u16x32 { +impl core::ops::ShlAssign for u32x16 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u16x32(*self, rhs); + *self = self.simd.shlv_u32x16(*self, rhs); } } -impl core::ops::Shr for u16x32 { +impl core::ops::Shr for u32x16 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u16x32(self, rhs) + self.simd.shr_u32x16(self, rhs) } } -impl core::ops::ShrAssign for u16x32 { +impl core::ops::ShrAssign for u32x16 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u16x32(*self, rhs); + *self = self.simd.shr_u32x16(*self, rhs); } } -impl core::ops::Shr for u16x32 { +impl core::ops::Shr for u32x16 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u16x32(self, rhs) + self.simd.shrv_u32x16(self, rhs) } } -impl core::ops::ShrAssign for u16x32 { +impl core::ops::ShrAssign for u32x16 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u16x32(*self, rhs); - } -} -impl core::ops::BitAnd for mask16x32 { - type Output = Self; - #[doc = "Compute the logical AND of two masks."] - #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask16x32(self, rhs) - } -} -impl core::ops::BitAndAssign for mask16x32 { - #[doc = "Compute the logical AND of two masks."] - #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask16x32(*self, rhs); - } -} -impl core::ops::BitOr for mask16x32 { - type Output = Self; - #[doc = "Compute the logical OR of two masks."] - #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask16x32(self, rhs) - } -} -impl core::ops::BitOrAssign for mask16x32 { - #[doc = "Compute the logical OR of two masks."] - #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask16x32(*self, rhs); - } -} -impl core::ops::BitXor for mask16x32 { - type Output = Self; - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask16x32(self, rhs) - } -} -impl core::ops::BitXorAssign for mask16x32 { - #[doc = "Compute the logical XOR of two masks."] - #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask16x32(*self, rhs); - } -} -impl core::ops::Not for mask16x32 { - type Output = Self; - #[doc = "Compute the logical NOT of the mask."] - #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask16x32(self) - } -} -impl core::ops::Neg for i32x16 { - type Output = Self; - #[doc = "Negate each element of the vector, wrapping on overflow."] - #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_i32x16(self) - } -} -impl core::ops::Add for i32x16 { - type Output = Self; - #[doc = "Add two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_i32x16(self, rhs) - } -} -impl core::ops::AddAssign for i32x16 { - #[doc = "Add two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_i32x16(*self, rhs); - } -} -impl core::ops::Add for i32x16 { - type Output = Self; - #[inline(always)] - fn add(self, rhs: i32) -> Self::Output { - self.simd.add_i32x16(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::AddAssign for i32x16 { - #[inline(always)] - fn add_assign(&mut self, rhs: i32) { - *self = self.simd.add_i32x16(*self, rhs.simd_into(self.simd)); - } -} -impl core::ops::Add> for i32 { - type Output = i32x16; - #[inline(always)] - fn add(self, rhs: i32x16) -> Self::Output { - rhs.simd.add_i32x16(self.simd_into(rhs.simd), rhs) - } -} -impl core::ops::Sub for i32x16 { - type Output = Self; - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_i32x16(self, rhs) - } -} -impl core::ops::SubAssign for i32x16 { - #[doc = "Subtract two vectors element-wise, wrapping on overflow."] - #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_i32x16(*self, rhs); - } -} -impl core::ops::Sub for i32x16 { - type Output = Self; - #[inline(always)] - fn sub(self, rhs: i32) -> Self::Output { - self.simd.sub_i32x16(self, rhs.simd_into(self.simd)) - } -} -impl core::ops::SubAssign for i32x16 { - #[inline(always)] - fn sub_assign(&mut self, rhs: i32) { - *self = self.simd.sub_i32x16(*self, rhs.simd_into(self.simd)); - } -} -impl core::ops::Sub> for i32 { - type Output = i32x16; - #[inline(always)] - fn sub(self, rhs: i32x16) -> Self::Output { - rhs.simd.sub_i32x16(self.simd_into(rhs.simd), rhs) + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u32x16(*self, rhs); } } -impl core::ops::Mul for i32x16 { +impl core::ops::BitAnd for mask32x16 { type Output = Self; - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_i32x16(self, rhs) + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_mask32x16(self, rhs) } } -impl core::ops::MulAssign for i32x16 { - #[doc = "Multiply two vectors element-wise, wrapping on overflow."] +impl core::ops::BitAndAssign for mask32x16 { + #[doc = "Compute the logical AND of two masks."] #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_i32x16(*self, rhs); + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_mask32x16(*self, rhs); } } -impl core::ops::Mul for i32x16 { +impl core::ops::BitOr for mask32x16 { type Output = Self; + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn mul(self, rhs: i32) -> Self::Output { - self.simd.mul_i32x16(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_mask32x16(self, rhs) } } -impl core::ops::MulAssign for i32x16 { +impl core::ops::BitOrAssign for mask32x16 { + #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn mul_assign(&mut self, rhs: i32) { - *self = self.simd.mul_i32x16(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_mask32x16(*self, rhs); } } -impl core::ops::Mul> for i32 { - type Output = i32x16; +impl core::ops::BitXor for mask32x16 { + type Output = Self; + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn mul(self, rhs: i32x16) -> Self::Output { - rhs.simd.mul_i32x16(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_mask32x16(self, rhs) } } -impl core::ops::BitAnd for i32x16 { - type Output = Self; - #[doc = "Compute the bitwise AND of two vectors."] +impl core::ops::BitXorAssign for mask32x16 { + #[doc = "Compute the logical XOR of two masks."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_i32x16(self, rhs) + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_mask32x16(*self, rhs); } } -impl core::ops::BitAndAssign for i32x16 { - #[doc = "Compute the bitwise AND of two vectors."] +impl core::ops::Not for mask32x16 { + type Output = Self; + #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_i32x16(*self, rhs); + fn not(self) -> Self::Output { + self.simd.not_mask32x16(self) } } -impl core::ops::BitAnd for i32x16 { +impl core::ops::Neg for f64x8 { type Output = Self; + #[doc = "Negate each element of the vector."] #[inline(always)] - fn bitand(self, rhs: i32) -> Self::Output { - self.simd.and_i32x16(self, rhs.simd_into(self.simd)) + fn neg(self) -> Self::Output { + self.simd.neg_f64x8(self) } } -impl core::ops::BitAndAssign for i32x16 { +impl core::ops::Add for f64x8 { + type Output = Self; + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn bitand_assign(&mut self, rhs: i32) { - *self = self.simd.and_i32x16(*self, rhs.simd_into(self.simd)); + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_f64x8(self, rhs) } } -impl core::ops::BitAnd> for i32 { - type Output = i32x16; +impl core::ops::AddAssign for f64x8 { + #[doc = "Add two vectors element-wise."] #[inline(always)] - fn bitand(self, rhs: i32x16) -> Self::Output { - rhs.simd.and_i32x16(self.simd_into(rhs.simd), rhs) + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_f64x8(*self, rhs); } } -impl core::ops::BitOr for i32x16 { +impl core::ops::Add for f64x8 { type Output = Self; - #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_i32x16(self, rhs) + fn add(self, rhs: f64) -> Self::Output { + self.simd.add_f64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for i32x16 { - #[doc = "Compute the bitwise OR of two vectors."] +impl core::ops::AddAssign for f64x8 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_i32x16(*self, rhs); + fn add_assign(&mut self, rhs: f64) { + *self = self.simd.add_f64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr for i32x16 { - type Output = Self; +impl core::ops::Add> for f64 { + type Output = f64x8; #[inline(always)] - fn bitor(self, rhs: i32) -> Self::Output { - self.simd.or_i32x16(self, rhs.simd_into(self.simd)) + fn add(self, rhs: f64x8) -> Self::Output { + rhs.simd.add_f64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOrAssign for i32x16 { +impl core::ops::Sub for f64x8 { + type Output = Self; + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitor_assign(&mut self, rhs: i32) { - *self = self.simd.or_i32x16(*self, rhs.simd_into(self.simd)); + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_f64x8(self, rhs) } } -impl core::ops::BitOr> for i32 { - type Output = i32x16; +impl core::ops::SubAssign for f64x8 { + #[doc = "Subtract two vectors element-wise."] #[inline(always)] - fn bitor(self, rhs: i32x16) -> Self::Output { - rhs.simd.or_i32x16(self.simd_into(rhs.simd), rhs) + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_f64x8(*self, rhs); } } -impl core::ops::BitXor for i32x16 { +impl core::ops::Sub for f64x8 { type Output = Self; - #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_i32x16(self, rhs) + fn sub(self, rhs: f64) -> Self::Output { + self.simd.sub_f64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for i32x16 { - #[doc = "Compute the bitwise XOR of two vectors."] +impl core::ops::SubAssign for f64x8 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_i32x16(*self, rhs); + fn sub_assign(&mut self, rhs: f64) { + *self = self.simd.sub_f64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for i32x16 { - type Output = Self; +impl core::ops::Sub> for f64 { + type Output = f64x8; #[inline(always)] - fn bitxor(self, rhs: i32) -> Self::Output { - self.simd.xor_i32x16(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: f64x8) -> Self::Output { + rhs.simd.sub_f64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXorAssign for i32x16 { +impl core::ops::Mul for f64x8 { + type Output = Self; + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: i32) { - *self = self.simd.xor_i32x16(*self, rhs.simd_into(self.simd)); + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_f64x8(self, rhs) } } -impl core::ops::BitXor> for i32 { - type Output = i32x16; +impl core::ops::MulAssign for f64x8 { + #[doc = "Multiply two vectors element-wise."] #[inline(always)] - fn bitxor(self, rhs: i32x16) -> Self::Output { - rhs.simd.xor_i32x16(self.simd_into(rhs.simd), rhs) + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_f64x8(*self, rhs); } } -impl core::ops::Not for i32x16 { +impl core::ops::Mul for f64x8 { type Output = Self; - #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_i32x16(self) + fn mul(self, rhs: f64) -> Self::Output { + self.simd.mul_f64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::Shl for i32x16 { - type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] +impl core::ops::MulAssign for f64x8 { #[inline(always)] - fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_i32x16(self, rhs) + fn mul_assign(&mut self, rhs: f64) { + *self = self.simd.mul_f64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::ShlAssign for i32x16 { +impl core::ops::Mul> for f64 { + type Output = f64x8; #[inline(always)] - fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_i32x16(*self, rhs); + fn mul(self, rhs: f64x8) -> Self::Output { + rhs.simd.mul_f64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Shl for i32x16 { +impl core::ops::Div for f64x8 { type Output = Self; - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_i32x16(self, rhs) + fn div(self, rhs: Self) -> Self::Output { + self.simd.div_f64x8(self, rhs) } } -impl core::ops::ShlAssign for i32x16 { - #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::DivAssign for f64x8 { + #[doc = "Divide two vectors element-wise."] #[inline(always)] - fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_i32x16(*self, rhs); + fn div_assign(&mut self, rhs: Self) { + *self = self.simd.div_f64x8(*self, rhs); } } -impl core::ops::Shr for i32x16 { +impl core::ops::Div for f64x8 { type Output = Self; - #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] - fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_i32x16(self, rhs) + fn div(self, rhs: f64) -> Self::Output { + self.simd.div_f64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::ShrAssign for i32x16 { +impl core::ops::DivAssign for f64x8 { #[inline(always)] - fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_i32x16(*self, rhs); + fn div_assign(&mut self, rhs: f64) { + *self = self.simd.div_f64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Shr for i32x16 { - type Output = Self; - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::Div> for f64 { + type Output = f64x8; #[inline(always)] - fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_i32x16(self, rhs) + fn div(self, rhs: f64x8) -> Self::Output { + rhs.simd.div_f64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::ShrAssign for i32x16 { - #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] +impl core::ops::Neg for i64x8 { + type Output = Self; + #[doc = "Negate each element of the vector, wrapping on overflow."] #[inline(always)] - fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_i32x16(*self, rhs); + fn neg(self) -> Self::Output { + self.simd.neg_i64x8(self) } } -impl core::ops::Add for u32x16 { +impl core::ops::Add for i64x8 { type Output = Self; #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add(self, rhs: Self) -> Self::Output { - self.simd.add_u32x16(self, rhs) + self.simd.add_i64x8(self, rhs) } } -impl core::ops::AddAssign for u32x16 { +impl core::ops::AddAssign for i64x8 { #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_u32x16(*self, rhs); + *self = self.simd.add_i64x8(*self, rhs); } } -impl core::ops::Add for u32x16 { +impl core::ops::Add for i64x8 { type Output = Self; #[inline(always)] - fn add(self, rhs: u32) -> Self::Output { - self.simd.add_u32x16(self, rhs.simd_into(self.simd)) + fn add(self, rhs: i64) -> Self::Output { + self.simd.add_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for u32x16 { +impl core::ops::AddAssign for i64x8 { #[inline(always)] - fn add_assign(&mut self, rhs: u32) { - *self = self.simd.add_u32x16(*self, rhs.simd_into(self.simd)); + fn add_assign(&mut self, rhs: i64) { + *self = self.simd.add_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add> for u32 { - type Output = u32x16; +impl core::ops::Add> for i64 { + type Output = i64x8; #[inline(always)] - fn add(self, rhs: u32x16) -> Self::Output { - rhs.simd.add_u32x16(self.simd_into(rhs.simd), rhs) + fn add(self, rhs: i64x8) -> Self::Output { + rhs.simd.add_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Sub for u32x16 { +impl core::ops::Sub for i64x8 { type Output = Self; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_u32x16(self, rhs) + self.simd.sub_i64x8(self, rhs) } } -impl core::ops::SubAssign for u32x16 { +impl core::ops::SubAssign for i64x8 { #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_u32x16(*self, rhs); + *self = self.simd.sub_i64x8(*self, rhs); } } -impl core::ops::Sub for u32x16 { +impl core::ops::Sub for i64x8 { type Output = Self; #[inline(always)] - fn sub(self, rhs: u32) -> Self::Output { - self.simd.sub_u32x16(self, rhs.simd_into(self.simd)) + fn sub(self, rhs: i64) -> Self::Output { + self.simd.sub_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for u32x16 { +impl core::ops::SubAssign for i64x8 { #[inline(always)] - fn sub_assign(&mut self, rhs: u32) { - *self = self.simd.sub_u32x16(*self, rhs.simd_into(self.simd)); + fn sub_assign(&mut self, rhs: i64) { + *self = self.simd.sub_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for u32 { - type Output = u32x16; - #[inline(always)] - fn sub(self, rhs: u32x16) -> Self::Output { - rhs.simd.sub_u32x16(self.simd_into(rhs.simd), rhs) +impl core::ops::Sub> for i64 { + type Output = i64x8; + #[inline(always)] + fn sub(self, rhs: i64x8) -> Self::Output { + rhs.simd.sub_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for u32x16 { +impl core::ops::Mul for i64x8 { type Output = Self; #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_u32x16(self, rhs) + self.simd.mul_i64x8(self, rhs) } } -impl core::ops::MulAssign for u32x16 { +impl core::ops::MulAssign for i64x8 { #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_u32x16(*self, rhs); + *self = self.simd.mul_i64x8(*self, rhs); } } -impl core::ops::Mul for u32x16 { +impl core::ops::Mul for i64x8 { type Output = Self; #[inline(always)] - fn mul(self, rhs: u32) -> Self::Output { - self.simd.mul_u32x16(self, rhs.simd_into(self.simd)) + fn mul(self, rhs: i64) -> Self::Output { + self.simd.mul_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for u32x16 { +impl core::ops::MulAssign for i64x8 { #[inline(always)] - fn mul_assign(&mut self, rhs: u32) { - *self = self.simd.mul_u32x16(*self, rhs.simd_into(self.simd)); + fn mul_assign(&mut self, rhs: i64) { + *self = self.simd.mul_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for u32 { - type Output = u32x16; +impl core::ops::Mul> for i64 { + type Output = i64x8; #[inline(always)] - fn mul(self, rhs: u32x16) -> Self::Output { - rhs.simd.mul_u32x16(self.simd_into(rhs.simd), rhs) + fn mul(self, rhs: i64x8) -> Self::Output { + rhs.simd.mul_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitAnd for u32x16 { +impl core::ops::BitAnd for i64x8 { type Output = Self; #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_u32x16(self, rhs) + self.simd.and_i64x8(self, rhs) } } -impl core::ops::BitAndAssign for u32x16 { +impl core::ops::BitAndAssign for i64x8 { #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_u32x16(*self, rhs); + *self = self.simd.and_i64x8(*self, rhs); } } -impl core::ops::BitAnd for u32x16 { +impl core::ops::BitAnd for i64x8 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: u32) -> Self::Output { - self.simd.and_u32x16(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: i64) -> Self::Output { + self.simd.and_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitAndAssign for u32x16 { +impl core::ops::BitAndAssign for i64x8 { #[inline(always)] - fn bitand_assign(&mut self, rhs: u32) { - *self = self.simd.and_u32x16(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: i64) { + *self = self.simd.and_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitAnd> for u32 { - type Output = u32x16; +impl core::ops::BitAnd> for i64 { + type Output = i64x8; #[inline(always)] - fn bitand(self, rhs: u32x16) -> Self::Output { - rhs.simd.and_u32x16(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: i64x8) -> Self::Output { + rhs.simd.and_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitOr for u32x16 { +impl core::ops::BitOr for i64x8 { type Output = Self; #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_u32x16(self, rhs) + self.simd.or_i64x8(self, rhs) } } -impl core::ops::BitOrAssign for u32x16 { +impl core::ops::BitOrAssign for i64x8 { #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_u32x16(*self, rhs); + *self = self.simd.or_i64x8(*self, rhs); } } -impl core::ops::BitOr for u32x16 { +impl core::ops::BitOr for i64x8 { type Output = Self; #[inline(always)] - fn bitor(self, rhs: u32) -> Self::Output { - self.simd.or_u32x16(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: i64) -> Self::Output { + self.simd.or_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for u32x16 { +impl core::ops::BitOrAssign for i64x8 { #[inline(always)] - fn bitor_assign(&mut self, rhs: u32) { - *self = self.simd.or_u32x16(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: i64) { + *self = self.simd.or_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitOr> for u32 { - type Output = u32x16; +impl core::ops::BitOr> for i64 { + type Output = i64x8; #[inline(always)] - fn bitor(self, rhs: u32x16) -> Self::Output { - rhs.simd.or_u32x16(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: i64x8) -> Self::Output { + rhs.simd.or_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::BitXor for u32x16 { +impl core::ops::BitXor for i64x8 { type Output = Self; #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_u32x16(self, rhs) + self.simd.xor_i64x8(self, rhs) } } -impl core::ops::BitXorAssign for u32x16 { +impl core::ops::BitXorAssign for i64x8 { #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_u32x16(*self, rhs); + *self = self.simd.xor_i64x8(*self, rhs); } } -impl core::ops::BitXor for u32x16 { +impl core::ops::BitXor for i64x8 { type Output = Self; #[inline(always)] - fn bitxor(self, rhs: u32) -> Self::Output { - self.simd.xor_u32x16(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: i64) -> Self::Output { + self.simd.xor_i64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitXorAssign for u32x16 { +impl core::ops::BitXorAssign for i64x8 { #[inline(always)] - fn bitxor_assign(&mut self, rhs: u32) { - *self = self.simd.xor_u32x16(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: i64) { + *self = self.simd.xor_i64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor> for u32 { - type Output = u32x16; +impl core::ops::BitXor> for i64 { + type Output = i64x8; #[inline(always)] - fn bitxor(self, rhs: u32x16) -> Self::Output { - rhs.simd.xor_u32x16(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: i64x8) -> Self::Output { + rhs.simd.xor_i64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Not for u32x16 { +impl core::ops::Not for i64x8 { type Output = Self; #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] fn not(self) -> Self::Output { - self.simd.not_u32x16(self) + self.simd.not_i64x8(self) } } -impl core::ops::Shl for u32x16 { +impl core::ops::Shl for i64x8 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] fn shl(self, rhs: u32) -> Self::Output { - self.simd.shl_u32x16(self, rhs) + self.simd.shl_i64x8(self, rhs) } } -impl core::ops::ShlAssign for u32x16 { +impl core::ops::ShlAssign for i64x8 { #[inline(always)] fn shl_assign(&mut self, rhs: u32) { - *self = self.simd.shl_u32x16(*self, rhs); + *self = self.simd.shl_i64x8(*self, rhs); } } -impl core::ops::Shl for u32x16 { +impl core::ops::Shl for i64x8 { type Output = Self; #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl(self, rhs: Self) -> Self::Output { - self.simd.shlv_u32x16(self, rhs) + self.simd.shlv_i64x8(self, rhs) } } -impl core::ops::ShlAssign for u32x16 { +impl core::ops::ShlAssign for i64x8 { #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shl_assign(&mut self, rhs: Self) { - *self = self.simd.shlv_u32x16(*self, rhs); + *self = self.simd.shlv_i64x8(*self, rhs); } } -impl core::ops::Shr for u32x16 { +impl core::ops::Shr for i64x8 { type Output = Self; #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] fn shr(self, rhs: u32) -> Self::Output { - self.simd.shr_u32x16(self, rhs) + self.simd.shr_i64x8(self, rhs) } } -impl core::ops::ShrAssign for u32x16 { +impl core::ops::ShrAssign for i64x8 { #[inline(always)] fn shr_assign(&mut self, rhs: u32) { - *self = self.simd.shr_u32x16(*self, rhs); + *self = self.simd.shr_i64x8(*self, rhs); } } -impl core::ops::Shr for u32x16 { +impl core::ops::Shr for i64x8 { type Output = Self; #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr(self, rhs: Self) -> Self::Output { - self.simd.shrv_u32x16(self, rhs) + self.simd.shrv_i64x8(self, rhs) } } -impl core::ops::ShrAssign for u32x16 { +impl core::ops::ShrAssign for i64x8 { #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] fn shr_assign(&mut self, rhs: Self) { - *self = self.simd.shrv_u32x16(*self, rhs); + *self = self.simd.shrv_i64x8(*self, rhs); } } -impl core::ops::BitAnd for mask32x16 { +impl core::ops::Add for u64x8 { type Output = Self; - #[doc = "Compute the logical AND of two masks."] + #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitand(self, rhs: Self) -> Self::Output { - self.simd.and_mask32x16(self, rhs) + fn add(self, rhs: Self) -> Self::Output { + self.simd.add_u64x8(self, rhs) } } -impl core::ops::BitAndAssign for mask32x16 { - #[doc = "Compute the logical AND of two masks."] +impl core::ops::AddAssign for u64x8 { + #[doc = "Add two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitand_assign(&mut self, rhs: Self) { - *self = self.simd.and_mask32x16(*self, rhs); + fn add_assign(&mut self, rhs: Self) { + *self = self.simd.add_u64x8(*self, rhs); } } -impl core::ops::BitOr for mask32x16 { +impl core::ops::Add for u64x8 { type Output = Self; - #[doc = "Compute the logical OR of two masks."] #[inline(always)] - fn bitor(self, rhs: Self) -> Self::Output { - self.simd.or_mask32x16(self, rhs) + fn add(self, rhs: u64) -> Self::Output { + self.simd.add_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::BitOrAssign for mask32x16 { - #[doc = "Compute the logical OR of two masks."] +impl core::ops::AddAssign for u64x8 { #[inline(always)] - fn bitor_assign(&mut self, rhs: Self) { - *self = self.simd.or_mask32x16(*self, rhs); + fn add_assign(&mut self, rhs: u64) { + *self = self.simd.add_u64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::BitXor for mask32x16 { +impl core::ops::Add> for u64 { + type Output = u64x8; + #[inline(always)] + fn add(self, rhs: u64x8) -> Self::Output { + rhs.simd.add_u64x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Sub for u64x8 { type Output = Self; - #[doc = "Compute the logical XOR of two masks."] + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitxor(self, rhs: Self) -> Self::Output { - self.simd.xor_mask32x16(self, rhs) + fn sub(self, rhs: Self) -> Self::Output { + self.simd.sub_u64x8(self, rhs) } } -impl core::ops::BitXorAssign for mask32x16 { - #[doc = "Compute the logical XOR of two masks."] +impl core::ops::SubAssign for u64x8 { + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn bitxor_assign(&mut self, rhs: Self) { - *self = self.simd.xor_mask32x16(*self, rhs); + fn sub_assign(&mut self, rhs: Self) { + *self = self.simd.sub_u64x8(*self, rhs); } } -impl core::ops::Not for mask32x16 { +impl core::ops::Sub for u64x8 { type Output = Self; - #[doc = "Compute the logical NOT of the mask."] #[inline(always)] - fn not(self) -> Self::Output { - self.simd.not_mask32x16(self) + fn sub(self, rhs: u64) -> Self::Output { + self.simd.sub_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::Neg for f64x8 { +impl core::ops::SubAssign for u64x8 { + #[inline(always)] + fn sub_assign(&mut self, rhs: u64) { + *self = self.simd.sub_u64x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::Sub> for u64 { + type Output = u64x8; + #[inline(always)] + fn sub(self, rhs: u64x8) -> Self::Output { + rhs.simd.sub_u64x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::Mul for u64x8 { type Output = Self; - #[doc = "Negate each element of the vector."] + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] #[inline(always)] - fn neg(self) -> Self::Output { - self.simd.neg_f64x8(self) + fn mul(self, rhs: Self) -> Self::Output { + self.simd.mul_u64x8(self, rhs) } } -impl core::ops::Add for f64x8 { +impl core::ops::MulAssign for u64x8 { + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + *self = self.simd.mul_u64x8(*self, rhs); + } +} +impl core::ops::Mul for u64x8 { type Output = Self; - #[doc = "Add two vectors element-wise."] #[inline(always)] - fn add(self, rhs: Self) -> Self::Output { - self.simd.add_f64x8(self, rhs) + fn mul(self, rhs: u64) -> Self::Output { + self.simd.mul_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::AddAssign for f64x8 { - #[doc = "Add two vectors element-wise."] +impl core::ops::MulAssign for u64x8 { #[inline(always)] - fn add_assign(&mut self, rhs: Self) { - *self = self.simd.add_f64x8(*self, rhs); + fn mul_assign(&mut self, rhs: u64) { + *self = self.simd.mul_u64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Add for f64x8 { +impl core::ops::Mul> for u64 { + type Output = u64x8; + #[inline(always)] + fn mul(self, rhs: u64x8) -> Self::Output { + rhs.simd.mul_u64x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitAnd for u64x8 { type Output = Self; + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn add(self, rhs: f64) -> Self::Output { - self.simd.add_f64x8(self, rhs.simd_into(self.simd)) + fn bitand(self, rhs: Self) -> Self::Output { + self.simd.and_u64x8(self, rhs) } } -impl core::ops::AddAssign for f64x8 { +impl core::ops::BitAndAssign for u64x8 { + #[doc = "Compute the bitwise AND of two vectors."] #[inline(always)] - fn add_assign(&mut self, rhs: f64) { - *self = self.simd.add_f64x8(*self, rhs.simd_into(self.simd)); + fn bitand_assign(&mut self, rhs: Self) { + *self = self.simd.and_u64x8(*self, rhs); } } -impl core::ops::Add> for f64 { - type Output = f64x8; +impl core::ops::BitAnd for u64x8 { + type Output = Self; #[inline(always)] - fn add(self, rhs: f64x8) -> Self::Output { - rhs.simd.add_f64x8(self.simd_into(rhs.simd), rhs) + fn bitand(self, rhs: u64) -> Self::Output { + self.simd.and_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::Sub for f64x8 { +impl core::ops::BitAndAssign for u64x8 { + #[inline(always)] + fn bitand_assign(&mut self, rhs: u64) { + *self = self.simd.and_u64x8(*self, rhs.simd_into(self.simd)); + } +} +impl core::ops::BitAnd> for u64 { + type Output = u64x8; + #[inline(always)] + fn bitand(self, rhs: u64x8) -> Self::Output { + rhs.simd.and_u64x8(self.simd_into(rhs.simd), rhs) + } +} +impl core::ops::BitOr for u64x8 { type Output = Self; - #[doc = "Subtract two vectors element-wise."] + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn sub(self, rhs: Self) -> Self::Output { - self.simd.sub_f64x8(self, rhs) + fn bitor(self, rhs: Self) -> Self::Output { + self.simd.or_u64x8(self, rhs) } } -impl core::ops::SubAssign for f64x8 { - #[doc = "Subtract two vectors element-wise."] +impl core::ops::BitOrAssign for u64x8 { + #[doc = "Compute the bitwise OR of two vectors."] #[inline(always)] - fn sub_assign(&mut self, rhs: Self) { - *self = self.simd.sub_f64x8(*self, rhs); + fn bitor_assign(&mut self, rhs: Self) { + *self = self.simd.or_u64x8(*self, rhs); } } -impl core::ops::Sub for f64x8 { +impl core::ops::BitOr for u64x8 { type Output = Self; #[inline(always)] - fn sub(self, rhs: f64) -> Self::Output { - self.simd.sub_f64x8(self, rhs.simd_into(self.simd)) + fn bitor(self, rhs: u64) -> Self::Output { + self.simd.or_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::SubAssign for f64x8 { +impl core::ops::BitOrAssign for u64x8 { #[inline(always)] - fn sub_assign(&mut self, rhs: f64) { - *self = self.simd.sub_f64x8(*self, rhs.simd_into(self.simd)); + fn bitor_assign(&mut self, rhs: u64) { + *self = self.simd.or_u64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Sub> for f64 { - type Output = f64x8; +impl core::ops::BitOr> for u64 { + type Output = u64x8; #[inline(always)] - fn sub(self, rhs: f64x8) -> Self::Output { - rhs.simd.sub_f64x8(self.simd_into(rhs.simd), rhs) + fn bitor(self, rhs: u64x8) -> Self::Output { + rhs.simd.or_u64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Mul for f64x8 { +impl core::ops::BitXor for u64x8 { type Output = Self; - #[doc = "Multiply two vectors element-wise."] + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn mul(self, rhs: Self) -> Self::Output { - self.simd.mul_f64x8(self, rhs) + fn bitxor(self, rhs: Self) -> Self::Output { + self.simd.xor_u64x8(self, rhs) } } -impl core::ops::MulAssign for f64x8 { - #[doc = "Multiply two vectors element-wise."] +impl core::ops::BitXorAssign for u64x8 { + #[doc = "Compute the bitwise XOR of two vectors."] #[inline(always)] - fn mul_assign(&mut self, rhs: Self) { - *self = self.simd.mul_f64x8(*self, rhs); + fn bitxor_assign(&mut self, rhs: Self) { + *self = self.simd.xor_u64x8(*self, rhs); } } -impl core::ops::Mul for f64x8 { +impl core::ops::BitXor for u64x8 { type Output = Self; #[inline(always)] - fn mul(self, rhs: f64) -> Self::Output { - self.simd.mul_f64x8(self, rhs.simd_into(self.simd)) + fn bitxor(self, rhs: u64) -> Self::Output { + self.simd.xor_u64x8(self, rhs.simd_into(self.simd)) } } -impl core::ops::MulAssign for f64x8 { +impl core::ops::BitXorAssign for u64x8 { #[inline(always)] - fn mul_assign(&mut self, rhs: f64) { - *self = self.simd.mul_f64x8(*self, rhs.simd_into(self.simd)); + fn bitxor_assign(&mut self, rhs: u64) { + *self = self.simd.xor_u64x8(*self, rhs.simd_into(self.simd)); } } -impl core::ops::Mul> for f64 { - type Output = f64x8; +impl core::ops::BitXor> for u64 { + type Output = u64x8; #[inline(always)] - fn mul(self, rhs: f64x8) -> Self::Output { - rhs.simd.mul_f64x8(self.simd_into(rhs.simd), rhs) + fn bitxor(self, rhs: u64x8) -> Self::Output { + rhs.simd.xor_u64x8(self.simd_into(rhs.simd), rhs) } } -impl core::ops::Div for f64x8 { +impl core::ops::Not for u64x8 { type Output = Self; - #[doc = "Divide two vectors element-wise."] + #[doc = "Compute the bitwise NOT of the vector."] #[inline(always)] - fn div(self, rhs: Self) -> Self::Output { - self.simd.div_f64x8(self, rhs) + fn not(self) -> Self::Output { + self.simd.not_u64x8(self) } } -impl core::ops::DivAssign for f64x8 { - #[doc = "Divide two vectors element-wise."] +impl core::ops::Shl for u64x8 { + type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] #[inline(always)] - fn div_assign(&mut self, rhs: Self) { - *self = self.simd.div_f64x8(*self, rhs); + fn shl(self, rhs: u32) -> Self::Output { + self.simd.shl_u64x8(self, rhs) } } -impl core::ops::Div for f64x8 { +impl core::ops::ShlAssign for u64x8 { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = self.simd.shl_u64x8(*self, rhs); + } +} +impl core::ops::Shl for u64x8 { type Output = Self; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn div(self, rhs: f64) -> Self::Output { - self.simd.div_f64x8(self, rhs.simd_into(self.simd)) + fn shl(self, rhs: Self) -> Self::Output { + self.simd.shlv_u64x8(self, rhs) } } -impl core::ops::DivAssign for f64x8 { +impl core::ops::ShlAssign for u64x8 { + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] #[inline(always)] - fn div_assign(&mut self, rhs: f64) { - *self = self.simd.div_f64x8(*self, rhs.simd_into(self.simd)); + fn shl_assign(&mut self, rhs: Self) { + *self = self.simd.shlv_u64x8(*self, rhs); } } -impl core::ops::Div> for f64 { - type Output = f64x8; +impl core::ops::Shr for u64x8 { + type Output = Self; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] #[inline(always)] - fn div(self, rhs: f64x8) -> Self::Output { - rhs.simd.div_f64x8(self.simd_into(rhs.simd), rhs) + fn shr(self, rhs: u32) -> Self::Output { + self.simd.shr_u64x8(self, rhs) + } +} +impl core::ops::ShrAssign for u64x8 { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = self.simd.shr_u64x8(*self, rhs); + } +} +impl core::ops::Shr for u64x8 { + type Output = Self; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr(self, rhs: Self) -> Self::Output { + self.simd.shrv_u64x8(self, rhs) + } +} +impl core::ops::ShrAssign for u64x8 { + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + #[inline(always)] + fn shr_assign(&mut self, rhs: Self) { + *self = self.simd.shrv_u64x8(*self, rhs); } } impl core::ops::BitAnd for mask64x8 { diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index bcef3ed5f..43ee9123b 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -9,9 +9,9 @@ use crate::{ }; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; #[doc = r" The main SIMD trait, implemented by all SIMD token types."] #[doc = r""] @@ -27,8 +27,8 @@ use crate::{ #[doc = r" # Associated Types"] #[doc = r""] #[doc = r#" The trait defines associated types for the highest "native" vector width of each scalar type (e.g. `f32s`,"#] -#[doc = r" `u32s`). These are always at least 128 bits, but may be larger. Currently, they are 128 bits everywhere but"] -#[doc = r" AVX2, where they are 256 bits."] +#[doc = r" `u32s`). These are always at least 128 bits, but may be larger. Currently, they are 128 bits on the"] +#[doc = r" fallback, NEON, WASM, and SSE4.2 backends, 256 bits on AVX2, and 512 bits on AVX-512."] #[doc = r""] #[doc = r" # Example"] #[doc = r""] @@ -61,7 +61,13 @@ pub trait Simd: > + SimdCvtFloat + SimdCvtFloat; #[doc = r" A native-width SIMD vector of [`f64`]s."] - type f64s: SimdFloat, Mask = Self::mask64s>; + type f64s: SimdFloat< + Self, + Element = f64, + Block = f64x2, + Mask = Self::mask64s, + Bytes = ::Bytes, + >; #[doc = r" A native-width SIMD vector of [`u8`]s."] type u8s: SimdInt, Mask = Self::mask8s>; #[doc = r" A native-width SIMD vector of [`i8`]s."] @@ -94,6 +100,16 @@ pub trait Simd: Bytes = ::Bytes, > + SimdCvtTruncate + core::ops::Neg; + #[doc = r" A native-width SIMD vector of [`u64`]s."] + type u64s: SimdInt, Mask = Self::mask64s>; + #[doc = r" A native-width SIMD vector of [`i64`]s."] + type i64s: SimdInt< + Self, + Element = i64, + Block = i64x2, + Mask = Self::mask64s, + Bytes = ::Bytes, + > + core::ops::Neg; #[doc = r" A native-width SIMD mask with 8-bit lanes."] type mask8s: SimdMask + Select @@ -111,7 +127,11 @@ pub trait Simd: + Select + Select; #[doc = r" A native-width SIMD mask with 64-bit lanes."] - type mask64s: SimdMask + Select + Select; + type mask64s: SimdMask + + Select + + Select + + Select + + Select; #[doc = r" This SIMD token's feature level."] fn level(self) -> Level; #[doc = r" Call function with CPU features enabled."] @@ -218,7 +238,7 @@ pub trait Simd: fn reinterpret_u8_f32x4(self, a: f32x4) -> u8x16; #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u32_f32x4(self, a: f32x4) -> u32x4; - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] fn cvt_u32_precise_f32x4(self, a: f32x4) -> u32x4; @@ -402,6 +422,8 @@ pub trait Simd: fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask8x16(self, a: mask8x16) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16; #[doc = "Compute the logical OR of two masks."] @@ -605,6 +627,8 @@ pub trait Simd: fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask16x8(self, a: mask16x8) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8; #[doc = "Compute the logical OR of two masks."] @@ -810,6 +834,8 @@ pub trait Simd: fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask32x4(self, a: mask32x4) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4; #[doc = "Compute the logical OR of two masks."] @@ -931,6 +957,172 @@ pub trait Simd: fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4; #[doc = "Reinterpret the bits of this vector as a vector of `f32` elements.\n\nThe number of elements in the result is twice that of the input."] fn reinterpret_f32_f64x2(self, a: f64x2) -> f32x4; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_i64x2(self, val: i64) -> i64x2; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_i64x2(self, a: i64x2) -> i64x2; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2; + #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4; + #[doc = "Negate each element of the vector, wrapping on overflow."] + fn neg_i64x2(self, a: i64x2) -> i64x2; + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_u64x2(self, val: u64) -> u64x2; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_u64x2( + self, + a: u64x2, + b: u64x2, + ) -> u64x2; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_u64x2(self, a: u64x2) -> u64x2; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2; + #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4; + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x2(self, val: bool) -> mask64x2; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -941,6 +1133,8 @@ pub trait Simd: fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask64x2(self, a: mask64x2) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2; #[doc = "Compute the logical OR of two masks."] @@ -1070,7 +1264,7 @@ pub trait Simd: fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32; #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8; - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x8(self, a: f32x8) -> u32x8; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8; @@ -1258,6 +1452,8 @@ pub trait Simd: fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask8x32(self, a: mask8x32) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32; #[doc = "Compute the logical OR of two masks."] @@ -1469,6 +1665,8 @@ pub trait Simd: fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask16x16(self, a: mask16x16) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16; #[doc = "Compute the logical OR of two masks."] @@ -1680,6 +1878,8 @@ pub trait Simd: fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask32x8(self, a: mask32x8) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8; #[doc = "Compute the logical OR of two masks."] @@ -1805,6 +2005,176 @@ pub trait Simd: fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2); #[doc = "Reinterpret the bits of this vector as a vector of `f32` elements.\n\nThe number of elements in the result is twice that of the input."] fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_i64x4(self, val: i64) -> i64x4; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_i64x4( + self, + a: i64x4, + b: i64x4, + ) -> i64x4; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_i64x4(self, a: i64x4) -> i64x4; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4; + #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8; + #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2); + #[doc = "Negate each element of the vector, wrapping on overflow."] + fn neg_i64x4(self, a: i64x4) -> i64x4; + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_u64x4(self, val: u64) -> u64x4; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_u64x4( + self, + a: u64x4, + b: u64x4, + ) -> u64x4; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_u64x4(self, a: u64x4) -> u64x4; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4; + #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8; + #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2); + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x4(self, val: bool) -> mask64x4; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -1815,6 +2185,8 @@ pub trait Simd: fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask64x4(self, a: mask64x4) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4; #[doc = "Compute the logical OR of two masks."] @@ -1940,15 +2312,15 @@ pub trait Simd: fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8; #[doc = "Reinterpret the bits of this vector as a vector of `i32` elements.\n\nThis is a bitwise reinterpretation only, and does not perform any conversions."] fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16; - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four consecutive 128-bit blocks and transposes those blocks into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` loads as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation transposes one vector into four consecutive 128-bit blocks in memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` stores as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> (); #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64; #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16; - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16; @@ -2118,9 +2490,9 @@ pub trait Simd: fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four consecutive 128-bit blocks and transposes those blocks into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` loads as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation transposes one vector into four consecutive 128-bit blocks in memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` stores as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> (); #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16; @@ -2134,6 +2506,8 @@ pub trait Simd: fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask8x64(self, a: mask8x64) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64; #[doc = "Compute the logical OR of two masks."] @@ -2323,9 +2697,9 @@ pub trait Simd: fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four consecutive 128-bit blocks and transposes those blocks into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` loads as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation transposes one vector into four consecutive 128-bit blocks in memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` stores as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> (); #[doc = "Truncate each element to a narrower integer type.\n\nThe number of elements in the result is twice that of the input."] fn narrow_u16x32(self, a: u16x32) -> u8x32; @@ -2343,6 +2717,8 @@ pub trait Simd: fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask16x32(self, a: mask16x32) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32; #[doc = "Compute the logical OR of two masks."] @@ -2534,9 +2910,9 @@ pub trait Simd: fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four consecutive 128-bit blocks and transposes those blocks into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` loads as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation transposes one vector into four consecutive 128-bit blocks in memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` stores as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> (); #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64; @@ -2552,6 +2928,8 @@ pub trait Simd: fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask32x16(self, a: mask32x16) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16; #[doc = "Compute the logical OR of two masks."] @@ -2673,6 +3051,176 @@ pub trait Simd: fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4); #[doc = "Reinterpret the bits of this vector as a vector of `f32` elements.\n\nThe number of elements in the result is twice that of the input."] fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_i64x8(self, val: i64) -> i64x8; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_i64x8( + self, + a: i64x8, + b: i64x8, + ) -> i64x8; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_i64x8(self, a: i64x8) -> i64x8; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8; + #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4); + #[doc = "Negate each element of the vector, wrapping on overflow."] + fn neg_i64x8(self, a: i64x8) -> i64x8; + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16; + #[doc = "Create a SIMD vector with all elements set to the given value."] + fn splat_u64x8(self, val: u64) -> u64x8; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8; + #[doc = "Create a SIMD vector from an array of the same length."] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8; + #[doc = "Convert a SIMD vector to an array."] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize]; + #[doc = "Project a reference to a SIMD vector to a reference to the equivalent array."] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize]; + #[doc = "Project a mutable reference to a SIMD vector to a mutable reference to the equivalent array."] + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize]; + #[doc = "Store a SIMD vector into an array of the same length."] + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> (); + #[doc = "Reinterpret a vector of bytes as a SIMD vector of a given type, with the equivalent byte length."] + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8; + #[doc = "Reinterpret a SIMD vector as a vector of bytes, with the equivalent byte length."] + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64; + #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Like `slide`, but operates independently on each 128-bit block."] + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8; + #[doc = "Add two vectors element-wise, wrapping on overflow."] + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Subtract two vectors element-wise, wrapping on overflow."] + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Multiply two vectors element-wise, wrapping on overflow."] + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Compute the bitwise AND of two vectors."] + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Compute the bitwise OR of two vectors."] + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Compute the bitwise XOR of two vectors."] + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Compute the bitwise NOT of the vector."] + fn not_u64x8(self, a: u64x8) -> u64x8; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right."] + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8; + #[doc = "Shift each element left by the given number of bits.\n\nBits shifted out of the left side are discarded, and zeros are shifted in on the right.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Shift each element right by the given number of bits.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated."] + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8; + #[doc = "Shift each element right by the corresponding element in another vector.\n\nFor unsigned integers, zeros are shifted in on the left. For signed integers, the sign bit is replicated.\n\nThis operation is not implemented in hardware on all platforms. On WebAssembly, and on x86 platforms without AVX2, this will use a fallback scalar implementation."] + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Compare two vectors element-wise for equality.\n\nReturns a mask where each logical lane is true if the corresponding elements are equal, and false if not."] + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for less than.\n\nReturns a mask where each logical lane is true if `a` is less than `b`, and false if not."] + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for less than or equal.\n\nReturns a mask where each logical lane is true if `a` is less than or equal to `b`, and false if not."] + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for greater than or equal.\n\nReturns a mask where each logical lane is true if `a` is greater than or equal to `b`, and false if not."] + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8; + #[doc = "Compare two vectors element-wise for greater than.\n\nReturns a mask where each logical lane is true if `a` is greater than `b`, and false if not."] + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8; + #[doc = "Interleave the lower half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, b0, a1, b1]`.\n\n**Note:** This operation is only useful if you need to discard elements `a2, a3, b2, b3`.\n For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Interleave the upper half elements of two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a2, b2, a3, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a1, b0, b1`.For fully interleaving two vectors prefer `interleave`,\n which is faster than `zip_low` followed by `zip_high` on some platforms."] + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Extract even-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a0, a2, b0, b2]`.\n\n**Note:** This operation is only useful if you need to discard elements `a1, a3, b1, b3`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Extract odd-indexed elements from two vectors.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `[a1, a3, b1, b3]`.\n\n**Note:** This operation is only useful if you need to discard elements `a0, a2, b0, b2`.For fully deinterleaving two vectors prefer `deinterleave`,\n which is faster than `unzip_low` followed by `unzip_high` on some platforms."] + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Interleave two vectors.\n\nThe resulting vectors contain elements taken alternately from `a` and `b`, first filling the first result, and then the second.\n\nThe reverse of this operation is `deinterleave`.\n\nFor vectors `[a0, a1, a2, a3]` and `[b0, b1, b2, b3]`, returns `([a0, b0, a1, b1], [a2, b2, a3, b3])`."] + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8); + #[doc = "Deinterleave two vectors.\n\nThe first result contains all even-indexed elements from `a` followed by all even-indexed elements from `b`. The second result contains all odd-indexed elements from `a` followed by all odd-indexed elements from `b`.\n\nThe reverse of this operation is `interleave`.\n\nFor vectors `[a0, b0, a1, b1]` and `[a2, b2, a3, b3]`, returns `([a0, a1, a2, a3], [b0, b1, b2, b3])`."] + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8); + #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8; + #[doc = "Return the element-wise minimum of two vectors."] + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Return the element-wise maximum of two vectors."] + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8; + #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4); + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> (); + #[doc = "Reinterpret the bits of this vector as a vector of `u8` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64; + #[doc = "Reinterpret the bits of this vector as a vector of `u32` elements.\n\nThe total bit width is preserved; the number of elements changes accordingly."] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x8(self, val: bool) -> mask64x8; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -2683,6 +3231,8 @@ pub trait Simd: fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8; #[doc = "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared."] fn to_bitmask_mask64x8(self, a: mask64x8) -> u64; + #[doc = "Set one logical lane of a SIMD mask."] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> (); #[doc = "Compute the logical AND of two masks."] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8; #[doc = "Compute the logical OR of two masks."] @@ -2729,6 +3279,8 @@ pub(crate) mod arch_types { type u32x4: Copy + Send + Sync + SimdPod; type mask32x4: Copy + Send + Sync + SimdPod; type f64x2: Copy + Send + Sync + SimdPod; + type i64x2: Copy + Send + Sync + SimdPod; + type u64x2: Copy + Send + Sync + SimdPod; type mask64x2: Copy + Send + Sync + SimdPod; type f32x8: Copy + Send + Sync + SimdPod; type i8x32: Copy + Send + Sync + SimdPod; @@ -2741,6 +3293,8 @@ pub(crate) mod arch_types { type u32x8: Copy + Send + Sync + SimdPod; type mask32x8: Copy + Send + Sync + SimdPod; type f64x4: Copy + Send + Sync + SimdPod; + type i64x4: Copy + Send + Sync + SimdPod; + type u64x4: Copy + Send + Sync + SimdPod; type mask64x4: Copy + Send + Sync + SimdPod; type f32x16: Copy + Send + Sync + SimdPod; type i8x64: Copy + Send + Sync + SimdPod; @@ -2753,6 +3307,8 @@ pub(crate) mod arch_types { type u32x16: Copy + Send + Sync + SimdPod; type mask32x16: Copy + Send + Sync + SimdPod; type f64x8: Copy + Send + Sync + SimdPod; + type i64x8: Copy + Send + Sync + SimdPod; + type u64x8: Copy + Send + Sync + SimdPod; type mask64x8: Copy + Send + Sync + SimdPod; } } diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index 416defc26..1ea0b330d 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -117,8 +117,8 @@ impl SimdBase for f32x4 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f32) -> Self { - simd.load_array_f32x4(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f32) -> Self { + simd.load_array_f32x4([f(0usize), f(1usize), f(2usize), f(3usize)]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -370,8 +370,25 @@ impl SimdBase for i8x16 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i8) -> Self { - simd.load_array_i8x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i8) -> Self { + simd.load_array_i8x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -555,8 +572,25 @@ impl SimdBase for u8x16 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u8) -> Self { - simd.load_array_u8x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u8) -> Self { + simd.load_array_u8x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -688,14 +722,7 @@ impl SimdMask for mask8x16 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 16, - "mask lane index {index} is out of bounds for {} lanes", - 16 - ); - let mut lanes = self.simd.as_array_mask8x16(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask8x16(lanes); + self.simd.set_mask8x16(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i8]) -> Self { @@ -838,8 +865,17 @@ impl SimdBase for i16x8 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i16) -> Self { - simd.load_array_i16x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i16) -> Self { + simd.load_array_i16x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -1023,8 +1059,17 @@ impl SimdBase for u16x8 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u16) -> Self { - simd.load_array_u16x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u16) -> Self { + simd.load_array_u16x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -1156,14 +1201,7 @@ impl SimdMask for mask16x8 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 8, - "mask lane index {index} is out of bounds for {} lanes", - 8 - ); - let mut lanes = self.simd.as_array_mask16x8(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask16x8(lanes); + self.simd.set_mask16x8(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i16]) -> Self { @@ -1306,8 +1344,8 @@ impl SimdBase for i32x4 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i32) -> Self { - simd.load_array_i32x4(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i32) -> Self { + simd.load_array_i32x4([f(0usize), f(1usize), f(2usize), f(3usize)]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -1503,8 +1541,8 @@ impl SimdBase for u32x4 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u32) -> Self { - simd.load_array_u32x4(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u32) -> Self { + simd.load_array_u32x4([f(0usize), f(1usize), f(2usize), f(3usize)]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -1572,7 +1610,7 @@ impl crate::SimdInt for u32x4 { } } impl SimdCvtTruncate> for u32x4 { - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] #[inline(always)] fn truncate_from(x: f32x4) -> Self { x.simd.cvt_u32_f32x4(x) @@ -1648,14 +1686,7 @@ impl SimdMask for mask32x4 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 4, - "mask lane index {index} is out of bounds for {} lanes", - 4 - ); - let mut lanes = self.simd.as_array_mask32x4(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask32x4(lanes); + self.simd.set_mask32x4(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i32]) -> Self { @@ -1798,8 +1829,8 @@ impl SimdBase for f64x2 { block } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f64) -> Self { - simd.load_array_f64x2(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f64) -> Self { + simd.load_array_f64x2([f(0usize), f(1usize)]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -1927,6 +1958,376 @@ impl crate::SimdCombine for f64x2 { self.simd.combine_f64x2(self, rhs.simd_into(self.simd)) } } +#[doc = "A SIMD vector of 2 [`i64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, i64x2};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = i64x2::splat(simd, 1);\n let b = i64x2::simd_from(simd, 1);\n\n // From a slice:\n let c = i64x2::from_slice(simd, &[1, 2]);\n\n // From an array:\n let d = i64x2::simd_from(simd, [1, 2]);\n\n // From an element-wise function:\n let e = i64x2::from_fn(simd, |i| i as i64);\n}\n```"] +#[derive(Clone, Copy)] +#[repr(C, align(16))] +pub struct i64x2 { + pub(crate) val: S::i64x2, + pub simd: S, +} +impl Seal for i64x2 {} +impl SimdFrom<[i64; 2], S> for i64x2 { + #[inline(always)] + fn simd_from(simd: S, val: [i64; 2]) -> Self { + simd.load_array_i64x2(val) + } +} +impl From> for [i64; 2] { + #[inline(always)] + fn from(value: i64x2) -> Self { + value.simd.as_array_i64x2(value) + } +} +impl core::ops::Deref for i64x2 { + type Target = [i64; 2]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_i64x2(self) + } +} +impl core::ops::DerefMut for i64x2 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_i64x2(self) + } +} +impl core::fmt::Debug for i64x2 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "i64x2", &self.simd, self.simd.as_array_ref_i64x2(self)) + } +} +impl SimdFrom for i64x2 { + #[inline(always)] + fn simd_from(simd: S, value: i64) -> Self { + simd.splat_i64x2(value) + } +} +impl core::ops::Index for i64x2 { + type Output = i64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_i64x2(self)[i] + } +} +impl core::ops::IndexMut for i64x2 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_i64x2(self)[i] + } +} +impl Select> for mask64x2 { + #[inline(always)] + fn select(self, if_true: i64x2, if_false: i64x2) -> i64x2 { + self.simd.select_i64x2(self, if_true, if_false) + } +} +impl Bytes for i64x2 { + type Bytes = u8x16; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_i64x2(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_i64x2(value) + } +} +impl SimdBase for i64x2 { + type Element = i64; + const N: usize = 2; + type Mask = mask64x2; + type Block = i64x2; + type Array = [i64; 2]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[i64] { + self.simd.as_array_ref_i64x2(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [i64] { + self.simd.as_array_mut_i64x2(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[i64]) -> Self { + simd.load_array_ref_i64x2(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [i64]) { + self.simd + .store_array_i64x2(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: i64) -> Self { + simd.splat_i64x2(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + block + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i64) -> Self { + simd.load_array_i64x2([f(0usize), f(1usize)]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_i64x2::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_i64x2::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for i64x2 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_i64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_i64x2(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdCombine for i64x2 { + type Combined = i64x4; + #[inline(always)] + fn combine(self, rhs: impl SimdInto) -> Self::Combined { + self.simd.combine_i64x2(self, rhs.simd_into(self.simd)) + } +} +#[doc = "A SIMD vector of 2 [`u64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, u64x2};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = u64x2::splat(simd, 1);\n let b = u64x2::simd_from(simd, 1);\n\n // From a slice:\n let c = u64x2::from_slice(simd, &[1, 2]);\n\n // From an array:\n let d = u64x2::simd_from(simd, [1, 2]);\n\n // From an element-wise function:\n let e = u64x2::from_fn(simd, |i| i as u64);\n}\n```"] +#[derive(Clone, Copy)] +#[repr(C, align(16))] +pub struct u64x2 { + pub(crate) val: S::u64x2, + pub simd: S, +} +impl Seal for u64x2 {} +impl SimdFrom<[u64; 2], S> for u64x2 { + #[inline(always)] + fn simd_from(simd: S, val: [u64; 2]) -> Self { + simd.load_array_u64x2(val) + } +} +impl From> for [u64; 2] { + #[inline(always)] + fn from(value: u64x2) -> Self { + value.simd.as_array_u64x2(value) + } +} +impl core::ops::Deref for u64x2 { + type Target = [u64; 2]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_u64x2(self) + } +} +impl core::ops::DerefMut for u64x2 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_u64x2(self) + } +} +impl core::fmt::Debug for u64x2 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "u64x2", &self.simd, self.simd.as_array_ref_u64x2(self)) + } +} +impl SimdFrom for u64x2 { + #[inline(always)] + fn simd_from(simd: S, value: u64) -> Self { + simd.splat_u64x2(value) + } +} +impl core::ops::Index for u64x2 { + type Output = u64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_u64x2(self)[i] + } +} +impl core::ops::IndexMut for u64x2 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_u64x2(self)[i] + } +} +impl Select> for mask64x2 { + #[inline(always)] + fn select(self, if_true: u64x2, if_false: u64x2) -> u64x2 { + self.simd.select_u64x2(self, if_true, if_false) + } +} +impl Bytes for u64x2 { + type Bytes = u8x16; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_u64x2(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_u64x2(value) + } +} +impl SimdBase for u64x2 { + type Element = u64; + const N: usize = 2; + type Mask = mask64x2; + type Block = u64x2; + type Array = [u64; 2]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[u64] { + self.simd.as_array_ref_u64x2(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [u64] { + self.simd.as_array_mut_u64x2(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[u64]) -> Self { + simd.load_array_ref_u64x2(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [u64]) { + self.simd + .store_array_u64x2(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: u64) -> Self { + simd.splat_u64x2(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + block + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u64) -> Self { + simd.load_array_u64x2([f(0usize), f(1usize)]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_u64x2::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_u64x2::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for u64x2 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_u64x2(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_u64x2(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdCombine for u64x2 { + type Combined = u64x4; + #[inline(always)] + fn combine(self, rhs: impl SimdInto) -> Self::Combined { + self.simd.combine_u64x2(self, rhs.simd_into(self.simd)) + } +} #[doc = "A SIMD mask of 2 logical lanes corresponding to 64-bit vector elements.\n\nThe storage representation of this type is intentionally opaque and may vary depending on the SIMD level.\n\nYou can construct this mask type using the [`Self::splat`], [`Self::from_bitmask`], [`Self::from_slice`], and [`Self::simd_from`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, mask64x2};\nfn construct_mask(simd: S) {\n // From a single boolean value:\n let a = mask64x2::splat(simd, true);\n let b = mask64x2::simd_from(simd, true);\n\n // From signed integer mask lanes:\n let c = mask64x2::from_slice(simd, &[-1, 0]);\n let d = mask64x2::simd_from(simd, [-1, 0]);\n\n // From a compact bitmask (same mask as above, least significant bit maps to lane 0):\n let e = mask64x2::from_bitmask(simd, 0b0001);\n\n // By setting individual lanes:\n let mut f = mask64x2::splat(simd, false);\n f.set(0, true);\n}\n```"] #[derive(Clone, Copy)] pub struct mask64x2 { @@ -1985,14 +2386,7 @@ impl SimdMask for mask64x2 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 2, - "mask lane index {index} is out of bounds for {} lanes", - 2 - ); - let mut lanes = self.simd.as_array_mask64x2(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask64x2(lanes); + self.simd.set_mask64x2(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i64]) -> Self { @@ -2135,8 +2529,17 @@ impl SimdBase for f32x8 { block.simd.combine_f32x4(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f32) -> Self { - simd.load_array_f32x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f32) -> Self { + simd.load_array_f32x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -2395,8 +2798,41 @@ impl SimdBase for i8x32 { block.simd.combine_i8x16(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i8) -> Self { - simd.load_array_i8x32(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i8) -> Self { + simd.load_array_i8x32([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -2587,8 +3023,41 @@ impl SimdBase for u8x32 { block.simd.combine_u8x16(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u8) -> Self { - simd.load_array_u8x32(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u8) -> Self { + simd.load_array_u8x32([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -2727,14 +3196,7 @@ impl SimdMask for mask8x32 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 32, - "mask lane index {index} is out of bounds for {} lanes", - 32 - ); - let mut lanes = self.simd.as_array_mask8x32(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask8x32(lanes); + self.simd.set_mask8x32(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i8]) -> Self { @@ -2882,8 +3344,25 @@ impl SimdBase for i16x16 { block.simd.combine_i16x8(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i16) -> Self { - simd.load_array_i16x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i16) -> Self { + simd.load_array_i16x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -3080,8 +3559,25 @@ impl SimdBase for u16x16 { block.simd.combine_u16x8(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u16) -> Self { - simd.load_array_u16x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u16) -> Self { + simd.load_array_u16x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -3221,14 +3717,7 @@ impl SimdMask for mask16x16 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 16, - "mask lane index {index} is out of bounds for {} lanes", - 16 - ); - let mut lanes = self.simd.as_array_mask16x16(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask16x16(lanes); + self.simd.set_mask16x16(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i16]) -> Self { @@ -3371,8 +3860,17 @@ impl SimdBase for i32x8 { block.simd.combine_i32x4(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i32) -> Self { - simd.load_array_i32x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i32) -> Self { + simd.load_array_i32x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -3575,8 +4073,17 @@ impl SimdBase for u32x8 { block.simd.combine_u32x4(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u32) -> Self { - simd.load_array_u32x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u32) -> Self { + simd.load_array_u32x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -3644,7 +4151,7 @@ impl crate::SimdInt for u32x8 { } } impl SimdCvtTruncate> for u32x8 { - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] #[inline(always)] fn truncate_from(x: f32x8) -> Self { x.simd.cvt_u32_f32x8(x) @@ -3727,14 +4234,7 @@ impl SimdMask for mask32x8 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 8, - "mask lane index {index} is out of bounds for {} lanes", - 8 - ); - let mut lanes = self.simd.as_array_mask32x8(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask32x8(lanes); + self.simd.set_mask32x8(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i32]) -> Self { @@ -3877,8 +4377,8 @@ impl SimdBase for f64x4 { block.simd.combine_f64x2(block, block) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f64) -> Self { - simd.load_array_f64x4(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f64) -> Self { + simd.load_array_f64x4([f(0usize), f(1usize), f(2usize), f(3usize)]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -4013,27 +4513,411 @@ impl crate::SimdCombine for f64x4 { self.simd.combine_f64x4(self, rhs.simd_into(self.simd)) } } -#[doc = "A SIMD mask of 4 logical lanes corresponding to 64-bit vector elements.\n\nThe storage representation of this type is intentionally opaque and may vary depending on the SIMD level.\n\nYou can construct this mask type using the [`Self::splat`], [`Self::from_bitmask`], [`Self::from_slice`], and [`Self::simd_from`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, mask64x4};\nfn construct_mask(simd: S) {\n // From a single boolean value:\n let a = mask64x4::splat(simd, true);\n let b = mask64x4::simd_from(simd, true);\n\n // From signed integer mask lanes:\n let c = mask64x4::from_slice(simd, &[-1, 0, 0, 0]);\n let d = mask64x4::simd_from(simd, [-1, 0, 0, 0]);\n\n // From a compact bitmask (same mask as above, least significant bit maps to lane 0):\n let e = mask64x4::from_bitmask(simd, 0b0001);\n\n // By setting individual lanes:\n let mut f = mask64x4::splat(simd, false);\n f.set(0, true);\n}\n```"] +#[doc = "A SIMD vector of 4 [`i64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, i64x4};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = i64x4::splat(simd, 1);\n let b = i64x4::simd_from(simd, 1);\n\n // From a slice:\n let c = i64x4::from_slice(simd, &[1, 2, 3, 4]);\n\n // From an array:\n let d = i64x4::simd_from(simd, [1, 2, 3, 4]);\n\n // From an element-wise function:\n let e = i64x4::from_fn(simd, |i| i as i64);\n # use fearless_simd::i64x2;\n // From `Self::Block`:\n let f = i64x4::block_splat(i64x2::simd_from(simd, [1, 2]));\n}\n```"] #[derive(Clone, Copy)] -pub struct mask64x4 { - pub(crate) val: S::mask64x4, - pub(crate) simd: S, +#[repr(C, align(32))] +pub struct i64x4 { + pub(crate) val: S::i64x4, + pub simd: S, } -impl Seal for mask64x4 {} -impl SimdFrom<[i64; 4], S> for mask64x4 { +impl Seal for i64x4 {} +impl SimdFrom<[i64; 4], S> for i64x4 { #[inline(always)] fn simd_from(simd: S, val: [i64; 4]) -> Self { - simd.load_array_mask64x4(val) + simd.load_array_i64x4(val) } } -impl From> for [i64; 4] { +impl From> for [i64; 4] { #[inline(always)] - fn from(value: mask64x4) -> Self { - value.simd.as_array_mask64x4(value) + fn from(value: i64x4) -> Self { + value.simd.as_array_i64x4(value) } } -impl core::fmt::Debug for mask64x4 { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl core::ops::Deref for i64x4 { + type Target = [i64; 4]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_i64x4(self) + } +} +impl core::ops::DerefMut for i64x4 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_i64x4(self) + } +} +impl core::fmt::Debug for i64x4 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "i64x4", &self.simd, self.simd.as_array_ref_i64x4(self)) + } +} +impl SimdFrom for i64x4 { + #[inline(always)] + fn simd_from(simd: S, value: i64) -> Self { + simd.splat_i64x4(value) + } +} +impl core::ops::Index for i64x4 { + type Output = i64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_i64x4(self)[i] + } +} +impl core::ops::IndexMut for i64x4 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_i64x4(self)[i] + } +} +impl Select> for mask64x4 { + #[inline(always)] + fn select(self, if_true: i64x4, if_false: i64x4) -> i64x4 { + self.simd.select_i64x4(self, if_true, if_false) + } +} +impl Bytes for i64x4 { + type Bytes = u8x32; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_i64x4(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_i64x4(value) + } +} +impl SimdBase for i64x4 { + type Element = i64; + const N: usize = 4; + type Mask = mask64x4; + type Block = i64x2; + type Array = [i64; 4]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[i64] { + self.simd.as_array_ref_i64x4(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [i64] { + self.simd.as_array_mut_i64x4(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[i64]) -> Self { + simd.load_array_ref_i64x4(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [i64]) { + self.simd + .store_array_i64x4(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: i64) -> Self { + simd.splat_i64x4(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + block.simd.combine_i64x2(block, block) + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i64) -> Self { + simd.load_array_i64x4([f(0usize), f(1usize), f(2usize), f(3usize)]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_i64x4::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_i64x4::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for i64x4 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_i64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_i64x4(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdSplit for i64x4 { + type Split = i64x2; + #[inline(always)] + fn split(self) -> (Self::Split, Self::Split) { + self.simd.split_i64x4(self) + } +} +impl crate::SimdCombine for i64x4 { + type Combined = i64x8; + #[inline(always)] + fn combine(self, rhs: impl SimdInto) -> Self::Combined { + self.simd.combine_i64x4(self, rhs.simd_into(self.simd)) + } +} +#[doc = "A SIMD vector of 4 [`u64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, u64x4};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = u64x4::splat(simd, 1);\n let b = u64x4::simd_from(simd, 1);\n\n // From a slice:\n let c = u64x4::from_slice(simd, &[1, 2, 3, 4]);\n\n // From an array:\n let d = u64x4::simd_from(simd, [1, 2, 3, 4]);\n\n // From an element-wise function:\n let e = u64x4::from_fn(simd, |i| i as u64);\n # use fearless_simd::u64x2;\n // From `Self::Block`:\n let f = u64x4::block_splat(u64x2::simd_from(simd, [1, 2]));\n}\n```"] +#[derive(Clone, Copy)] +#[repr(C, align(32))] +pub struct u64x4 { + pub(crate) val: S::u64x4, + pub simd: S, +} +impl Seal for u64x4 {} +impl SimdFrom<[u64; 4], S> for u64x4 { + #[inline(always)] + fn simd_from(simd: S, val: [u64; 4]) -> Self { + simd.load_array_u64x4(val) + } +} +impl From> for [u64; 4] { + #[inline(always)] + fn from(value: u64x4) -> Self { + value.simd.as_array_u64x4(value) + } +} +impl core::ops::Deref for u64x4 { + type Target = [u64; 4]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_u64x4(self) + } +} +impl core::ops::DerefMut for u64x4 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_u64x4(self) + } +} +impl core::fmt::Debug for u64x4 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "u64x4", &self.simd, self.simd.as_array_ref_u64x4(self)) + } +} +impl SimdFrom for u64x4 { + #[inline(always)] + fn simd_from(simd: S, value: u64) -> Self { + simd.splat_u64x4(value) + } +} +impl core::ops::Index for u64x4 { + type Output = u64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_u64x4(self)[i] + } +} +impl core::ops::IndexMut for u64x4 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_u64x4(self)[i] + } +} +impl Select> for mask64x4 { + #[inline(always)] + fn select(self, if_true: u64x4, if_false: u64x4) -> u64x4 { + self.simd.select_u64x4(self, if_true, if_false) + } +} +impl Bytes for u64x4 { + type Bytes = u8x32; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_u64x4(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_u64x4(value) + } +} +impl SimdBase for u64x4 { + type Element = u64; + const N: usize = 4; + type Mask = mask64x4; + type Block = u64x2; + type Array = [u64; 4]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[u64] { + self.simd.as_array_ref_u64x4(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [u64] { + self.simd.as_array_mut_u64x4(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[u64]) -> Self { + simd.load_array_ref_u64x4(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [u64]) { + self.simd + .store_array_u64x4(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: u64) -> Self { + simd.splat_u64x4(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + block.simd.combine_u64x2(block, block) + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u64) -> Self { + simd.load_array_u64x4([f(0usize), f(1usize), f(2usize), f(3usize)]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_u64x4::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_u64x4::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for u64x4 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_u64x4(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_u64x4(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdSplit for u64x4 { + type Split = u64x2; + #[inline(always)] + fn split(self) -> (Self::Split, Self::Split) { + self.simd.split_u64x4(self) + } +} +impl crate::SimdCombine for u64x4 { + type Combined = u64x8; + #[inline(always)] + fn combine(self, rhs: impl SimdInto) -> Self::Combined { + self.simd.combine_u64x4(self, rhs.simd_into(self.simd)) + } +} +#[doc = "A SIMD mask of 4 logical lanes corresponding to 64-bit vector elements.\n\nThe storage representation of this type is intentionally opaque and may vary depending on the SIMD level.\n\nYou can construct this mask type using the [`Self::splat`], [`Self::from_bitmask`], [`Self::from_slice`], and [`Self::simd_from`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, mask64x4};\nfn construct_mask(simd: S) {\n // From a single boolean value:\n let a = mask64x4::splat(simd, true);\n let b = mask64x4::simd_from(simd, true);\n\n // From signed integer mask lanes:\n let c = mask64x4::from_slice(simd, &[-1, 0, 0, 0]);\n let d = mask64x4::simd_from(simd, [-1, 0, 0, 0]);\n\n // From a compact bitmask (same mask as above, least significant bit maps to lane 0):\n let e = mask64x4::from_bitmask(simd, 0b0001);\n\n // By setting individual lanes:\n let mut f = mask64x4::splat(simd, false);\n f.set(0, true);\n}\n```"] +#[derive(Clone, Copy)] +pub struct mask64x4 { + pub(crate) val: S::mask64x4, + pub(crate) simd: S, +} +impl Seal for mask64x4 {} +impl SimdFrom<[i64; 4], S> for mask64x4 { + #[inline(always)] + fn simd_from(simd: S, val: [i64; 4]) -> Self { + simd.load_array_mask64x4(val) + } +} +impl From> for [i64; 4] { + #[inline(always)] + fn from(value: mask64x4) -> Self { + value.simd.as_array_mask64x4(value) + } +} +impl core::fmt::Debug for mask64x4 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let lanes = self.simd.as_array_mask64x4(*self); crate::support::simd_debug_impl(f, "mask64x4", &self.simd, &lanes) } @@ -4071,14 +4955,7 @@ impl SimdMask for mask64x4 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 4, - "mask lane index {index} is out of bounds for {} lanes", - 4 - ); - let mut lanes = self.simd.as_array_mask64x4(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask64x4(lanes); + self.simd.set_mask64x4(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i64]) -> Self { @@ -4227,8 +5104,25 @@ impl SimdBase for f32x16 { block2.simd.combine_f32x8(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f32) -> Self { - simd.load_array_f32x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f32) -> Self { + simd.load_array_f32x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -4482,8 +5376,73 @@ impl SimdBase for i8x64 { block2.simd.combine_i8x32(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i8) -> Self { - simd.load_array_i8x64(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i8) -> Self { + simd.load_array_i8x64([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + f(32usize), + f(33usize), + f(34usize), + f(35usize), + f(36usize), + f(37usize), + f(38usize), + f(39usize), + f(40usize), + f(41usize), + f(42usize), + f(43usize), + f(44usize), + f(45usize), + f(46usize), + f(47usize), + f(48usize), + f(49usize), + f(50usize), + f(51usize), + f(52usize), + f(53usize), + f(54usize), + f(55usize), + f(56usize), + f(57usize), + f(58usize), + f(59usize), + f(60usize), + f(61usize), + f(62usize), + f(63usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -4668,8 +5627,73 @@ impl SimdBase for u8x64 { block2.simd.combine_u8x32(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u8) -> Self { - simd.load_array_u8x64(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u8) -> Self { + simd.load_array_u8x64([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + f(32usize), + f(33usize), + f(34usize), + f(35usize), + f(36usize), + f(37usize), + f(38usize), + f(39usize), + f(40usize), + f(41usize), + f(42usize), + f(43usize), + f(44usize), + f(45usize), + f(46usize), + f(47usize), + f(48usize), + f(49usize), + f(50usize), + f(51usize), + f(52usize), + f(53usize), + f(54usize), + f(55usize), + f(56usize), + f(57usize), + f(58usize), + f(59usize), + f(60usize), + f(61usize), + f(62usize), + f(63usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -4801,14 +5825,7 @@ impl SimdMask for mask8x64 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 64, - "mask lane index {index} is out of bounds for {} lanes", - 64 - ); - let mut lanes = self.simd.as_array_mask8x64(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask8x64(lanes); + self.simd.set_mask8x64(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i8]) -> Self { @@ -4957,8 +5974,41 @@ impl SimdBase for i16x32 { block2.simd.combine_i16x16(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i16) -> Self { - simd.load_array_i16x32(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i16) -> Self { + simd.load_array_i16x32([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -5149,8 +6199,41 @@ impl SimdBase for u16x32 { block2.simd.combine_u16x16(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u16) -> Self { - simd.load_array_u16x32(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u16) -> Self { + simd.load_array_u16x32([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + f(16usize), + f(17usize), + f(18usize), + f(19usize), + f(20usize), + f(21usize), + f(22usize), + f(23usize), + f(24usize), + f(25usize), + f(26usize), + f(27usize), + f(28usize), + f(29usize), + f(30usize), + f(31usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -5283,14 +6366,7 @@ impl SimdMask for mask16x32 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 32, - "mask lane index {index} is out of bounds for {} lanes", - 32 - ); - let mut lanes = self.simd.as_array_mask16x32(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask16x32(lanes); + self.simd.set_mask16x32(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i16]) -> Self { @@ -5439,8 +6515,25 @@ impl SimdBase for i32x16 { block2.simd.combine_i32x8(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> i32) -> Self { - simd.load_array_i32x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i32) -> Self { + simd.load_array_i32x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -5643,8 +6736,25 @@ impl SimdBase for u32x16 { block2.simd.combine_u32x8(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> u32) -> Self { - simd.load_array_u32x16(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u32) -> Self { + simd.load_array_u32x16([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + f(8usize), + f(9usize), + f(10usize), + f(11usize), + f(12usize), + f(13usize), + f(14usize), + f(15usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -5713,7 +6823,7 @@ impl crate::SimdInt for u32x16 { } } impl SimdCvtTruncate> for u32x16 { - #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] + #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] #[inline(always)] fn truncate_from(x: f32x16) -> Self { x.simd.cvt_u32_f32x16(x) @@ -5789,14 +6899,7 @@ impl SimdMask for mask32x16 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 16, - "mask lane index {index} is out of bounds for {} lanes", - 16 - ); - let mut lanes = self.simd.as_array_mask32x16(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask32x16(lanes); + self.simd.set_mask32x16(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i32]) -> Self { @@ -5940,8 +7043,17 @@ impl SimdBase for f64x8 { block2.simd.combine_f64x4(block2, block2) } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> f64) -> Self { - simd.load_array_f64x8(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> f64) -> Self { + simd.load_array_f64x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) } #[inline(always)] fn slide(self, rhs: impl SimdInto) -> Self { @@ -6069,6 +7181,396 @@ impl crate::SimdSplit for f64x8 { self.simd.split_f64x8(self) } } +#[doc = "A SIMD vector of 8 [`i64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, i64x8};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = i64x8::splat(simd, 1);\n let b = i64x8::simd_from(simd, 1);\n\n // From a slice:\n let c = i64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]);\n\n // From an array:\n let d = i64x8::simd_from(simd, [1, 2, 3, 4, 5, 6, 7, 8]);\n\n // From an element-wise function:\n let e = i64x8::from_fn(simd, |i| i as i64);\n # use fearless_simd::i64x2;\n // From `Self::Block`:\n let f = i64x8::block_splat(i64x2::simd_from(simd, [1, 2]));\n}\n```"] +#[derive(Clone, Copy)] +#[repr(C, align(64))] +pub struct i64x8 { + pub(crate) val: S::i64x8, + pub simd: S, +} +impl Seal for i64x8 {} +impl SimdFrom<[i64; 8], S> for i64x8 { + #[inline(always)] + fn simd_from(simd: S, val: [i64; 8]) -> Self { + simd.load_array_i64x8(val) + } +} +impl From> for [i64; 8] { + #[inline(always)] + fn from(value: i64x8) -> Self { + value.simd.as_array_i64x8(value) + } +} +impl core::ops::Deref for i64x8 { + type Target = [i64; 8]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_i64x8(self) + } +} +impl core::ops::DerefMut for i64x8 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_i64x8(self) + } +} +impl core::fmt::Debug for i64x8 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "i64x8", &self.simd, self.simd.as_array_ref_i64x8(self)) + } +} +impl SimdFrom for i64x8 { + #[inline(always)] + fn simd_from(simd: S, value: i64) -> Self { + simd.splat_i64x8(value) + } +} +impl core::ops::Index for i64x8 { + type Output = i64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_i64x8(self)[i] + } +} +impl core::ops::IndexMut for i64x8 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_i64x8(self)[i] + } +} +impl Select> for mask64x8 { + #[inline(always)] + fn select(self, if_true: i64x8, if_false: i64x8) -> i64x8 { + self.simd.select_i64x8(self, if_true, if_false) + } +} +impl Bytes for i64x8 { + type Bytes = u8x64; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_i64x8(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_i64x8(value) + } +} +impl SimdBase for i64x8 { + type Element = i64; + const N: usize = 8; + type Mask = mask64x8; + type Block = i64x2; + type Array = [i64; 8]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[i64] { + self.simd.as_array_ref_i64x8(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [i64] { + self.simd.as_array_mut_i64x8(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[i64]) -> Self { + simd.load_array_ref_i64x8(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [i64]) { + self.simd + .store_array_i64x8(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: i64) -> Self { + simd.splat_i64x8(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + let block2 = block.simd.combine_i64x2(block, block); + block2.simd.combine_i64x4(block2, block2) + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> i64) -> Self { + simd.load_array_i64x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_i64x8::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_i64x8::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for i64x8 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_i64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_i64x8(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdSplit for i64x8 { + type Split = i64x4; + #[inline(always)] + fn split(self) -> (Self::Split, Self::Split) { + self.simd.split_i64x8(self) + } +} +#[doc = "A SIMD vector of 8 [`u64`] elements.\n\nYou may construct this vector type using the [`Self::splat`], [`Self::from_slice`], [`Self::simd_from`], [`Self::from_fn`], and [`Self::block_splat`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, u64x8};\nfn construct_simd(simd: S) {\n // From a single scalar value:\n let a = u64x8::splat(simd, 1);\n let b = u64x8::simd_from(simd, 1);\n\n // From a slice:\n let c = u64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]);\n\n // From an array:\n let d = u64x8::simd_from(simd, [1, 2, 3, 4, 5, 6, 7, 8]);\n\n // From an element-wise function:\n let e = u64x8::from_fn(simd, |i| i as u64);\n # use fearless_simd::u64x2;\n // From `Self::Block`:\n let f = u64x8::block_splat(u64x2::simd_from(simd, [1, 2]));\n}\n```"] +#[derive(Clone, Copy)] +#[repr(C, align(64))] +pub struct u64x8 { + pub(crate) val: S::u64x8, + pub simd: S, +} +impl Seal for u64x8 {} +impl SimdFrom<[u64; 8], S> for u64x8 { + #[inline(always)] + fn simd_from(simd: S, val: [u64; 8]) -> Self { + simd.load_array_u64x8(val) + } +} +impl From> for [u64; 8] { + #[inline(always)] + fn from(value: u64x8) -> Self { + value.simd.as_array_u64x8(value) + } +} +impl core::ops::Deref for u64x8 { + type Target = [u64; 8]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.simd.as_array_ref_u64x8(self) + } +} +impl core::ops::DerefMut for u64x8 { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.simd.as_array_mut_u64x8(self) + } +} +impl core::fmt::Debug for u64x8 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::support::simd_debug_impl(f, "u64x8", &self.simd, self.simd.as_array_ref_u64x8(self)) + } +} +impl SimdFrom for u64x8 { + #[inline(always)] + fn simd_from(simd: S, value: u64) -> Self { + simd.splat_u64x8(value) + } +} +impl core::ops::Index for u64x8 { + type Output = u64; + #[inline(always)] + fn index(&self, i: usize) -> &Self::Output { + &self.simd.as_array_ref_u64x8(self)[i] + } +} +impl core::ops::IndexMut for u64x8 { + #[inline(always)] + fn index_mut(&mut self, i: usize) -> &mut Self::Output { + &mut self.simd.as_array_mut_u64x8(self)[i] + } +} +impl Select> for mask64x8 { + #[inline(always)] + fn select(self, if_true: u64x8, if_false: u64x8) -> u64x8 { + self.simd.select_u64x8(self, if_true, if_false) + } +} +impl Bytes for u64x8 { + type Bytes = u8x64; + #[inline(always)] + fn to_bytes(self) -> Self::Bytes { + self.simd.cvt_to_bytes_u64x8(self) + } + #[inline(always)] + fn from_bytes(value: Self::Bytes) -> Self { + value.simd.cvt_from_bytes_u64x8(value) + } +} +impl SimdBase for u64x8 { + type Element = u64; + const N: usize = 8; + type Mask = mask64x8; + type Block = u64x2; + type Array = [u64; 8]; + #[inline(always)] + fn witness(&self) -> S { + self.simd + } + #[inline(always)] + fn as_slice(&self) -> &[u64] { + self.simd.as_array_ref_u64x8(self).as_slice() + } + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [u64] { + self.simd.as_array_mut_u64x8(self).as_mut_slice() + } + #[inline(always)] + fn from_slice(simd: S, slice: &[u64]) -> Self { + simd.load_array_ref_u64x8(slice.try_into().unwrap()) + } + #[inline(always)] + fn store_slice(&self, slice: &mut [u64]) { + self.simd + .store_array_u64x8(*self, slice.try_into().unwrap()); + } + #[inline(always)] + fn splat(simd: S, val: u64) -> Self { + simd.splat_u64x8(val) + } + #[inline(always)] + fn block_splat(block: Self::Block) -> Self { + let block2 = block.simd.combine_u64x2(block, block); + block2.simd.combine_u64x4(block2, block2) + } + #[inline(always)] + fn from_fn(simd: S, mut f: impl FnMut(usize) -> u64) -> Self { + simd.load_array_u64x8([ + f(0usize), + f(1usize), + f(2usize), + f(3usize), + f(4usize), + f(5usize), + f(6usize), + f(7usize), + ]) + } + #[inline(always)] + fn slide(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_u64x8::(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { + self.simd + .slide_within_blocks_u64x8::(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdInt for u64x8 { + #[inline(always)] + fn simd_eq(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_eq_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_lt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_lt_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_le(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_le_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_ge(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_ge_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn simd_gt(self, rhs: impl SimdInto) -> Self::Mask { + self.simd.simd_gt_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_low(self, rhs: impl SimdInto) -> Self { + self.simd.zip_low_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn zip_high(self, rhs: impl SimdInto) -> Self { + self.simd.zip_high_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_low(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_low_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn unzip_high(self, rhs: impl SimdInto) -> Self { + self.simd.unzip_high_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn interleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.interleave_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn deinterleave(self, rhs: impl SimdInto) -> (Self, Self) { + self.simd.deinterleave_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn min(self, rhs: impl SimdInto) -> Self { + self.simd.min_u64x8(self, rhs.simd_into(self.simd)) + } + #[inline(always)] + fn max(self, rhs: impl SimdInto) -> Self { + self.simd.max_u64x8(self, rhs.simd_into(self.simd)) + } +} +impl crate::SimdSplit for u64x8 { + type Split = u64x4; + #[inline(always)] + fn split(self) -> (Self::Split, Self::Split) { + self.simd.split_u64x8(self) + } +} #[doc = "A SIMD mask of 8 logical lanes corresponding to 64-bit vector elements.\n\nThe storage representation of this type is intentionally opaque and may vary depending on the SIMD level.\n\nYou can construct this mask type using the [`Self::splat`], [`Self::from_bitmask`], [`Self::from_slice`], and [`Self::simd_from`] methods.\n\n```rust\n# use fearless_simd::{prelude::*, mask64x8};\nfn construct_mask(simd: S) {\n // From a single boolean value:\n let a = mask64x8::splat(simd, true);\n let b = mask64x8::simd_from(simd, true);\n\n // From signed integer mask lanes:\n let c = mask64x8::from_slice(simd, &[-1, 0, 0, 0, 0, 0, 0, 0]);\n let d = mask64x8::simd_from(simd, [-1, 0, 0, 0, 0, 0, 0, 0]);\n\n // From a compact bitmask (same mask as above, least significant bit maps to lane 0):\n let e = mask64x8::from_bitmask(simd, 0b0001);\n\n // By setting individual lanes:\n let mut f = mask64x8::splat(simd, false);\n f.set(0, true);\n}\n```"] #[derive(Clone, Copy)] pub struct mask64x8 { @@ -6127,14 +7629,7 @@ impl SimdMask for mask64x8 { } #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < 8, - "mask lane index {index} is out of bounds for {} lanes", - 8 - ); - let mut lanes = self.simd.as_array_mask64x8(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.load_array_mask64x8(lanes); + self.simd.set_mask64x8(self, index, value); } #[inline(always)] fn from_slice(simd: S, slice: &[i64]) -> Self { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 5ebc8fe74..d61ce565a 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -6,9 +6,9 @@ use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; #[cfg(target_arch = "x86")] use core::arch::x86::*; @@ -44,6 +44,8 @@ impl ArchTypes for Sse4_2 { type u32x4 = crate::support::Aligned128<__m128i>; type mask32x4 = crate::support::Aligned128<__m128i>; type f64x2 = crate::support::Aligned128<__m128d>; + type i64x2 = crate::support::Aligned128<__m128i>; + type u64x2 = crate::support::Aligned128<__m128i>; type mask64x2 = crate::support::Aligned128<__m128i>; type f32x8 = crate::support::Aligned256<[__m128; 2usize]>; type i8x32 = crate::support::Aligned256<[__m128i; 2usize]>; @@ -56,6 +58,8 @@ impl ArchTypes for Sse4_2 { type u32x8 = crate::support::Aligned256<[__m128i; 2usize]>; type mask32x8 = crate::support::Aligned256<[__m128i; 2usize]>; type f64x4 = crate::support::Aligned256<[__m128d; 2usize]>; + type i64x4 = crate::support::Aligned256<[__m128i; 2usize]>; + type u64x4 = crate::support::Aligned256<[__m128i; 2usize]>; type mask64x4 = crate::support::Aligned256<[__m128i; 2usize]>; type f32x16 = crate::support::Aligned512<[__m128; 4usize]>; type i8x64 = crate::support::Aligned512<[__m128i; 4usize]>; @@ -68,6 +72,8 @@ impl ArchTypes for Sse4_2 { type u32x16 = crate::support::Aligned512<[__m128i; 4usize]>; type mask32x16 = crate::support::Aligned512<[__m128i; 4usize]>; type f64x8 = crate::support::Aligned512<[__m128d; 4usize]>; + type i64x8 = crate::support::Aligned512<[__m128i; 4usize]>; + type u64x8 = crate::support::Aligned512<[__m128i; 4usize]>; type mask64x8 = crate::support::Aligned512<[__m128i; 4usize]>; } impl Simd for Sse4_2 { @@ -79,6 +85,8 @@ impl Simd for Sse4_2 { type i16s = i16x8; type u32s = u32x4; type i32s = i32x4; + type u64s = u64x2; + type i64s = i64x2; type mask8s = mask8x16; type mask16s = mask16x8; type mask32s = mask32x4; @@ -796,7 +804,27 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i8x16(self, a: i8x16, shift: u32) -> i8x16 { @@ -816,7 +844,27 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { @@ -1161,7 +1209,27 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u8x16(self, a: u8x16, shift: u32) -> u8x16 { @@ -1181,7 +1249,27 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { @@ -1411,6 +1499,17 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask8x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x16(lanes); + } + #[inline(always)] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { crate::kernel!( #[inline(always)] @@ -1675,7 +1774,19 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i16x8(self, a: i16x8, shift: u32) -> i16x8 { @@ -1689,7 +1800,19 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { @@ -2021,7 +2144,19 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u16x8(self, a: u16x8, shift: u32) -> u16x8 { @@ -2035,7 +2170,19 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { @@ -2260,6 +2407,17 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask16x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x8(lanes); + } + #[inline(always)] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -2524,7 +2682,15 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i32; 4usize] = a.into(); + let b: [i32; 4usize] = b.into(); + let result: [i32; 4usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i32x4(self, a: i32x4, shift: u32) -> i32x4 { @@ -2538,7 +2704,15 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i32; 4usize] = a.into(); + let b: [i32; 4usize] = b.into(); + let result: [i32; 4usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { @@ -2878,7 +3052,15 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shlv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u32; 4usize] = a.into(); + let b: [u32; 4usize] = b.into(); + let result: [u32; 4usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u32x4(self, a: u32x4, shift: u32) -> u32x4 { @@ -2892,7 +3074,15 @@ impl Simd for Sse4_2 { } #[inline(always)] fn shrv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u32; 4usize] = a.into(); + let b: [u32; 4usize] = b.into(); + let result: [u32; 4usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { @@ -3121,6 +3311,17 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask32x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x4(lanes); + } + #[inline(always)] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { crate::kernel!( #[inline(always)] @@ -3626,4773 +3827,5409 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] - fn splat_mask64x2(self, val: bool) -> mask64x2 { + fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, val: bool) -> mask64x2 { - let val: i64 = if val { !0 } else { 0 }; + fn kernel(token: Sse4_2, val: i64) -> i64x2 { _mm_set1_epi64x(val).simd_into(token) } ); kernel(self, val) } #[inline(always)] - fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { - mask64x2 { + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0) } #[inline(always)] - fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [i64; 2usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [i64; 2usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_alignr_128( + self, + self.cvt_to_bytes_i64x2(b).val.0, + self.cvt_to_bytes_i64x2(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, bits: u64) -> mask64x2 { - { - let bit_lanes = _mm_set1_epi64x(bits.cast_signed()); - let bit_mask = _mm_set_epi64x(2, 1); - _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask) - } - .simd_into(token) + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, bits) + kernel(self, a, b) } #[inline(always)] - fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2) -> u64 { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64 + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { _mm_and_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { _mm_or_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { _mm_xor_si128(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn not_mask64x2(self, a: mask64x2) -> mask64x2 { - self.xor_mask64x2(a, self.splat_mask64x2(true)) + fn not_i64x2(self, a: i64x2) -> i64x2 { + a ^ !0 } #[inline(always)] - fn select_mask64x2( - self, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel( - token: Sse4_2, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { - _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + fn kernel(token: Sse4_2, a: i64x2, shift: u32) -> i64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) } ); - kernel(self, a, b, c) + kernel(self, a, shift) } #[inline(always)] - fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let result: [i64; 2usize] = [ + core::ops::Shr::shr(a[0usize], shift), + core::ops::Shr::shr(a[1usize], shift), + ]; + result.simd_into(self) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> mask64x2 { _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) } #[inline(always)] - fn any_true_mask64x2(self, a: mask64x2) -> bool { + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0 + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_true_mask64x2(self, a: mask64x2) -> bool { + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11 + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn any_false_mask64x2(self, a: mask64x2) -> bool { + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11 + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn all_false_mask64x2(self, a: mask64x2) -> bool { + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: mask64x2) -> bool { - _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0 + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> mask64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) } ); - kernel(self, a) + kernel(self, a, b) } #[inline(always)] - fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { - mask64x4 { + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, b: i64x2) -> i64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) + } + #[inline(always)] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b)) + } + #[inline(always)] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + a: mask64x2, + b: i64x2, + c: i64x2, + ) -> i64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) + } + #[inline(always)] + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) + } + #[inline(always)] + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) + } + #[inline(always)] + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { + i64x4 { val: crate::support::Aligned256([a.val.0, b.val.0]), simd: self, } } #[inline(always)] - fn splat_f32x8(self, val: f32) -> f32x8 { - let half = self.splat_f32x4(val); - self.combine_f32x4(half, half) + fn neg_i64x2(self, a: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2) -> i64x2 { + _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { - f32x8 { + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn splat_u64x2(self, val: u64) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, val: u64) -> u64x2 { + _mm_set1_epi64x(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { - f32x8 { + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128; 2usize], [f32; 8usize]>(&a.val.0) + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u64; 2usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { - crate::transmute::checked_cast_ref::<[__m128; 2usize], [f32; 8usize]>(&a.val.0) + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + crate::transmute::checked_cast_ref::<__m128i, [u64; 2usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { - crate::transmute::checked_cast_mut::<[__m128; 2usize], [f32; 8usize]>(&mut a.val.0) + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + crate::transmute::checked_cast_mut::<__m128i, [u64; 2usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { - f32x8 { + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { - u8x32 { + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - if SHIFT >= 8usize { + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + if SHIFT >= 2usize { return b; } - let result = cross_block_alignr_128x2( + let result = dyn_alignr_128( self, - self.cvt_to_bytes_f32x8(b).val.0, - self.cvt_to_bytes_f32x8(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_u64x2(b).val.0, + self.cvt_to_bytes_u64x2(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_f32x8(u8x32 { - val: crate::support::Aligned256(result), + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x8( + fn slide_within_blocks_u64x2( self, - a: f32x8, - b: f32x8, - ) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.slide_within_blocks_f32x4::(a0, b0), - self.slide_within_blocks_f32x4::(a1, b1), - ) + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) } #[inline(always)] - fn abs_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn neg_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) - } + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_sub_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } #[inline(always)] - fn sqrt_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + a[0usize].wrapping_mul(b[0usize]), + a[1usize].wrapping_mul(b[1usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.approximate_recip_f32x4(a0), - self.approximate_recip_f32x4(a1), - ) + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) + fn not_u64x2(self, a: u64x2) -> u64x2 { + a ^ !0 } #[inline(always)] - fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, shift: u32) -> u64x2 { + _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) } #[inline(always)] - fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, shift: u32) -> u64x2 { + _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token) + } + ); + kernel(self, a, shift) } #[inline(always)] - fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> mask64x2 { + _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, _) = self.split_f32x8(a); - let (b0, _) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (_, a1) = self.split_f32x8(a); - let (_, b1) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_lo = self.zip_low_f32x4(a0, b0); - let lo_hi = self.zip_high_f32x4(a0, b0); - let hi_lo = self.zip_low_f32x4(a1, b1); - let hi_hi = self.zip_high_f32x4(a1, b1); - ( - self.combine_f32x4(lo_lo, lo_hi), - self.combine_f32x4(hi_lo, hi_hi), - ) + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_even = self.unzip_low_f32x4(a0, a1); - let lo_odd = self.unzip_high_f32x4(a0, a1); - let hi_even = self.unzip_low_f32x4(b0, b1); - let hi_odd = self.unzip_high_f32x4(b0, b1); - ( - self.combine_f32x4(lo_even, hi_even), - self.combine_f32x4(lo_odd, hi_odd), - ) + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, b: u64x2) -> u64x2 { + _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) } #[inline(always)] - fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b)) } #[inline(always)] - fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.max_precise_f32x4(a0, b0), - self.max_precise_f32x4(a1, b1), - ) + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + a: mask64x2, + b: u64x2, + c: u64x2, + ) -> u64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) } #[inline(always)] - fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.min_precise_f32x4(a0, b0), - self.min_precise_f32x4(a1, b1), - ) + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_add_f32x4(a0, b0, c0), - self.mul_add_f32x4(a1, b1, c1), - ) + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_sub_f32x4(a0, b0, c0), - self.mul_sub_f32x4(a1, b1, c1), - ) + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { + u64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } } #[inline(always)] - fn floor_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2) -> u8x16 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn ceil_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2) -> u32x4 { + __m128i::from(a).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.round_ties_even_f32x4(a0), - self.round_ties_even_f32x4(a1), - ) + fn splat_mask64x2(self, val: bool) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, val: bool) -> mask64x2 { + let val: i64 = if val { !0 } else { 0 }; + _mm_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) } #[inline(always)] - fn fract_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { + mask64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn trunc_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0) } #[inline(always)] - fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, bits: u64) -> mask64x2 { + { + let bit_lanes = _mm_set1_epi64x(bits.cast_signed()); + let bit_mask = _mm_set_epi64x(2, 1); + _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask) + } + .simd_into(token) + } + ); + kernel(self, bits) } #[inline(always)] - fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { - f32x16 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2) -> u64 { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64 + } + ); + kernel(self, a) } #[inline(always)] - fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { - ( - f32x4 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - f32x4 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let mut lanes = self.as_array_mask64x2(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x2(lanes); } #[inline(always)] - fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f64x2( - self.reinterpret_f64_f32x4(a0), - self.reinterpret_f64_f32x4(a1), - ) + fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.reinterpret_i32_f32x4(a0), - self.reinterpret_i32_f32x4(a1), - ) + fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_or_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_xor_si128(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.reinterpret_u32_f32x4(a0), - self.reinterpret_u32_f32x4(a1), - ) + fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + self.xor_mask64x2(a, self.splat_mask64x2(true)) } #[inline(always)] - fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) + fn select_mask64x2( + self, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) } #[inline(always)] - fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.cvt_u32_precise_f32x4(a0), - self.cvt_u32_precise_f32x4(a1), - ) + fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2, b: mask64x2) -> mask64x2 { + _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) } #[inline(always)] - fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + fn any_true_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0 + } + ); + kernel(self, a) } #[inline(always)] - fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.cvt_i32_precise_f32x4(a0), - self.cvt_i32_precise_f32x4(a1), - ) + fn all_true_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11 + } + ); + kernel(self, a) } #[inline(always)] - fn splat_i8x32(self, val: i8) -> i8x32 { - let half = self.splat_i8x16(val); - self.combine_i8x16(half, half) + fn any_false_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11 + } + ); + kernel(self, a) } #[inline(always)] - fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { - i8x32 { + fn all_false_mask64x2(self, a: mask64x2) -> bool { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask64x2) -> bool { + _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0 + } + ); + kernel(self, a) + } + #[inline(always)] + fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { + mask64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn splat_f32x8(self, val: f32) -> f32x8 { + let half = self.splat_f32x4(val); + self.combine_f32x4(half, half) + } + #[inline(always)] + fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) + fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128; 2usize], [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) + fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { + crate::transmute::checked_cast_ref::<[__m128; 2usize], [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i8; 32usize]>(&mut a.val.0) + fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { + crate::transmute::checked_cast_mut::<[__m128; 2usize], [f32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { - i8x32 { + fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - if SHIFT >= 32usize { + fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_i8x32(b).val.0, - self.cvt_to_bytes_i8x32(a).val.0, - SHIFT, + self.cvt_to_bytes_f32x8(b).val.0, + self.cvt_to_bytes_f32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i8x32(u8x32 { + self.cvt_from_bytes_f32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x32( + fn slide_within_blocks_f32x8( self, - a: i8x32, - b: i8x32, - ) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16( - self.slide_within_blocks_i8x16::(a0, b0), - self.slide_within_blocks_i8x16::(a1, b1), + a: f32x8, + b: f32x8, + ) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.slide_within_blocks_f32x4::(a0, b0), + self.slide_within_blocks_f32x4::(a1, b1), ) } #[inline(always)] - fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) - } - #[inline(always)] - fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) - } - #[inline(always)] - fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) + fn abs_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) } #[inline(always)] - fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) + fn neg_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) } #[inline(always)] - fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) + fn sqrt_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) } #[inline(always)] - fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) + fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.approximate_recip_f32x4(a0), + self.approximate_recip_f32x4(a1), + ) } #[inline(always)] - fn not_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) + fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) } #[inline(always)] - fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) + fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) } #[inline(always)] - fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) + fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) } #[inline(always)] - fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) + fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) } #[inline(always)] - fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) + fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) } #[inline(always)] - fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) + fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) } #[inline(always)] - fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) + fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) } #[inline(always)] - fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) + fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) } #[inline(always)] - fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) + fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) } #[inline(always)] - fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) + fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) } #[inline(always)] - fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, _) = self.split_i8x32(a); - let (b0, _) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) + fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, _) = self.split_f32x8(a); + let (b0, _) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) } #[inline(always)] - fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (_, a1) = self.split_i8x32(a); - let (_, b1) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) + fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (_, a1) = self.split_f32x8(a); + let (_, b1) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) } #[inline(always)] - fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) + fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) } #[inline(always)] - fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) + fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) } #[inline(always)] - fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_lo = self.zip_low_i8x16(a0, b0); - let lo_hi = self.zip_high_i8x16(a0, b0); - let hi_lo = self.zip_low_i8x16(a1, b1); - let hi_hi = self.zip_high_i8x16(a1, b1); + fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_lo = self.zip_low_f32x4(a0, b0); + let lo_hi = self.zip_high_f32x4(a0, b0); + let hi_lo = self.zip_low_f32x4(a1, b1); + let hi_hi = self.zip_high_f32x4(a1, b1); ( - self.combine_i8x16(lo_lo, lo_hi), - self.combine_i8x16(hi_lo, hi_hi), + self.combine_f32x4(lo_lo, lo_hi), + self.combine_f32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_even = self.unzip_low_i8x16(a0, a1); - let lo_odd = self.unzip_high_i8x16(a0, a1); - let hi_even = self.unzip_low_i8x16(b0, b1); - let hi_odd = self.unzip_high_i8x16(b0, b1); + fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_even = self.unzip_low_f32x4(a0, a1); + let lo_odd = self.unzip_high_f32x4(a0, a1); + let hi_even = self.unzip_low_f32x4(b0, b1); + let hi_odd = self.unzip_high_f32x4(b0, b1); ( - self.combine_i8x16(lo_even, hi_even), - self.combine_i8x16(lo_odd, hi_odd), + self.combine_f32x4(lo_even, hi_even), + self.combine_f32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_i8x32(b); - let (c0, c1) = self.split_i8x32(c); - self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) + fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) } #[inline(always)] - fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) + fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) } #[inline(always)] - fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) + fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.max_precise_f32x4(a0, b0), + self.max_precise_f32x4(a1, b1), + ) } #[inline(always)] - fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { - i8x64 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.min_precise_f32x4(a0, b0), + self.min_precise_f32x4(a1, b1), + ) } #[inline(always)] - fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { - ( - i8x16 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - i8x16 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, + fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_add_f32x4(a0, b0, c0), + self.mul_add_f32x4(a1, b1, c1), ) } #[inline(always)] - fn neg_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) + fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_sub_f32x4(a0, b0, c0), + self.mul_sub_f32x4(a1, b1, c1), + ) } #[inline(always)] - fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + fn floor_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) } #[inline(always)] - fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { - let (a0, a1) = self.split_i8x32(a); + fn ceil_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + } + #[inline(always)] + fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.round_ties_even_f32x4(a0), + self.round_ties_even_f32x4(a1), + ) + } + #[inline(always)] + fn fract_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + } + #[inline(always)] + fn trunc_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + } + #[inline(always)] + fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + } + #[inline(always)] + fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { + f32x16 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { + ( + f32x4 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + f32x4 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] + fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f64x2( + self.reinterpret_f64_f32x4(a0), + self.reinterpret_f64_f32x4(a1), + ) + } + #[inline(always)] + fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.reinterpret_i32_f32x4(a0), + self.reinterpret_i32_f32x4(a1), + ) + } + #[inline(always)] + fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); self.combine_u32x4( - self.reinterpret_u32_i8x16(a0), - self.reinterpret_u32_i8x16(a1), + self.reinterpret_u32_f32x4(a0), + self.reinterpret_u32_f32x4(a1), ) } #[inline(always)] - fn splat_u8x32(self, val: u8) -> u8x32 { - let half = self.splat_u8x16(val); - self.combine_u8x16(half, half) + fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) } #[inline(always)] - fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { - u8x32 { + fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4( + self.cvt_u32_precise_f32x4(a0), + self.cvt_u32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + } + #[inline(always)] + fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.cvt_i32_precise_f32x4(a0), + self.cvt_i32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn splat_i8x32(self, val: i8) -> i8x32 { + let half = self.splat_i8x16(val); + self.combine_i8x16(half, half) + } + #[inline(always)] + fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { - u8x32 { + fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0) + fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0) + fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u8; 32usize]>(&mut a.val.0) + fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i8; 32usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { - u8x32 { + fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { if SHIFT >= 32usize { return b; } let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_u8x32(b).val.0, - self.cvt_to_bytes_u8x32(a).val.0, + self.cvt_to_bytes_i8x32(b).val.0, + self.cvt_to_bytes_i8x32(a).val.0, SHIFT, ); - self.cvt_from_bytes_u8x32(u8x32 { + self.cvt_from_bytes_i8x32(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x32( + fn slide_within_blocks_i8x32( self, - a: u8x32, - b: u8x32, - ) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16( - self.slide_within_blocks_u8x16::(a0, b0), - self.slide_within_blocks_u8x16::(a1, b1), + a: i8x32, + b: i8x32, + ) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16( + self.slide_within_blocks_i8x16::(a0, b0), + self.slide_within_blocks_i8x16::(a1, b1), ) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) - } - #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) + fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) } #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) + fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) } #[inline(always)] - fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) + fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) } #[inline(always)] - fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) + fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) } #[inline(always)] - fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) + fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) } #[inline(always)] - fn not_u8x32(self, a: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) + fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) } #[inline(always)] - fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) + fn not_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) } #[inline(always)] - fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) + fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) } #[inline(always)] - fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) + fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) } #[inline(always)] - fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) + fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) } #[inline(always)] - fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) + fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) } #[inline(always)] - fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) + fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) } #[inline(always)] - fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) + fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) } #[inline(always)] - fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) + fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) } #[inline(always)] - fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) + fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) } #[inline(always)] - fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, _) = self.split_u8x32(a); - let (b0, _) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) + fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) } #[inline(always)] - fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (_, a1) = self.split_u8x32(a); - let (_, b1) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) + fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, _) = self.split_i8x32(a); + let (b0, _) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) } #[inline(always)] - fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) + fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (_, a1) = self.split_i8x32(a); + let (_, b1) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) } #[inline(always)] - fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) + fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) } #[inline(always)] - fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_lo = self.zip_low_u8x16(a0, b0); - let lo_hi = self.zip_high_u8x16(a0, b0); - let hi_lo = self.zip_low_u8x16(a1, b1); - let hi_hi = self.zip_high_u8x16(a1, b1); + fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) + } + #[inline(always)] + fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_lo = self.zip_low_i8x16(a0, b0); + let lo_hi = self.zip_high_i8x16(a0, b0); + let hi_lo = self.zip_low_i8x16(a1, b1); + let hi_hi = self.zip_high_i8x16(a1, b1); ( - self.combine_u8x16(lo_lo, lo_hi), - self.combine_u8x16(hi_lo, hi_hi), + self.combine_i8x16(lo_lo, lo_hi), + self.combine_i8x16(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_even = self.unzip_low_u8x16(a0, a1); - let lo_odd = self.unzip_high_u8x16(a0, a1); - let hi_even = self.unzip_low_u8x16(b0, b1); - let hi_odd = self.unzip_high_u8x16(b0, b1); + fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_even = self.unzip_low_i8x16(a0, a1); + let lo_odd = self.unzip_high_i8x16(a0, a1); + let hi_even = self.unzip_low_i8x16(b0, b1); + let hi_odd = self.unzip_high_i8x16(b0, b1); ( - self.combine_u8x16(lo_even, hi_even), - self.combine_u8x16(lo_odd, hi_odd), + self.combine_i8x16(lo_even, hi_even), + self.combine_i8x16(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_u8x32(b); - let (c0, c1) = self.split_u8x32(c); - self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) + let (b0, b1) = self.split_i8x32(b); + let (c0, c1) = self.split_i8x32(c); + self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) } #[inline(always)] - fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) + fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) } #[inline(always)] - fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) + fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) } #[inline(always)] - fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { - u8x64 { + fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { + i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { ( - u8x16 { + i8x16 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u8x16 { + i8x16 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn widen_u8x32(self, a: u8x32) -> u16x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) + fn neg_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) } #[inline(always)] - fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { - let (a0, a1) = self.split_u8x32(a); + fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + } + #[inline(always)] + fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { + let (a0, a1) = self.split_i8x32(a); self.combine_u32x4( - self.reinterpret_u32_u8x16(a0), - self.reinterpret_u32_u8x16(a1), + self.reinterpret_u32_i8x16(a0), + self.reinterpret_u32_i8x16(a1), ) } #[inline(always)] - fn splat_mask8x32(self, val: bool) -> mask8x32 { - let half = self.splat_mask8x16(val); - self.combine_mask8x16(half, half) + fn splat_u8x32(self, val: u8) -> u8x32 { + let half = self.splat_u8x16(val); + self.combine_u8x16(half, half) } #[inline(always)] - fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { - mask8x32 { + fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) + fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { - let lo = self.from_bitmask_mask8x16(bits); - let hi = self.from_bitmask_mask8x16(bits >> 16usize); - self.combine_mask8x16(lo, hi) + fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0) } #[inline(always)] - fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { - let (lo, hi) = self.split_mask8x32(a); - let lo = self.to_bitmask_mask8x16(lo); - let hi = self.to_bitmask_mask8x16(hi); - lo | (hi << 16usize) + fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0) } #[inline(always)] - fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) + fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u8; 32usize]>(&mut a.val.0) } #[inline(always)] - fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) + fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) + fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn not_mask8x32(self, a: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) + fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn select_mask8x32( + fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_alignr_128x2( + self, + self.cvt_to_bytes_u8x32(b).val.0, + self.cvt_to_bytes_u8x32(a).val.0, + SHIFT, + ); + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u8x32( self, - a: mask8x32, - b: mask8x32, - c: mask8x32, - ) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - let (c0, c1) = self.split_mask8x32(c); - self.combine_mask8x16( - self.select_mask8x16(a0, b0, c0), - self.select_mask8x16(a1, b1, c1), + a: u8x32, + b: u8x32, + ) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16( + self.slide_within_blocks_u8x16::(a0, b0), + self.slide_within_blocks_u8x16::(a1, b1), ) } #[inline(always)] - fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) } #[inline(always)] - fn any_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) } #[inline(always)] - fn all_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) } #[inline(always)] - fn any_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) } #[inline(always)] - fn all_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) } #[inline(always)] - fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { - mask8x64 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) } #[inline(always)] - fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { - ( - mask8x16 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - mask8x16 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn not_u8x32(self, a: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) } #[inline(always)] - fn splat_i16x16(self, val: i16) -> i16x16 { - let half = self.splat_i16x8(val); - self.combine_i16x8(half, half) + fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) } #[inline(always)] - fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) } #[inline(always)] - fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) } #[inline(always)] - fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) + fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) } #[inline(always)] - fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) + fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) } #[inline(always)] - fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i16; 16usize]>(&mut a.val.0) + fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) } #[inline(always)] - fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) } #[inline(always)] - fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) } #[inline(always)] - fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - if SHIFT >= 16usize { - return b; - } - let result = cross_block_alignr_128x2( - self, - self.cvt_to_bytes_i16x16(b).val.0, - self.cvt_to_bytes_i16x16(a).val.0, - SHIFT * 2usize, - ); - self.cvt_from_bytes_i16x16(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, _) = self.split_u8x32(a); + let (b0, _) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) } #[inline(always)] - fn slide_within_blocks_i16x16( - self, - a: i16x16, - b: i16x16, - ) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8( - self.slide_within_blocks_i16x8::(a0, b0), - self.slide_within_blocks_i16x8::(a1, b1), - ) + fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (_, a1) = self.split_u8x32(a); + let (_, b1) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) } #[inline(always)] - fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) } #[inline(always)] - fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_lo = self.zip_low_u8x16(a0, b0); + let lo_hi = self.zip_high_u8x16(a0, b0); + let hi_lo = self.zip_low_u8x16(a1, b1); + let hi_hi = self.zip_high_u8x16(a1, b1); + ( + self.combine_u8x16(lo_lo, lo_hi), + self.combine_u8x16(hi_lo, hi_hi), + ) } #[inline(always)] - fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_even = self.unzip_low_u8x16(a0, a1); + let lo_odd = self.unzip_high_u8x16(a0, a1); + let hi_even = self.unzip_low_u8x16(b0, b1); + let hi_odd = self.unzip_high_u8x16(b0, b1); + ( + self.combine_u8x16(lo_even, hi_even), + self.combine_u8x16(lo_odd, hi_odd), + ) } #[inline(always)] - fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_u8x32(b); + let (c0, c1) = self.split_u8x32(c); + self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) } #[inline(always)] - fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) + fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) } #[inline(always)] - fn not_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) + fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) } #[inline(always)] - fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) + fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { + u8x64 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) + fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + ( + u8x16 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + u8x16 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] - fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) + fn widen_u8x32(self, a: u8x32) -> u16x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) } #[inline(always)] - fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) + fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u32x4( + self.reinterpret_u32_u8x16(a0), + self.reinterpret_u32_u8x16(a1), + ) } #[inline(always)] - fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) + fn splat_mask8x32(self, val: bool) -> mask8x32 { + let half = self.splat_mask8x16(val); + self.combine_mask8x16(half, half) } #[inline(always)] - fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) + fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { + mask8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) + fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) + fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { + let lo = self.from_bitmask_mask8x16(bits); + let hi = self.from_bitmask_mask8x16(bits >> 16usize); + self.combine_mask8x16(lo, hi) } #[inline(always)] - fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) + fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { + let (lo, hi) = self.split_mask8x32(a); + let lo = self.to_bitmask_mask8x16(lo); + let hi = self.to_bitmask_mask8x16(hi); + lo | (hi << 16usize) } #[inline(always)] - fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, _) = self.split_i16x16(a); - let (b0, _) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask8x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x32(lanes); } #[inline(always)] - fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (_, a1) = self.split_i16x16(a); - let (_, b1) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) + fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) } #[inline(always)] - fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) + fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) } #[inline(always)] - fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) + fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) } #[inline(always)] - fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_lo = self.zip_low_i16x8(a0, b0); - let lo_hi = self.zip_high_i16x8(a0, b0); - let hi_lo = self.zip_low_i16x8(a1, b1); - let hi_hi = self.zip_high_i16x8(a1, b1); - ( - self.combine_i16x8(lo_lo, lo_hi), - self.combine_i16x8(hi_lo, hi_hi), - ) + fn not_mask8x32(self, a: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) } #[inline(always)] - fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_even = self.unzip_low_i16x8(a0, a1); - let lo_odd = self.unzip_high_i16x8(a0, a1); - let hi_even = self.unzip_low_i16x8(b0, b1); - let hi_odd = self.unzip_high_i16x8(b0, b1); - ( - self.combine_i16x8(lo_even, hi_even), - self.combine_i16x8(lo_odd, hi_odd), + fn select_mask8x32( + self, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + let (c0, c1) = self.split_mask8x32(c); + self.combine_mask8x16( + self.select_mask8x16(a0, b0, c0), + self.select_mask8x16(a1, b1, c1), ) } #[inline(always)] - fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_i16x16(b); - let (c0, c1) = self.split_i16x16(c); - self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) + fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) } #[inline(always)] - fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) + fn any_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) } #[inline(always)] - fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) + fn all_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) } #[inline(always)] - fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { - i16x32 { + fn any_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + } + #[inline(always)] + fn all_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + } + #[inline(always)] + fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { + mask8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { + fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { ( - i16x8 { + mask8x16 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i16x8 { + mask8x16 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) - } - #[inline(always)] - fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u32x4( - self.reinterpret_u32_i16x8(a0), - self.reinterpret_u32_i16x8(a1), - ) - } - #[inline(always)] - fn splat_u16x16(self, val: u16) -> u16x16 { - let half = self.splat_u16x8(val); - self.combine_u16x8(half, half) + fn splat_i16x16(self, val: i16) -> i16x16 { + let half = self.splat_i16x8(val); + self.combine_i16x8(half, half) } #[inline(always)] - fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { - u16x16 { + fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { - u16x16 { + fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0) + fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0) + fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u16; 16usize]>(&mut a.val.0) + fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i16; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { - u16x16 { + fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { if SHIFT >= 16usize { return b; } let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_u16x16(b).val.0, - self.cvt_to_bytes_u16x16(a).val.0, + self.cvt_to_bytes_i16x16(b).val.0, + self.cvt_to_bytes_i16x16(a).val.0, SHIFT * 2usize, ); - self.cvt_from_bytes_u16x16(u8x32 { + self.cvt_from_bytes_i16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u16x16( + fn slide_within_blocks_i16x16( self, - a: u16x16, - b: u16x16, - ) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8( - self.slide_within_blocks_u16x8::(a0, b0), - self.slide_within_blocks_u16x8::(a1, b1), + a: i16x16, + b: i16x16, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8( + self.slide_within_blocks_i16x8::(a0, b0), + self.slide_within_blocks_i16x8::(a1, b1), ) } #[inline(always)] - fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) - } - #[inline(always)] - fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) } #[inline(always)] - fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) + fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) } #[inline(always)] - fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) + fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) } #[inline(always)] - fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) + fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) } #[inline(always)] - fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) + fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) } #[inline(always)] - fn not_u16x16(self, a: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) + fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) } #[inline(always)] - fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) + fn not_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) } #[inline(always)] - fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) + fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) } #[inline(always)] - fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) + fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) } #[inline(always)] - fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) + fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) } #[inline(always)] - fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) + fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) } #[inline(always)] - fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) + fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) } #[inline(always)] - fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) + fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) } #[inline(always)] - fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) + fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) } #[inline(always)] - fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) + fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) } #[inline(always)] - fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, _) = self.split_u16x16(a); - let (b0, _) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) + fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) } #[inline(always)] - fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (_, a1) = self.split_u16x16(a); - let (_, b1) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) + fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, _) = self.split_i16x16(a); + let (b0, _) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) } #[inline(always)] - fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) + fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (_, a1) = self.split_i16x16(a); + let (_, b1) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) } #[inline(always)] - fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) + fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) } #[inline(always)] - fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_lo = self.zip_low_u16x8(a0, b0); - let lo_hi = self.zip_high_u16x8(a0, b0); - let hi_lo = self.zip_low_u16x8(a1, b1); - let hi_hi = self.zip_high_u16x8(a1, b1); + fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) + } + #[inline(always)] + fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_lo = self.zip_low_i16x8(a0, b0); + let lo_hi = self.zip_high_i16x8(a0, b0); + let hi_lo = self.zip_low_i16x8(a1, b1); + let hi_hi = self.zip_high_i16x8(a1, b1); ( - self.combine_u16x8(lo_lo, lo_hi), - self.combine_u16x8(hi_lo, hi_hi), + self.combine_i16x8(lo_lo, lo_hi), + self.combine_i16x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_even = self.unzip_low_u16x8(a0, a1); - let lo_odd = self.unzip_high_u16x8(a0, a1); - let hi_even = self.unzip_low_u16x8(b0, b1); - let hi_odd = self.unzip_high_u16x8(b0, b1); + fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_even = self.unzip_low_i16x8(a0, a1); + let lo_odd = self.unzip_high_i16x8(a0, a1); + let hi_even = self.unzip_low_i16x8(b0, b1); + let hi_odd = self.unzip_high_i16x8(b0, b1); ( - self.combine_u16x8(lo_even, hi_even), - self.combine_u16x8(lo_odd, hi_odd), + self.combine_i16x8(lo_even, hi_even), + self.combine_i16x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_u16x16(b); - let (c0, c1) = self.split_u16x16(c); - self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) + let (b0, b1) = self.split_i16x16(b); + let (c0, c1) = self.split_i16x16(c); + self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) } #[inline(always)] - fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) + fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) } #[inline(always)] - fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) + fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { + fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { + i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { ( - u16x8 { + i16x8 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u16x8 { + i16x8 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn narrow_u16x16(self, a: u16x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u16x16) -> u8x16 { - let (a, b) = token.split_u16x16(a); - let mask = _mm_set1_epi16(0xFF); - let lo_masked = _mm_and_si128(a.into(), mask); - let hi_masked = _mm_and_si128(b.into(), mask); - let result = _mm_packus_epi16(lo_masked, hi_masked); - result.simd_into(token) - } - ); - kernel(self, a) + fn neg_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) } #[inline(always)] - fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) + fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { + let (a0, a1) = self.split_i16x16(a); + self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) } #[inline(always)] - fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { - let (a0, a1) = self.split_u16x16(a); + fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { + let (a0, a1) = self.split_i16x16(a); self.combine_u32x4( - self.reinterpret_u32_u16x8(a0), - self.reinterpret_u32_u16x8(a1), + self.reinterpret_u32_i16x8(a0), + self.reinterpret_u32_i16x8(a1), ) } #[inline(always)] - fn splat_mask16x16(self, val: bool) -> mask16x16 { - let half = self.splat_mask16x8(val); - self.combine_mask16x8(half, half) + fn splat_u16x16(self, val: u16) -> u16x16 { + let half = self.splat_u16x8(val); + self.combine_u16x8(half, half) } #[inline(always)] - fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { - mask16x16 { + fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) + fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { + u16x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { - let lo = self.from_bitmask_mask16x8(bits); - let hi = self.from_bitmask_mask16x8(bits >> 8usize); - self.combine_mask16x8(lo, hi) - } - #[inline(always)] - fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: mask16x16) -> u64 { - { - let packed = _mm_packs_epi16(a.val.0[0], a.val.0[1]); - _mm_movemask_epi8(packed) as u32 as u64 - } - } - ); - kernel(self, a) - } - #[inline(always)] - fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) - } - #[inline(always)] - fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) - } - #[inline(always)] - fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) - } - #[inline(always)] - fn not_mask16x16(self, a: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) - } - #[inline(always)] - fn select_mask16x16( - self, - a: mask16x16, - b: mask16x16, - c: mask16x16, - ) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - let (c0, c1) = self.split_mask16x16(c); - self.combine_mask16x8( - self.select_mask16x8(a0, b0, c0), - self.select_mask16x8(a1, b1, c1), - ) - } - #[inline(always)] - fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) - } - #[inline(always)] - fn any_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) - } - #[inline(always)] - fn all_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) - } - #[inline(always)] - fn any_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) - } - #[inline(always)] - fn all_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) - } - #[inline(always)] - fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { - mask16x32 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } - } - #[inline(always)] - fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { - ( - mask16x8 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - mask16x8 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn splat_i32x8(self, val: i32) -> i32x8 { - let half = self.splat_i32x4(val); - self.combine_i32x4(half, half) - } - #[inline(always)] - fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { - i32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { - i32x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) + fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) + fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i32; 8usize]>(&mut a.val.0) + fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u16; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { - i32x8 { + fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - if SHIFT >= 8usize { + fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + if SHIFT >= 16usize { return b; } let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_i32x8(b).val.0, - self.cvt_to_bytes_i32x8(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_u16x16(b).val.0, + self.cvt_to_bytes_u16x16(a).val.0, + SHIFT * 2usize, ); - self.cvt_from_bytes_i32x8(u8x32 { + self.cvt_from_bytes_u16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x8( + fn slide_within_blocks_u16x16( self, - a: i32x8, - b: i32x8, - ) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4( - self.slide_within_blocks_i32x4::(a0, b0), - self.slide_within_blocks_i32x4::(a1, b1), + a: u16x16, + b: u16x16, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8( + self.slide_within_blocks_u16x8::(a0, b0), + self.slide_within_blocks_u16x8::(a1, b1), ) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) - } - #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) + fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) } #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) + fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) } #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) + fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) } #[inline(always)] - fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) + fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) } #[inline(always)] - fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) + fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) } #[inline(always)] - fn not_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) + fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) } #[inline(always)] - fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) + fn not_u16x16(self, a: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) } #[inline(always)] - fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) + fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) } #[inline(always)] - fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) + fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) } #[inline(always)] - fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) + fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) } #[inline(always)] - fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) + fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) } #[inline(always)] - fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) + fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) } #[inline(always)] - fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) + fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) } #[inline(always)] - fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) + fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) } #[inline(always)] - fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) + fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) } #[inline(always)] - fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, _) = self.split_i32x8(a); - let (b0, _) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) + fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) } #[inline(always)] - fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (_, a1) = self.split_i32x8(a); - let (_, b1) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) + fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, _) = self.split_u16x16(a); + let (b0, _) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) } #[inline(always)] - fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) + fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (_, a1) = self.split_u16x16(a); + let (_, b1) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) } #[inline(always)] - fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) + fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) } #[inline(always)] - fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_lo = self.zip_low_i32x4(a0, b0); - let lo_hi = self.zip_high_i32x4(a0, b0); - let hi_lo = self.zip_low_i32x4(a1, b1); - let hi_hi = self.zip_high_i32x4(a1, b1); + fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) + } + #[inline(always)] + fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_lo = self.zip_low_u16x8(a0, b0); + let lo_hi = self.zip_high_u16x8(a0, b0); + let hi_lo = self.zip_low_u16x8(a1, b1); + let hi_hi = self.zip_high_u16x8(a1, b1); ( - self.combine_i32x4(lo_lo, lo_hi), - self.combine_i32x4(hi_lo, hi_hi), + self.combine_u16x8(lo_lo, lo_hi), + self.combine_u16x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_even = self.unzip_low_i32x4(a0, a1); - let lo_odd = self.unzip_high_i32x4(a0, a1); - let hi_even = self.unzip_low_i32x4(b0, b1); - let hi_odd = self.unzip_high_i32x4(b0, b1); + fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_even = self.unzip_low_u16x8(a0, a1); + let lo_odd = self.unzip_high_u16x8(a0, a1); + let hi_even = self.unzip_low_u16x8(b0, b1); + let hi_odd = self.unzip_high_u16x8(b0, b1); ( - self.combine_i32x4(lo_even, hi_even), - self.combine_i32x4(lo_odd, hi_odd), + self.combine_u16x8(lo_even, hi_even), + self.combine_u16x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_i32x8(b); - let (c0, c1) = self.split_i32x8(c); - self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) + fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_u16x16(b); + let (c0, c1) = self.split_u16x16(c); + self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) } #[inline(always)] - fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) + fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) } #[inline(always)] - fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) + fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) } #[inline(always)] - fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { - i32x16 { + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { ( - i32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) + fn narrow_u16x16(self, a: u16x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x16) -> u8x16 { + let (a, b) = token.split_u16x16(a); + let mask = _mm_set1_epi16(0xFF); + let lo_masked = _mm_and_si128(a.into(), mask); + let hi_masked = _mm_and_si128(b.into(), mask); + let result = _mm_packus_epi16(lo_masked, hi_masked); + result.simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] - fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { - let (a0, a1) = self.split_i32x8(a); - self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) + fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) } #[inline(always)] - fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { - let (a0, a1) = self.split_i32x8(a); + fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { + let (a0, a1) = self.split_u16x16(a); self.combine_u32x4( - self.reinterpret_u32_i32x4(a0), - self.reinterpret_u32_i32x4(a1), + self.reinterpret_u32_u16x8(a0), + self.reinterpret_u32_u16x8(a1), ) } #[inline(always)] - fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) - } - #[inline(always)] - fn splat_u32x8(self, val: u32) -> u32x8 { - let half = self.splat_u32x4(val); - self.combine_u32x4(half, half) + fn splat_mask16x16(self, val: bool) -> mask16x16 { + let half = self.splat_mask16x8(val); + self.combine_mask16x8(half, half) } #[inline(always)] - fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { + mask16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { - u32x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0) + fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { - crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0) + fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { + let lo = self.from_bitmask_mask16x8(bits); + let hi = self.from_bitmask_mask16x8(bits >> 8usize); + self.combine_mask16x8(lo, hi) } #[inline(always)] - fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { - crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u32; 8usize]>(&mut a.val.0) + fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask16x16) -> u64 { + { + let packed = _mm_packs_epi16(a.val.0[0], a.val.0[1]); + _mm_movemask_epi8(packed) as u32 as u64 + } + } + ); + kernel(self, a) } #[inline(always)] - fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask16x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x16(lanes); } #[inline(always)] - fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { - u32x8 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) } #[inline(always)] - fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - if SHIFT >= 8usize { - return b; - } - let result = cross_block_alignr_128x2( - self, - self.cvt_to_bytes_u32x8(b).val.0, - self.cvt_to_bytes_u32x8(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_u32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) } #[inline(always)] - fn slide_within_blocks_u32x8( + fn not_mask16x16(self, a: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) + } + #[inline(always)] + fn select_mask16x16( self, - a: u32x8, - b: u32x8, - ) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4( - self.slide_within_blocks_u32x4::(a0, b0), - self.slide_within_blocks_u32x4::(a1, b1), + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + let (c0, c1) = self.split_mask16x16(c); + self.combine_mask16x8( + self.select_mask16x8(a0, b0, c0), + self.select_mask16x8(a1, b1, c1), ) } #[inline(always)] - fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) + fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) } #[inline(always)] - fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) + fn any_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) } #[inline(always)] - fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) + fn all_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) } #[inline(always)] - fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) + fn any_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) } #[inline(always)] - fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) + fn all_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) } #[inline(always)] - fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) + fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { + mask16x32 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn not_u32x8(self, a: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) + fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + ( + mask16x8 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + mask16x8 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] - fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) + fn splat_i32x8(self, val: i32) -> i32x8 { + let half = self.splat_i32x4(val); + self.combine_i32x4(half, half) } #[inline(always)] - fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) + fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) + fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) + fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) + fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) + fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) + fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) + fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) + fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, _) = self.split_u32x8(a); - let (b0, _) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) + fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_alignr_128x2( + self, + self.cvt_to_bytes_i32x8(b).val.0, + self.cvt_to_bytes_i32x8(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (_, a1) = self.split_u32x8(a); - let (_, b1) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) + fn slide_within_blocks_i32x8( + self, + a: i32x8, + b: i32x8, + ) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4( + self.slide_within_blocks_i32x4::(a0, b0), + self.slide_within_blocks_i32x4::(a1, b1), + ) } #[inline(always)] - fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) } #[inline(always)] - fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) } #[inline(always)] - fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_lo = self.zip_low_u32x4(a0, b0); - let lo_hi = self.zip_high_u32x4(a0, b0); - let hi_lo = self.zip_low_u32x4(a1, b1); - let hi_hi = self.zip_high_u32x4(a1, b1); - ( - self.combine_u32x4(lo_lo, lo_hi), - self.combine_u32x4(hi_lo, hi_hi), - ) + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) } #[inline(always)] - fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_even = self.unzip_low_u32x4(a0, a1); - let lo_odd = self.unzip_high_u32x4(a0, a1); - let hi_even = self.unzip_low_u32x4(b0, b1); - let hi_odd = self.unzip_high_u32x4(b0, b1); - ( - self.combine_u32x4(lo_even, hi_even), - self.combine_u32x4(lo_odd, hi_odd), - ) + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) } #[inline(always)] - fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_u32x8(b); - let (c0, c1) = self.split_u32x8(c); - self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) } #[inline(always)] - fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) + fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) } #[inline(always)] - fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) + fn not_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) } #[inline(always)] - fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { - u32x16 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) } #[inline(always)] - fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { - ( - u32x4 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - u32x4 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) } #[inline(always)] - fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) + fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) } #[inline(always)] - fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) + fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) } #[inline(always)] - fn splat_mask32x8(self, val: bool) -> mask32x8 { - let half = self.splat_mask32x4(val); - self.combine_mask32x4(half, half) + fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) } #[inline(always)] - fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { - mask32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) } #[inline(always)] - fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) + fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) } #[inline(always)] - fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { - let lo = self.from_bitmask_mask32x4(bits); - let hi = self.from_bitmask_mask32x4(bits >> 4usize); - self.combine_mask32x4(lo, hi) + fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) } #[inline(always)] - fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { - let (lo, hi) = self.split_mask32x8(a); - let lo = self.to_bitmask_mask32x4(lo); - let hi = self.to_bitmask_mask32x4(hi); - lo | (hi << 4usize) + fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) } #[inline(always)] - fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) + fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, _) = self.split_i32x8(a); + let (b0, _) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) } #[inline(always)] - fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) + fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (_, a1) = self.split_i32x8(a); + let (_, b1) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) } #[inline(always)] - fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) + fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) } #[inline(always)] - fn not_mask32x8(self, a: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) + fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) } #[inline(always)] - fn select_mask32x8( - self, - a: mask32x8, - b: mask32x8, - c: mask32x8, - ) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - let (c0, c1) = self.split_mask32x8(c); - self.combine_mask32x4( - self.select_mask32x4(a0, b0, c0), - self.select_mask32x4(a1, b1, c1), + fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_lo = self.zip_low_i32x4(a0, b0); + let lo_hi = self.zip_high_i32x4(a0, b0); + let hi_lo = self.zip_low_i32x4(a1, b1); + let hi_hi = self.zip_high_i32x4(a1, b1); + ( + self.combine_i32x4(lo_lo, lo_hi), + self.combine_i32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) - } - #[inline(always)] - fn any_true_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) + fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_even = self.unzip_low_i32x4(a0, a1); + let lo_odd = self.unzip_high_i32x4(a0, a1); + let hi_even = self.unzip_low_i32x4(b0, b1); + let hi_odd = self.unzip_high_i32x4(b0, b1); + ( + self.combine_i32x4(lo_even, hi_even), + self.combine_i32x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn all_true_mask32x8(self, a: mask32x8) -> bool { + fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { let (a0, a1) = self.split_mask32x8(a); - self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) + let (b0, b1) = self.split_i32x8(b); + let (c0, c1) = self.split_i32x8(c); + self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) } #[inline(always)] - fn any_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) + fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) } #[inline(always)] - fn all_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) + fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) } #[inline(always)] - fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { - mask32x16 { + fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { + i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { + fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { ( - mask32x4 { + i32x4 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - mask32x4 { + i32x4 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_f64x4(self, val: f64) -> f64x4 { - let half = self.splat_f64x2(val); - self.combine_f64x2(half, half) + fn neg_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) } #[inline(always)] - fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { - f64x4 { + fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { + let (a0, a1) = self.split_i32x8(a); + self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_u32x4( + self.reinterpret_u32_i32x4(a0), + self.reinterpret_u32_i32x4(a1), + ) + } + #[inline(always)] + fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) + } + #[inline(always)] + fn splat_u32x8(self, val: u32) -> u32x8 { + let half = self.splat_u32x4(val); + self.combine_u32x4(half, half) + } + #[inline(always)] + fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { - f64x4 { + fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { - crate::transmute::checked_transmute_copy::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0) + fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { - crate::transmute::checked_cast_ref::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0) + fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { - crate::transmute::checked_cast_mut::<[__m128d; 2usize], [f64; 4usize]>(&mut a.val.0) + fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { + fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { - f64x4 { + fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - if SHIFT >= 4usize { + fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_f64x4(b).val.0, - self.cvt_to_bytes_f64x4(a).val.0, - SHIFT * 8usize, + self.cvt_to_bytes_u32x8(b).val.0, + self.cvt_to_bytes_u32x8(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_f64x4(u8x32 { + self.cvt_from_bytes_u32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x4( + fn slide_within_blocks_u32x8( self, - a: f64x4, - b: f64x4, - ) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.slide_within_blocks_f64x2::(a0, b0), - self.slide_within_blocks_f64x2::(a1, b1), - ) - } - #[inline(always)] - fn abs_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) + a: u32x8, + b: u32x8, + ) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4( + self.slide_within_blocks_u32x4::(a0, b0), + self.slide_within_blocks_u32x4::(a1, b1), + ) } #[inline(always)] - fn neg_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) + fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) } #[inline(always)] - fn sqrt_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) + fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) } #[inline(always)] - fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.approximate_recip_f64x2(a0), - self.approximate_recip_f64x2(a1), - ) + fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) } #[inline(always)] - fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) + fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) } #[inline(always)] - fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) + fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) } #[inline(always)] - fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) + fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) } #[inline(always)] - fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) + fn not_u32x8(self, a: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) } #[inline(always)] - fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) + fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) } #[inline(always)] - fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) + fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) + fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) } #[inline(always)] - fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) + fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) } #[inline(always)] - fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) + fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) } #[inline(always)] - fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) + fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, _) = self.split_f64x4(a); - let (b0, _) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) + fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) } #[inline(always)] - fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (_, a1) = self.split_f64x4(a); - let (_, b1) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) + fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) } #[inline(always)] - fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) + fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) + fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, _) = self.split_u32x8(a); + let (b0, _) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) } #[inline(always)] - fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_lo = self.zip_low_f64x2(a0, b0); - let lo_hi = self.zip_high_f64x2(a0, b0); - let hi_lo = self.zip_low_f64x2(a1, b1); - let hi_hi = self.zip_high_f64x2(a1, b1); + fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (_, a1) = self.split_u32x8(a); + let (_, b1) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) + } + #[inline(always)] + fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_lo = self.zip_low_u32x4(a0, b0); + let lo_hi = self.zip_high_u32x4(a0, b0); + let hi_lo = self.zip_low_u32x4(a1, b1); + let hi_hi = self.zip_high_u32x4(a1, b1); ( - self.combine_f64x2(lo_lo, lo_hi), - self.combine_f64x2(hi_lo, hi_hi), + self.combine_u32x4(lo_lo, lo_hi), + self.combine_u32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_even = self.unzip_low_f64x2(a0, a1); - let lo_odd = self.unzip_high_f64x2(a0, a1); - let hi_even = self.unzip_low_f64x2(b0, b1); - let hi_odd = self.unzip_high_f64x2(b0, b1); + fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_even = self.unzip_low_u32x4(a0, a1); + let lo_odd = self.unzip_high_u32x4(a0, a1); + let hi_even = self.unzip_low_u32x4(b0, b1); + let hi_odd = self.unzip_high_u32x4(b0, b1); ( - self.combine_f64x2(lo_even, hi_even), - self.combine_f64x2(lo_odd, hi_odd), + self.combine_u32x4(lo_even, hi_even), + self.combine_u32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) + fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_u32x8(b); + let (c0, c1) = self.split_u32x8(c); + self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) } #[inline(always)] - fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) + fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) } #[inline(always)] - fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.max_precise_f64x2(a0, b0), - self.max_precise_f64x2(a1, b1), - ) + fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.min_precise_f64x2(a0, b0), - self.min_precise_f64x2(a1, b1), - ) + fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { + u32x16 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_add_f64x2(a0, b0, c0), - self.mul_add_f64x2(a1, b1, c1), + fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { + ( + u32x4 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + u32x4 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, ) } #[inline(always)] - fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_sub_f64x2(a0, b0, c0), - self.mul_sub_f64x2(a1, b1, c1), - ) - } - #[inline(always)] - fn floor_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) - } - #[inline(always)] - fn ceil_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) - } - #[inline(always)] - fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.round_ties_even_f64x2(a0), - self.round_ties_even_f64x2(a1), - ) - } - #[inline(always)] - fn fract_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) + fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) } #[inline(always)] - fn trunc_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) + fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) } #[inline(always)] - fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) + fn splat_mask32x8(self, val: bool) -> mask32x8 { + let half = self.splat_mask32x4(val); + self.combine_mask32x4(half, half) } #[inline(always)] - fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { - f64x8 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { + mask32x8 { + val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { - ( - f64x2 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - f64x2 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f32x4( - self.reinterpret_f32_f64x2(a0), - self.reinterpret_f32_f64x2(a1), - ) - } - #[inline(always)] - fn splat_mask64x4(self, val: bool) -> mask64x4 { - let half = self.splat_mask64x2(val); - self.combine_mask64x2(half, half) - } - #[inline(always)] - fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { - mask64x4 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i64; 4usize]>(&a.val.0) + fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { + let lo = self.from_bitmask_mask32x4(bits); + let hi = self.from_bitmask_mask32x4(bits >> 4usize); + self.combine_mask32x4(lo, hi) } #[inline(always)] - fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { - let lo = self.from_bitmask_mask64x2(bits); - let hi = self.from_bitmask_mask64x2(bits >> 2usize); - self.combine_mask64x2(lo, hi) + fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { + let (lo, hi) = self.split_mask32x8(a); + let lo = self.to_bitmask_mask32x4(lo); + let hi = self.to_bitmask_mask32x4(hi); + lo | (hi << 4usize) } #[inline(always)] - fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { - let (lo, hi) = self.split_mask64x4(a); - let lo = self.to_bitmask_mask64x2(lo); - let hi = self.to_bitmask_mask64x2(hi); - lo | (hi << 2usize) + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask32x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x8(lanes); } #[inline(always)] - fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) + fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) } #[inline(always)] - fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) + fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) } #[inline(always)] - fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) + fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) } #[inline(always)] - fn not_mask64x4(self, a: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) + fn not_mask32x8(self, a: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) } #[inline(always)] - fn select_mask64x4( + fn select_mask32x8( self, - a: mask64x4, - b: mask64x4, - c: mask64x4, - ) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - let (c0, c1) = self.split_mask64x4(c); - self.combine_mask64x2( - self.select_mask64x2(a0, b0, c0), - self.select_mask64x2(a1, b1, c1), + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + let (c0, c1) = self.split_mask32x8(c); + self.combine_mask32x4( + self.select_mask32x4(a0, b0, c0), + self.select_mask32x4(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) + fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) } #[inline(always)] - fn any_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) + fn any_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) } #[inline(always)] - fn all_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) + fn all_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) } #[inline(always)] - fn any_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) + fn any_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) } #[inline(always)] - fn all_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + fn all_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) } #[inline(always)] - fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { - mask64x8 { + fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { + mask32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { ( - mask64x2 { + mask32x4 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - mask64x2 { + mask32x4 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_f32x16(self, val: f32) -> f32x16 { - let half = self.splat_f32x8(val); - self.combine_f32x8(half, half) + fn splat_f64x4(self, val: f64) -> f64x4 { + let half = self.splat_f64x2(val); + self.combine_f64x2(half, half) } #[inline(always)] - fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128; 4usize], [f32; 16usize]>(&a.val.0) + fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { + crate::transmute::checked_transmute_copy::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { - crate::transmute::checked_cast_ref::<[__m128; 4usize], [f32; 16usize]>(&a.val.0) + fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { + crate::transmute::checked_cast_ref::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { - crate::transmute::checked_cast_mut::<[__m128; 4usize], [f32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { + crate::transmute::checked_cast_mut::<[__m128d; 2usize], [f64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { - f32x16 { + fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { - u8x64 { + fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - if SHIFT >= 16usize { + fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_alignr_128x4( + let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_f32x16(b).val.0, - self.cvt_to_bytes_f32x16(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_f64x4(b).val.0, + self.cvt_to_bytes_f64x4(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_f32x16(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x16( + fn slide_within_blocks_f64x4( self, - a: f32x16, - b: f32x16, - ) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.slide_within_blocks_f32x8::(a0, b0), - self.slide_within_blocks_f32x8::(a1, b1), + a: f64x4, + b: f64x4, + ) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.slide_within_blocks_f64x2::(a0, b0), + self.slide_within_blocks_f64x2::(a1, b1), ) } #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + fn abs_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) } #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + fn neg_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) } #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) } #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.approximate_recip_f64x2(a0), + self.approximate_recip_f64x2(a1), ) } #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) } #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) } #[inline(always)] - fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) } #[inline(always)] - fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) } #[inline(always)] - fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) } #[inline(always)] - fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) } #[inline(always)] - fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) } #[inline(always)] - fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) } #[inline(always)] - fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) } #[inline(always)] - fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) } #[inline(always)] - fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, _) = self.split_f32x16(a); - let (b0, _) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, _) = self.split_f64x4(a); + let (b0, _) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) } #[inline(always)] - fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (_, a1) = self.split_f32x16(a); - let (_, b1) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) + fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (_, a1) = self.split_f64x4(a); + let (_, b1) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) } #[inline(always)] - fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) + fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) } #[inline(always)] - fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) + fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) } #[inline(always)] - fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_lo = self.zip_low_f32x8(a0, b0); - let lo_hi = self.zip_high_f32x8(a0, b0); - let hi_lo = self.zip_low_f32x8(a1, b1); - let hi_hi = self.zip_high_f32x8(a1, b1); + fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_lo = self.zip_low_f64x2(a0, b0); + let lo_hi = self.zip_high_f64x2(a0, b0); + let hi_lo = self.zip_low_f64x2(a1, b1); + let hi_hi = self.zip_high_f64x2(a1, b1); ( - self.combine_f32x8(lo_lo, lo_hi), - self.combine_f32x8(hi_lo, hi_hi), + self.combine_f64x2(lo_lo, lo_hi), + self.combine_f64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_even = self.unzip_low_f32x8(a0, a1); - let lo_odd = self.unzip_high_f32x8(a0, a1); - let hi_even = self.unzip_low_f32x8(b0, b1); - let hi_odd = self.unzip_high_f32x8(b0, b1); + fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_even = self.unzip_low_f64x2(a0, a1); + let lo_odd = self.unzip_high_f64x2(a0, a1); + let hi_even = self.unzip_low_f64x2(b0, b1); + let hi_odd = self.unzip_high_f64x2(b0, b1); ( - self.combine_f32x8(lo_even, hi_even), - self.combine_f32x8(lo_odd, hi_odd), + self.combine_f64x2(lo_even, hi_even), + self.combine_f64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) - } - #[inline(always)] - fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) + fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) } #[inline(always)] - fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.max_precise_f32x8(a0, b0), - self.max_precise_f32x8(a1, b1), - ) + fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) } #[inline(always)] - fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.min_precise_f32x8(a0, b0), - self.min_precise_f32x8(a1, b1), + fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.max_precise_f64x2(a0, b0), + self.max_precise_f64x2(a1, b1), ) } #[inline(always)] - fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_add_f32x8(a0, b0, c0), - self.mul_add_f32x8(a1, b1, c1), + fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.min_precise_f64x2(a0, b0), + self.min_precise_f64x2(a1, b1), ) } #[inline(always)] - fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_sub_f32x8(a0, b0, c0), - self.mul_sub_f32x8(a1, b1, c1), + fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_add_f64x2(a0, b0, c0), + self.mul_add_f64x2(a1, b1, c1), ) } #[inline(always)] - fn floor_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) - } - #[inline(always)] - fn ceil_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) - } - #[inline(always)] - fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.round_ties_even_f32x8(a0), - self.round_ties_even_f32x8(a1), + fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_sub_f64x2(a0, b0, c0), + self.mul_sub_f64x2(a1, b1, c1), ) } #[inline(always)] - fn fract_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) - } - #[inline(always)] - fn trunc_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) - } - #[inline(always)] - fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) - } - #[inline(always)] - fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { - ( - f32x8 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - f32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) + fn floor_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) } #[inline(always)] - fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f64x4( - self.reinterpret_f64_f32x8(a0), - self.reinterpret_f64_f32x8(a1), - ) + fn ceil_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) } #[inline(always)] - fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.reinterpret_i32_f32x8(a0), - self.reinterpret_i32_f32x8(a1), + fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.round_ties_even_f64x2(a0), + self.round_ties_even_f64x2(a1), ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + fn fract_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) } #[inline(always)] - fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) + fn trunc_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) } #[inline(always)] - fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.reinterpret_u32_f32x8(a0), - self.reinterpret_u32_f32x8(a1), - ) + fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) } #[inline(always)] - fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) + fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { + f64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.cvt_u32_precise_f32x8(a0), - self.cvt_u32_precise_f32x8(a1), + fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { + ( + f64x2 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + f64x2 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, ) } #[inline(always)] - fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) - } - #[inline(always)] - fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.cvt_i32_precise_f32x8(a0), - self.cvt_i32_precise_f32x8(a1), + fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f32x4( + self.reinterpret_f32_f64x2(a0), + self.reinterpret_f32_f64x2(a1), ) } #[inline(always)] - fn splat_i8x64(self, val: i8) -> i8x64 { - let half = self.splat_i8x32(val); - self.combine_i8x32(half, half) + fn splat_i64x4(self, val: i64) -> i64x4 { + let half = self.splat_i64x2(val); + self.combine_i64x2(half, half) } #[inline(always)] - fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i8; 64usize]>(&mut a.val.0) + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { - i8x64 { + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { - u8x64 { + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - if SHIFT >= 64usize { + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_alignr_128x4( + let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_i8x64(b).val.0, - self.cvt_to_bytes_i8x64(a).val.0, - SHIFT, + self.cvt_to_bytes_i64x4(b).val.0, + self.cvt_to_bytes_i64x4(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_i8x64(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x64( + fn slide_within_blocks_i64x4( self, - a: i8x64, - b: i8x64, - ) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32( - self.slide_within_blocks_i8x32::(a0, b0), - self.slide_within_blocks_i8x32::(a1, b1), + a: i64x4, + b: i64x4, + ) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2( + self.slide_within_blocks_i64x2::(a0, b0), + self.slide_within_blocks_i64x2::(a1, b1), ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) } #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) } #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) } #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) } #[inline(always)] - fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) } #[inline(always)] - fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) } #[inline(always)] - fn not_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) } #[inline(always)] - fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) } #[inline(always)] - fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) } #[inline(always)] - fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) } #[inline(always)] - fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) } #[inline(always)] - fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) } #[inline(always)] - fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) } #[inline(always)] - fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) } #[inline(always)] - fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) } #[inline(always)] - fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) } #[inline(always)] - fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, _) = self.split_i8x64(a); - let (b0, _) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) } #[inline(always)] - fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (_, a1) = self.split_i8x64(a); - let (_, b1) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) } #[inline(always)] - fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) } #[inline(always)] - fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) } #[inline(always)] - fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_lo = self.zip_low_i8x32(a0, b0); - let lo_hi = self.zip_high_i8x32(a0, b0); - let hi_lo = self.zip_low_i8x32(a1, b1); - let hi_hi = self.zip_high_i8x32(a1, b1); + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_lo = self.zip_low_i64x2(a0, b0); + let lo_hi = self.zip_high_i64x2(a0, b0); + let hi_lo = self.zip_low_i64x2(a1, b1); + let hi_hi = self.zip_high_i64x2(a1, b1); ( - self.combine_i8x32(lo_lo, lo_hi), - self.combine_i8x32(hi_lo, hi_hi), + self.combine_i64x2(lo_lo, lo_hi), + self.combine_i64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_even = self.unzip_low_i8x32(a0, a1); - let lo_odd = self.unzip_high_i8x32(a0, a1); - let hi_even = self.unzip_low_i8x32(b0, b1); - let hi_odd = self.unzip_high_i8x32(b0, b1); + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_even = self.unzip_low_i64x2(a0, a1); + let lo_odd = self.unzip_high_i64x2(a0, a1); + let hi_even = self.unzip_low_i64x2(b0, b1); + let hi_odd = self.unzip_high_i64x2(b0, b1); ( - self.combine_i8x32(lo_even, hi_even), - self.combine_i8x32(lo_odd, hi_odd), + self.combine_i64x2(lo_even, hi_even), + self.combine_i64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_i8x64(b); - let (c0, c1) = self.split_i8x64(c); - self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_i64x4(b); + let (c0, c1) = self.split_i64x4(c); + self.combine_i64x2(self.select_i64x2(a0, b0, c0), self.select_i64x2(a1, b1, c1)) } #[inline(always)] - fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.min_i64x2(a0, b0), self.min_i64x2(a1, b1)) } #[inline(always)] - fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.max_i64x2(a0, b0), self.max_i64x2(a1, b1)) } #[inline(always)] - fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + i64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { ( - i8x32 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + i64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i8x32 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + i64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) + fn neg_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] - fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u8x16(self.reinterpret_u8_i64x2(a0), self.reinterpret_u8_i64x2(a1)) } #[inline(always)] - fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { - let (a0, a1) = self.split_i8x64(a); - self.combine_u32x8( - self.reinterpret_u32_i8x32(a0), - self.reinterpret_u32_i8x32(a1), + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u32x4( + self.reinterpret_u32_i64x2(a0), + self.reinterpret_u32_i64x2(a1), ) } #[inline(always)] - fn splat_u8x64(self, val: u8) -> u8x64 { - let half = self.splat_u8x32(val); - self.combine_u8x32(half, half) + fn splat_u64x4(self, val: u64) -> u64x4 { + let half = self.splat_u64x2(val); + self.combine_u64x2(half, half) } #[inline(always)] - fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0) + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0) + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u8; 64usize]>(&mut a.val.0) + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { - u8x64 { + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { - u8x64 { + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - if SHIFT >= 64usize { + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_alignr_128x4( + let result = cross_block_alignr_128x2( self, - self.cvt_to_bytes_u8x64(b).val.0, - self.cvt_to_bytes_u8x64(a).val.0, - SHIFT, + self.cvt_to_bytes_u64x4(b).val.0, + self.cvt_to_bytes_u64x4(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_u8x64(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x64( + fn slide_within_blocks_u64x4( self, - a: u8x64, - b: u8x64, - ) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32( - self.slide_within_blocks_u8x32::(a0, b0), - self.slide_within_blocks_u8x32::(a1, b1), + a: u64x4, + b: u64x4, + ) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2( + self.slide_within_blocks_u64x2::(a0, b0), + self.slide_within_blocks_u64x2::(a1, b1), ) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) } #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) } #[inline(always)] - fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1)) } #[inline(always)] - fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.and_u64x2(a0, b0), self.and_u64x2(a1, b1)) } #[inline(always)] - fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.or_u64x2(a0, b0), self.or_u64x2(a1, b1)) } #[inline(always)] - fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.xor_u64x2(a0, b0), self.xor_u64x2(a1, b1)) } #[inline(always)] - fn not_u8x64(self, a: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + fn not_u64x4(self, a: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.not_u64x2(a0), self.not_u64x2(a1)) } #[inline(always)] - fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shl_u64x2(a0, shift), self.shl_u64x2(a1, shift)) } #[inline(always)] - fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shlv_u64x2(a0, b0), self.shlv_u64x2(a1, b1)) } #[inline(always)] - fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shr_u64x2(a0, shift), self.shr_u64x2(a1, shift)) } #[inline(always)] - fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shrv_u64x2(a0, b0), self.shrv_u64x2(a1, b1)) } #[inline(always)] - fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_eq_u64x2(a0, b0), self.simd_eq_u64x2(a1, b1)) } #[inline(always)] - fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_lt_u64x2(a0, b0), self.simd_lt_u64x2(a1, b1)) } #[inline(always)] - fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_le_u64x2(a0, b0), self.simd_le_u64x2(a1, b1)) } #[inline(always)] - fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_ge_u64x2(a0, b0), self.simd_ge_u64x2(a1, b1)) } #[inline(always)] - fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_gt_u64x2(a0, b0), self.simd_gt_u64x2(a1, b1)) } #[inline(always)] - fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, _) = self.split_u8x64(a); - let (b0, _) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, _) = self.split_u64x4(a); + let (b0, _) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a0, b0), self.zip_high_u64x2(a0, b0)) } #[inline(always)] - fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (_, a1) = self.split_u8x64(a); - let (_, b1) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (_, a1) = self.split_u64x4(a); + let (_, b1) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a1, b1), self.zip_high_u64x2(a1, b1)) } #[inline(always)] - fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_low_u64x2(a0, a1), self.unzip_low_u64x2(b0, b1)) } #[inline(always)] - fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_high_u64x2(a0, a1), self.unzip_high_u64x2(b0, b1)) } #[inline(always)] - fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_lo = self.zip_low_u8x32(a0, b0); - let lo_hi = self.zip_high_u8x32(a0, b0); - let hi_lo = self.zip_low_u8x32(a1, b1); - let hi_hi = self.zip_high_u8x32(a1, b1); + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_lo = self.zip_low_u64x2(a0, b0); + let lo_hi = self.zip_high_u64x2(a0, b0); + let hi_lo = self.zip_low_u64x2(a1, b1); + let hi_hi = self.zip_high_u64x2(a1, b1); ( - self.combine_u8x32(lo_lo, lo_hi), - self.combine_u8x32(hi_lo, hi_hi), + self.combine_u64x2(lo_lo, lo_hi), + self.combine_u64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_even = self.unzip_low_u8x32(a0, a1); - let lo_odd = self.unzip_high_u8x32(a0, a1); - let hi_even = self.unzip_low_u8x32(b0, b1); - let hi_odd = self.unzip_high_u8x32(b0, b1); + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_even = self.unzip_low_u64x2(a0, a1); + let lo_odd = self.unzip_high_u64x2(a0, a1); + let hi_even = self.unzip_low_u64x2(b0, b1); + let hi_odd = self.unzip_high_u64x2(b0, b1); ( - self.combine_u8x32(lo_even, hi_even), - self.combine_u8x32(lo_odd, hi_odd), + self.combine_u64x2(lo_even, hi_even), + self.combine_u64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_u8x64(b); - let (c0, c1) = self.split_u8x64(c); - self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_u64x4(b); + let (c0, c1) = self.split_u64x4(c); + self.combine_u64x2(self.select_u64x2(a0, b0, c0), self.select_u64x2(a1, b1, c1)) } #[inline(always)] - fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.min_u64x2(a0, b0), self.min_u64x2(a1, b1)) } #[inline(always)] - fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.max_u64x2(a0, b0), self.max_u64x2(a1, b1)) } #[inline(always)] - fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + u64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { ( - u8x32 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + u64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u8x32 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + u64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u8x32( - token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (v01, v23) = token.split_u8x64(a); - let (v0, v1) = token.split_u8x32(v01); - let (v2, v3) = token.split_u8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u8x16(self.reinterpret_u8_u64x2(a0), self.reinterpret_u8_u64x2(a1)) } #[inline(always)] - fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u32x8( - self.reinterpret_u32_u8x32(a0), - self.reinterpret_u32_u8x32(a1), + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u32x4( + self.reinterpret_u32_u64x2(a0), + self.reinterpret_u32_u64x2(a1), ) } #[inline(always)] - fn splat_mask8x64(self, val: bool) -> mask8x64 { - let half = self.splat_mask8x32(val); - self.combine_mask8x32(half, half) + fn splat_mask64x4(self, val: bool) -> mask64x4 { + let half = self.splat_mask64x2(val); + self.combine_mask64x2(half, half) } #[inline(always)] - fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { - mask8x64 { + fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { + mask64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i64; 4usize]>(&a.val.0) } #[inline(always)] - fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, bits: u64) -> mask8x64 { - { - let bit_bytes = _mm_set1_epi64x(bits.cast_signed()); - let bit_mask = - _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128); - mask8x64 { - val: crate::support::Aligned512([ - { - let bit_bytes = _mm_shuffle_epi8( - bit_bytes, - _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1), - ); - _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) - }, - { - let bit_bytes = _mm_shuffle_epi8( - bit_bytes, - _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3), - ); - _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) - }, - { - let bit_bytes = _mm_shuffle_epi8( - bit_bytes, - _mm_setr_epi8(4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5), - ); - _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) - }, - { - let bit_bytes = _mm_shuffle_epi8( - bit_bytes, - _mm_setr_epi8(6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7), - ); - _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) - }, - ]), - simd: token, - } - } - } - ); - kernel(self, bits) + fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { + let lo = self.from_bitmask_mask64x2(bits); + let hi = self.from_bitmask_mask64x2(bits >> 2usize); + self.combine_mask64x2(lo, hi) } #[inline(always)] - fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { - let (lo, hi) = self.split_mask8x64(a); - let lo = self.to_bitmask_mask8x32(lo); - let hi = self.to_bitmask_mask8x32(hi); - lo | (hi << 32usize) + fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + let (lo, hi) = self.split_mask64x4(a); + let lo = self.to_bitmask_mask64x2(lo); + let hi = self.to_bitmask_mask64x2(hi); + lo | (hi << 2usize) } #[inline(always)] - fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask64x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x4(lanes); } #[inline(always)] - fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) + fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) } #[inline(always)] - fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) + fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) } #[inline(always)] - fn not_mask8x64(self, a: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) + fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) } #[inline(always)] - fn select_mask8x64( - self, - a: mask8x64, - b: mask8x64, - c: mask8x64, - ) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - let (c0, c1) = self.split_mask8x64(c); - self.combine_mask8x32( - self.select_mask8x32(a0, b0, c0), - self.select_mask8x32(a1, b1, c1), - ) + fn not_mask64x4(self, a: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) } #[inline(always)] - fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) + fn select_mask64x4( + self, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + let (c0, c1) = self.split_mask64x4(c); + self.combine_mask64x2( + self.select_mask64x2(a0, b0, c0), + self.select_mask64x2(a1, b1, c1), + ) } #[inline(always)] - fn any_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) + fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) } #[inline(always)] - fn all_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) + fn any_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) } #[inline(always)] - fn any_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) + fn all_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) } #[inline(always)] - fn all_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) + fn any_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) } #[inline(always)] - fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + fn all_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + } + #[inline(always)] + fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { + mask64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { ( - mask8x32 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + mask64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - mask8x32 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + mask64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_i16x32(self, val: i16) -> i16x32 { - let half = self.splat_i16x16(val); - self.combine_i16x16(half, half) + fn splat_f32x16(self, val: f32) -> f32x16 { + let half = self.splat_f32x8(val); + self.combine_f32x8(half, half) } #[inline(always)] - fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { - i16x32 { + fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { - i16x32 { + fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128; 4usize], [f32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { + crate::transmute::checked_cast_ref::<[__m128; 4usize], [f32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i16; 32usize]>(&mut a.val.0) + fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { + crate::transmute::checked_cast_mut::<[__m128; 4usize], [f32; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { - i16x32 { + fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { + f32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - if SHIFT >= 32usize { + fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + if SHIFT >= 16usize { return b; } let result = cross_block_alignr_128x4( self, - self.cvt_to_bytes_i16x32(b).val.0, - self.cvt_to_bytes_i16x32(a).val.0, - SHIFT * 2usize, + self.cvt_to_bytes_f32x16(b).val.0, + self.cvt_to_bytes_f32x16(a).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i16x32(u8x64 { + self.cvt_from_bytes_f32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i16x32( + fn slide_within_blocks_f32x16( self, - a: i16x32, - b: i16x32, - ) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.slide_within_blocks_i16x16::(a0, b0), - self.slide_within_blocks_i16x16::(a1, b1), + a: f32x16, + b: f32x16, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.slide_within_blocks_f32x8::(a0, b0), + self.slide_within_blocks_f32x8::(a1, b1), ) } #[inline(always)] - fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) - } - #[inline(always)] - fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) } #[inline(always)] - fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) } #[inline(always)] - fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) } #[inline(always)] - fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) } #[inline(always)] - fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) } #[inline(always)] - fn not_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) } #[inline(always)] - fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) } #[inline(always)] - fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) } #[inline(always)] - fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) } #[inline(always)] - fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) } #[inline(always)] - fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) } #[inline(always)] - fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) } #[inline(always)] - fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) } #[inline(always)] - fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) } #[inline(always)] - fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, _) = self.split_f32x16(a); + let (b0, _) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) } #[inline(always)] - fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, _) = self.split_i16x32(a); - let (b0, _) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) + fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (_, a1) = self.split_f32x16(a); + let (_, b1) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) } #[inline(always)] - fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (_, a1) = self.split_i16x32(a); - let (_, b1) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) + fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) } #[inline(always)] - fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) + fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) } #[inline(always)] - fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.unzip_high_i16x16(a0, a1), - self.unzip_high_i16x16(b0, b1), - ) - } - #[inline(always)] - fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_lo = self.zip_low_i16x16(a0, b0); - let lo_hi = self.zip_high_i16x16(a0, b0); - let hi_lo = self.zip_low_i16x16(a1, b1); - let hi_hi = self.zip_high_i16x16(a1, b1); + fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_lo = self.zip_low_f32x8(a0, b0); + let lo_hi = self.zip_high_f32x8(a0, b0); + let hi_lo = self.zip_low_f32x8(a1, b1); + let hi_hi = self.zip_high_f32x8(a1, b1); ( - self.combine_i16x16(lo_lo, lo_hi), - self.combine_i16x16(hi_lo, hi_hi), + self.combine_f32x8(lo_lo, lo_hi), + self.combine_f32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_even = self.unzip_low_i16x16(a0, a1); - let lo_odd = self.unzip_high_i16x16(a0, a1); - let hi_even = self.unzip_low_i16x16(b0, b1); - let hi_odd = self.unzip_high_i16x16(b0, b1); + fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_even = self.unzip_low_f32x8(a0, a1); + let lo_odd = self.unzip_high_f32x8(a0, a1); + let hi_even = self.unzip_low_f32x8(b0, b1); + let hi_odd = self.unzip_high_f32x8(b0, b1); ( - self.combine_i16x16(lo_even, hi_even), - self.combine_i16x16(lo_odd, hi_odd), - ) - } - #[inline(always)] - fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_i16x32(b); - let (c0, c1) = self.split_i16x32(c); - self.combine_i16x16( - self.select_i16x16(a0, b0, c0), - self.select_i16x16(a1, b1, c1), + self.combine_f32x8(lo_even, hi_even), + self.combine_f32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) + fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) } #[inline(always)] - fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) + fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) } #[inline(always)] - fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { - ( - i16x16 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - i16x16 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, + fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.max_precise_f32x8(a0, b0), + self.max_precise_f32x8(a1, b1), ) } #[inline(always)] - fn neg_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) - } - #[inline(always)] - fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { - let (a0, a1) = self.split_i16x32(a); - self.combine_u8x32( - self.reinterpret_u8_i16x16(a0), - self.reinterpret_u8_i16x16(a1), + fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.min_precise_f32x8(a0, b0), + self.min_precise_f32x8(a1, b1), ) } #[inline(always)] - fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { - let (a0, a1) = self.split_i16x32(a); - self.combine_u32x8( - self.reinterpret_u32_i16x16(a0), - self.reinterpret_u32_i16x16(a1), + fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_add_f32x8(a0, b0, c0), + self.mul_add_f32x8(a1, b1, c1), ) } #[inline(always)] - fn splat_u16x32(self, val: u16) -> u16x32 { - let half = self.splat_u16x16(val); - self.combine_u16x16(half, half) - } - #[inline(always)] - fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { - u16x32 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_sub_f32x8(a0, b0, c0), + self.mul_sub_f32x8(a1, b1, c1), + ) } #[inline(always)] - fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { - u16x32 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn floor_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) } #[inline(always)] - fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0) + fn ceil_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) } #[inline(always)] - fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0) + fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.round_ties_even_f32x8(a0), + self.round_ties_even_f32x8(a1), + ) } #[inline(always)] - fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u16; 32usize]>(&mut a.val.0) + fn fract_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) } #[inline(always)] - fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn trunc_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) } #[inline(always)] - fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { - u16x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) } #[inline(always)] - fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { - u8x64 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + ( + f32x8 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + f32x8 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] - fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - if SHIFT >= 32usize { - return b; - } - let result = cross_block_alignr_128x4( - self, - self.cvt_to_bytes_u16x32(b).val.0, - self.cvt_to_bytes_u16x32(a).val.0, - SHIFT * 2usize, - ); - self.cvt_from_bytes_u16x32(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) + fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f64x4( + self.reinterpret_f64_f32x8(a0), + self.reinterpret_f64_f32x8(a1), + ) } #[inline(always)] - fn slide_within_blocks_u16x32( - self, - a: u16x32, - b: u16x32, - ) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.slide_within_blocks_u16x16::(a0, b0), - self.slide_within_blocks_u16x16::(a1, b1), + fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.reinterpret_i32_f32x8(a0), + self.reinterpret_i32_f32x8(a1), ) } #[inline(always)] - fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) + fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> f32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + token.combine_f32x8( + token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) } #[inline(always)] - fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) - } - #[inline(always)] - fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) + fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x16, dest: &mut [f32; 16usize]) -> () { + let (v01, v23) = token.split_f32x16(a); + let (v0, v1) = token.split_f32x8(v01); + let (v2, v3) = token.split_f32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); } #[inline(always)] - fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) + fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) } #[inline(always)] - fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) + fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.reinterpret_u32_f32x8(a0), + self.reinterpret_u32_f32x8(a1), + ) } #[inline(always)] - fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) + fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) } #[inline(always)] - fn not_u16x32(self, a: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) + fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.cvt_u32_precise_f32x8(a0), + self.cvt_u32_precise_f32x8(a1), + ) } #[inline(always)] - fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) + fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) } #[inline(always)] - fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) + fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.cvt_i32_precise_f32x8(a0), + self.cvt_i32_precise_f32x8(a1), + ) } #[inline(always)] - fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) + fn splat_i8x64(self, val: i8) -> i8x64 { + let half = self.splat_i8x32(val); + self.combine_i8x32(half, half) } #[inline(always)] - fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) + fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) + fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) + fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) } #[inline(always)] - fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) + fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) } #[inline(always)] - fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) + fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i8; 64usize]>(&mut a.val.0) } #[inline(always)] - fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) + fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, _) = self.split_u16x32(a); - let (b0, _) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) + fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (_, a1) = self.split_u16x32(a); - let (_, b1) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) + fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) + fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + if SHIFT >= 64usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_i8x64(b).val.0, + self.cvt_to_bytes_i8x64(a).val.0, + SHIFT, + ); + self.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.unzip_high_u16x16(a0, a1), - self.unzip_high_u16x16(b0, b1), + fn slide_within_blocks_i8x64( + self, + a: i8x64, + b: i8x64, + ) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32( + self.slide_within_blocks_i8x32::(a0, b0), + self.slide_within_blocks_i8x32::(a1, b1), ) } #[inline(always)] - fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_lo = self.zip_low_u16x16(a0, b0); - let lo_hi = self.zip_high_u16x16(a0, b0); - let hi_lo = self.zip_low_u16x16(a1, b1); - let hi_hi = self.zip_high_u16x16(a1, b1); - ( - self.combine_u16x16(lo_lo, lo_hi), - self.combine_u16x16(hi_lo, hi_hi), - ) + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) } #[inline(always)] - fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_even = self.unzip_low_u16x16(a0, a1); - let lo_odd = self.unzip_high_u16x16(a0, a1); - let hi_even = self.unzip_low_u16x16(b0, b1); - let hi_odd = self.unzip_high_u16x16(b0, b1); - ( - self.combine_u16x16(lo_even, hi_even), - self.combine_u16x16(lo_odd, hi_odd), - ) + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) } #[inline(always)] - fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_u16x32(b); - let (c0, c1) = self.split_u16x32(c); - self.combine_u16x16( - self.select_u16x16(a0, b0, c0), - self.select_u16x16(a1, b1, c1), - ) + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) } #[inline(always)] - fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) } #[inline(always)] - fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) } #[inline(always)] - fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + } + #[inline(always)] + fn not_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + } + #[inline(always)] + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + } + #[inline(always)] + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + } + #[inline(always)] + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + } + #[inline(always)] + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + } + #[inline(always)] + fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, _) = self.split_i8x64(a); + let (b0, _) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) + } + #[inline(always)] + fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (_, a1) = self.split_i8x64(a); + let (_, b1) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) + } + #[inline(always)] + fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) + } + #[inline(always)] + fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) + } + #[inline(always)] + fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_lo = self.zip_low_i8x32(a0, b0); + let lo_hi = self.zip_high_i8x32(a0, b0); + let hi_lo = self.zip_low_i8x32(a1, b1); + let hi_hi = self.zip_high_i8x32(a1, b1); ( - u16x16 { + self.combine_i8x32(lo_lo, lo_hi), + self.combine_i8x32(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_even = self.unzip_low_i8x32(a0, a1); + let lo_odd = self.unzip_high_i8x32(a0, a1); + let hi_even = self.unzip_low_i8x32(b0, b1); + let hi_odd = self.unzip_high_i8x32(b0, b1); + ( + self.combine_i8x32(lo_even, hi_even), + self.combine_i8x32(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_i8x64(b); + let (c0, c1) = self.split_i8x64(c); + self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) + } + #[inline(always)] + fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + } + #[inline(always)] + fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + } + #[inline(always)] + fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + ( + i8x32 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - u16x16 { + i8x32 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u16x16( - token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } + fn neg_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) + } + #[inline(always)] + fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) + } + #[inline(always)] + fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u32x8( + self.reinterpret_u32_i8x32(a0), + self.reinterpret_u32_i8x32(a1), + ) + } + #[inline(always)] + fn splat_u8x64(self, val: u8) -> u8x64 { + let half = self.splat_u8x32(val); + self.combine_u8x32(half, half) + } + #[inline(always)] + fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u8; 64usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + if SHIFT >= 64usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_u8x64(b).val.0, + self.cvt_to_bytes_u8x64(a).val.0, + SHIFT, ); - kernel(self, src) + self.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (v01, v23) = token.split_u16x32(a); - let (v0, v1) = token.split_u16x16(v01); - let (v2, v3) = token.split_u16x16(v23); + fn slide_within_blocks_u8x64( + self, + a: u8x64, + b: u8x64, + ) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32( + self.slide_within_blocks_u8x32::(a0, b0), + self.slide_within_blocks_u8x32::(a1, b1), + ) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + } + #[inline(always)] + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + } + #[inline(always)] + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + } + #[inline(always)] + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + } + #[inline(always)] + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + } + #[inline(always)] + fn not_u8x64(self, a: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + } + #[inline(always)] + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + } + #[inline(always)] + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + } + #[inline(always)] + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + } + #[inline(always)] + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) + } + #[inline(always)] + fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, _) = self.split_u8x64(a); + let (b0, _) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) + } + #[inline(always)] + fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (_, a1) = self.split_u8x64(a); + let (_, b1) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) + } + #[inline(always)] + fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + } + #[inline(always)] + fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + } + #[inline(always)] + fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_lo = self.zip_low_u8x32(a0, b0); + let lo_hi = self.zip_high_u8x32(a0, b0); + let hi_lo = self.zip_low_u8x32(a1, b1); + let hi_hi = self.zip_high_u8x32(a1, b1); + ( + self.combine_u8x32(lo_lo, lo_hi), + self.combine_u8x32(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_even = self.unzip_low_u8x32(a0, a1); + let lo_odd = self.unzip_high_u8x32(a0, a1); + let hi_even = self.unzip_low_u8x32(b0, b1); + let hi_odd = self.unzip_high_u8x32(b0, b1); + ( + self.combine_u8x32(lo_even, hi_even), + self.combine_u8x32(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_u8x64(b); + let (c0, c1) = self.split_u8x64(c); + self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) + } + #[inline(always)] + fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) + } + #[inline(always)] + fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) + } + #[inline(always)] + fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + ( + u8x32 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + u8x32 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> u8x64 { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_u8x32( + token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), + token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x64, dest: &mut [u8; 64usize]) -> () { + let (v01, v23) = token.split_u8x64(a); + let (v0, v1) = token.split_u8x32(v01); + let (v2, v3) = token.split_u8x32(v23); let v0 = v0.into(); let v1 = v1.into(); let v2 = v2.into(); @@ -8405,27 +9242,27 @@ impl Simd for Sse4_2 { let out1 = _mm_unpackhi_epi64(tmp0, tmp2); let out2 = _mm_unpacklo_epi64(tmp1, tmp3); let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); let out0 = _mm_shuffle_epi8(out0, mask); let out1 = _mm_shuffle_epi8(out1, mask); let out2 = _mm_shuffle_epi8(out2, mask); let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { unreachable!() }; - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( out0, &mut chunks[0], ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( out1, &mut chunks[1], ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( out2, &mut chunks[2], ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( out3, &mut chunks[3], ); @@ -8434,1179 +9271,2595 @@ impl Simd for Sse4_2 { kernel(self, a, dest); } #[inline(always)] - fn narrow_u16x32(self, a: u16x32) -> u8x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u32x8( + self.reinterpret_u32_u8x32(a0), + self.reinterpret_u32_u8x32(a1), + ) + } + #[inline(always)] + fn splat_mask8x64(self, val: bool) -> mask8x64 { + let half = self.splat_mask8x32(val); + self.combine_mask8x32(half, half) + } + #[inline(always)] + fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { + mask8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, bits: u64) -> mask8x64 { + { + let bit_bytes = _mm_set1_epi64x(bits.cast_signed()); + let bit_mask = + _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128); + mask8x64 { + val: crate::support::Aligned512([ + { + let bit_bytes = _mm_shuffle_epi8( + bit_bytes, + _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1), + ); + _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) + }, + { + let bit_bytes = _mm_shuffle_epi8( + bit_bytes, + _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3), + ); + _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) + }, + { + let bit_bytes = _mm_shuffle_epi8( + bit_bytes, + _mm_setr_epi8(4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5), + ); + _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) + }, + { + let bit_bytes = _mm_shuffle_epi8( + bit_bytes, + _mm_setr_epi8(6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7), + ); + _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask) + }, + ]), + simd: token, + } + } + } + ); + kernel(self, bits) + } + #[inline(always)] + fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { + let (lo, hi) = self.split_mask8x64(a); + let lo = self.to_bitmask_mask8x32(lo); + let hi = self.to_bitmask_mask8x32(hi); + lo | (hi << 32usize) + } + #[inline(always)] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let mut lanes = self.as_array_mask8x64(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x64(lanes); + } + #[inline(always)] + fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) + } + #[inline(always)] + fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) + } + #[inline(always)] + fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) + } + #[inline(always)] + fn not_mask8x64(self, a: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) + } + #[inline(always)] + fn select_mask8x64( + self, + a: mask8x64, + b: mask8x64, + c: mask8x64, + ) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + let (c0, c1) = self.split_mask8x64(c); + self.combine_mask8x32( + self.select_mask8x32(a0, b0, c0), + self.select_mask8x32(a1, b1, c1), + ) + } + #[inline(always)] + fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) + } + #[inline(always)] + fn any_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) + } + #[inline(always)] + fn all_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) + } + #[inline(always)] + fn any_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) + } + #[inline(always)] + fn all_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) + } + #[inline(always)] + fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + ( + mask8x32 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + mask8x32 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i16x32(self, val: i16) -> i16x32 { + let half = self.splat_i16x16(val); + self.combine_i16x16(half, half) + } + #[inline(always)] + fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_i16x32(b).val.0, + self.cvt_to_bytes_i16x32(a).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i16x32( + self, + a: i16x32, + b: i16x32, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.slide_within_blocks_i16x16::(a0, b0), + self.slide_within_blocks_i16x16::(a1, b1), + ) + } + #[inline(always)] + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + } + #[inline(always)] + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + } + #[inline(always)] + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + } + #[inline(always)] + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) + } + #[inline(always)] + fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) + } + #[inline(always)] + fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) + } + #[inline(always)] + fn not_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) + } + #[inline(always)] + fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) + } + #[inline(always)] + fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) + } + #[inline(always)] + fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) + } + #[inline(always)] + fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) + } + #[inline(always)] + fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, _) = self.split_i16x32(a); + let (b0, _) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) + } + #[inline(always)] + fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (_, a1) = self.split_i16x32(a); + let (_, b1) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.unzip_high_i16x16(a0, a1), + self.unzip_high_i16x16(b0, b1), + ) + } + #[inline(always)] + fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_lo = self.zip_low_i16x16(a0, b0); + let lo_hi = self.zip_high_i16x16(a0, b0); + let hi_lo = self.zip_low_i16x16(a1, b1); + let hi_hi = self.zip_high_i16x16(a1, b1); + ( + self.combine_i16x16(lo_lo, lo_hi), + self.combine_i16x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_even = self.unzip_low_i16x16(a0, a1); + let lo_odd = self.unzip_high_i16x16(a0, a1); + let hi_even = self.unzip_low_i16x16(b0, b1); + let hi_odd = self.unzip_high_i16x16(b0, b1); + ( + self.combine_i16x16(lo_even, hi_even), + self.combine_i16x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_i16x32(b); + let (c0, c1) = self.split_i16x32(c); + self.combine_i16x16( + self.select_i16x16(a0, b0, c0), + self.select_i16x16(a1, b1, c1), + ) + } + #[inline(always)] + fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) + } + #[inline(always)] + fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) + } + #[inline(always)] + fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { + ( + i16x16 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i16x16 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { + let (a0, a1) = self.split_i16x32(a); + self.combine_u8x32( + self.reinterpret_u8_i16x16(a0), + self.reinterpret_u8_i16x16(a1), + ) + } + #[inline(always)] + fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { + let (a0, a1) = self.split_i16x32(a); + self.combine_u32x8( + self.reinterpret_u32_i16x16(a0), + self.reinterpret_u32_i16x16(a1), + ) + } + #[inline(always)] + fn splat_u16x32(self, val: u16) -> u16x32 { + let half = self.splat_u16x16(val); + self.combine_u16x16(half, half) + } + #[inline(always)] + fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_u16x32(b).val.0, + self.cvt_to_bytes_u16x32(a).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u16x32( + self, + a: u16x32, + b: u16x32, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.slide_within_blocks_u16x16::(a0, b0), + self.slide_within_blocks_u16x16::(a1, b1), + ) + } + #[inline(always)] + fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) + } + #[inline(always)] + fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) + } + #[inline(always)] + fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) + } + #[inline(always)] + fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) + } + #[inline(always)] + fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) + } + #[inline(always)] + fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) + } + #[inline(always)] + fn not_u16x32(self, a: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) + } + #[inline(always)] + fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) + } + #[inline(always)] + fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) + } + #[inline(always)] + fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) + } + #[inline(always)] + fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) + } + #[inline(always)] + fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, _) = self.split_u16x32(a); + let (b0, _) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) + } + #[inline(always)] + fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (_, a1) = self.split_u16x32(a); + let (_, b1) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.unzip_high_u16x16(a0, a1), + self.unzip_high_u16x16(b0, b1), + ) + } + #[inline(always)] + fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_lo = self.zip_low_u16x16(a0, b0); + let lo_hi = self.zip_high_u16x16(a0, b0); + let hi_lo = self.zip_low_u16x16(a1, b1); + let hi_hi = self.zip_high_u16x16(a1, b1); + ( + self.combine_u16x16(lo_lo, lo_hi), + self.combine_u16x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_even = self.unzip_low_u16x16(a0, a1); + let lo_odd = self.unzip_high_u16x16(a0, a1); + let hi_even = self.unzip_low_u16x16(b0, b1); + let hi_odd = self.unzip_high_u16x16(b0, b1); + ( + self.combine_u16x16(lo_even, hi_even), + self.combine_u16x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_u16x32(b); + let (c0, c1) = self.split_u16x32(c); + self.combine_u16x16( + self.select_u16x16(a0, b0, c0), + self.select_u16x16(a1, b1, c1), + ) + } + #[inline(always)] + fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) + } + #[inline(always)] + fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + } + #[inline(always)] + fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + ( + u16x16 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + u16x16 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> u16x32 { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_u16x16( + token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), + token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x32, dest: &mut [u16; 32usize]) -> () { + let (v01, v23) = token.split_u16x32(a); + let (v0, v1) = token.split_u16x16(v01); + let (v2, v3) = token.split_u16x16(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn narrow_u16x32(self, a: u16x32) -> u8x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x32( + self.reinterpret_u8_u16x16(a0), + self.reinterpret_u8_u16x16(a1), + ) + } + #[inline(always)] + fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u32x8( + self.reinterpret_u32_u16x16(a0), + self.reinterpret_u32_u16x16(a1), + ) + } + #[inline(always)] + fn splat_mask16x32(self, val: bool) -> mask16x32 { + let half = self.splat_mask16x16(val); + self.combine_mask16x16(half, half) + } + #[inline(always)] + fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { + mask16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { + let lo = self.from_bitmask_mask16x16(bits); + let hi = self.from_bitmask_mask16x16(bits >> 16usize); + self.combine_mask16x16(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: mask16x32) -> u64 { + { + let lo = _mm_packs_epi16(a.val.0[0], a.val.0[1]); + let hi = _mm_packs_epi16(a.val.0[2], a.val.0[3]); + let lo = _mm_movemask_epi8(lo) as u32 as u64; + let hi = _mm_movemask_epi8(hi) as u32 as u64; + lo | (hi << 16usize) + } + } + ); + kernel(self, a) + } + #[inline(always)] + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask16x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x32(lanes); + } + #[inline(always)] + fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) + } + #[inline(always)] + fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) + } + #[inline(always)] + fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) + } + #[inline(always)] + fn not_mask16x32(self, a: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + } + #[inline(always)] + fn select_mask16x32( + self, + a: mask16x32, + b: mask16x32, + c: mask16x32, + ) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + let (c0, c1) = self.split_mask16x32(c); + self.combine_mask16x16( + self.select_mask16x16(a0, b0, c0), + self.select_mask16x16(a1, b1, c1), + ) + } + #[inline(always)] + fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16( + self.simd_eq_mask16x16(a0, b0), + self.simd_eq_mask16x16(a1, b1), + ) + } + #[inline(always)] + fn any_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + } + #[inline(always)] + fn all_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + } + #[inline(always)] + fn any_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + } + #[inline(always)] + fn all_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + } + #[inline(always)] + fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + ( + mask16x16 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + mask16x16 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i32x16(self, val: i32) -> i32x16 { + let half = self.splat_i32x8(val); + self.combine_i32x8(half, half) + } + #[inline(always)] + fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { + i32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_i32x16(b).val.0, + self.cvt_to_bytes_i32x16(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i32x16( + self, + a: i32x16, + b: i32x16, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8( + self.slide_within_blocks_i32x8::(a0, b0), + self.slide_within_blocks_i32x8::(a1, b1), + ) + } + #[inline(always)] + fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) + } + #[inline(always)] + fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) + } + #[inline(always)] + fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) + } + #[inline(always)] + fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) + } + #[inline(always)] + fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) + } + #[inline(always)] + fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) + } + #[inline(always)] + fn not_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) + } + #[inline(always)] + fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) + } + #[inline(always)] + fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) + } + #[inline(always)] + fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) + } + #[inline(always)] + fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, _) = self.split_i32x16(a); + let (b0, _) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) + } + #[inline(always)] + fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (_, a1) = self.split_i32x16(a); + let (_, b1) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + } + #[inline(always)] + fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + } + #[inline(always)] + fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_lo = self.zip_low_i32x8(a0, b0); + let lo_hi = self.zip_high_i32x8(a0, b0); + let hi_lo = self.zip_low_i32x8(a1, b1); + let hi_hi = self.zip_high_i32x8(a1, b1); + ( + self.combine_i32x8(lo_lo, lo_hi), + self.combine_i32x8(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_even = self.unzip_low_i32x8(a0, a1); + let lo_odd = self.unzip_high_i32x8(a0, a1); + let hi_even = self.unzip_low_i32x8(b0, b1); + let hi_odd = self.unzip_high_i32x8(b0, b1); + ( + self.combine_i32x8(lo_even, hi_even), + self.combine_i32x8(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_i32x16(b); + let (c0, c1) = self.split_i32x16(c); + self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) + } + #[inline(always)] + fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) + } + #[inline(always)] + fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + } + #[inline(always)] + fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { + ( + i32x8 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i32x8 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x32( - self.reinterpret_u8_u16x16(a0), - self.reinterpret_u8_u16x16(a1), - ) + fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { + let (a0, a1) = self.split_i32x16(a); + self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) } #[inline(always)] - fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { - let (a0, a1) = self.split_u16x32(a); + fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { + let (a0, a1) = self.split_i32x16(a); self.combine_u32x8( - self.reinterpret_u32_u16x16(a0), - self.reinterpret_u32_u16x16(a1), + self.reinterpret_u32_i32x8(a0), + self.reinterpret_u32_i32x8(a1), ) } #[inline(always)] - fn splat_mask16x32(self, val: bool) -> mask16x32 { - let half = self.splat_mask16x16(val); - self.combine_mask16x16(half, half) + fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) } #[inline(always)] - fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { - mask16x32 { + fn splat_u32x16(self, val: u32) -> u32x16 { + let half = self.splat_u32x8(val); + self.combine_u32x8(half, half) + } + #[inline(always)] + fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0) + fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { + u32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { - let lo = self.from_bitmask_mask16x16(bits); - let hi = self.from_bitmask_mask16x16(bits >> 16usize); - self.combine_mask16x16(lo, hi) + fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: mask16x32) -> u64 { - { - let lo = _mm_packs_epi16(a.val.0[0], a.val.0[1]); - let hi = _mm_packs_epi16(a.val.0[2], a.val.0[3]); - let lo = _mm_movemask_epi8(lo) as u32 as u64; - let hi = _mm_movemask_epi8(hi) as u32 as u64; - lo | (hi << 16usize) - } - } - ); - kernel(self, a) + fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) + fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u32; 16usize]>(&mut a.val.0) } #[inline(always)] - fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) + fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) + fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { + u32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn not_mask16x32(self, a: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn select_mask16x32( + fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_u32x16(b).val.0, + self.cvt_to_bytes_u32x16(a).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u32x16( self, - a: mask16x32, - b: mask16x32, - c: mask16x32, - ) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - let (c0, c1) = self.split_mask16x32(c); - self.combine_mask16x16( - self.select_mask16x16(a0, b0, c0), - self.select_mask16x16(a1, b1, c1), + a: u32x16, + b: u32x16, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8( + self.slide_within_blocks_u32x8::(a0, b0), + self.slide_within_blocks_u32x8::(a1, b1), ) } #[inline(always)] - fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16( - self.simd_eq_mask16x16(a0, b0), - self.simd_eq_mask16x16(a1, b1), - ) + fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) } #[inline(always)] - fn any_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) } #[inline(always)] - fn all_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) } #[inline(always)] - fn any_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) } #[inline(always)] - fn all_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) } #[inline(always)] - fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { - ( - mask16x16 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - mask16x16 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) + fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) } #[inline(always)] - fn splat_i32x16(self, val: i32) -> i32x16 { - let half = self.splat_i32x8(val); - self.combine_i32x8(half, half) + fn not_u32x16(self, a: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) } #[inline(always)] - fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { - i32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) } #[inline(always)] - fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { - i32x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) } #[inline(always)] - fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) + fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) } #[inline(always)] - fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) + fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) } #[inline(always)] - fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i32; 16usize]>(&mut a.val.0) + fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) } #[inline(always)] - fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, _) = self.split_u32x16(a); + let (b0, _) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) } #[inline(always)] - fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { - i32x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (_, a1) = self.split_u32x16(a); + let (_, b1) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { - u8x64 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) } #[inline(always)] - fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - if SHIFT >= 16usize { - return b; - } - let result = cross_block_alignr_128x4( - self, - self.cvt_to_bytes_i32x16(b).val.0, - self.cvt_to_bytes_i32x16(a).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_i32x16(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) + fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) } #[inline(always)] - fn slide_within_blocks_i32x16( - self, - a: i32x16, - b: i32x16, - ) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8( - self.slide_within_blocks_i32x8::(a0, b0), - self.slide_within_blocks_i32x8::(a1, b1), + fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_lo = self.zip_low_u32x8(a0, b0); + let lo_hi = self.zip_high_u32x8(a0, b0); + let hi_lo = self.zip_low_u32x8(a1, b1); + let hi_hi = self.zip_high_u32x8(a1, b1); + ( + self.combine_u32x8(lo_lo, lo_hi), + self.combine_u32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) - } - #[inline(always)] - fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) + fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_even = self.unzip_low_u32x8(a0, a1); + let lo_odd = self.unzip_high_u32x8(a0, a1); + let hi_even = self.unzip_low_u32x8(b0, b1); + let hi_odd = self.unzip_high_u32x8(b0, b1); + ( + self.combine_u32x8(lo_even, hi_even), + self.combine_u32x8(lo_odd, hi_odd), + ) } #[inline(always)] - fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) + fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_u32x16(b); + let (c0, c1) = self.split_u32x16(c); + self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) } #[inline(always)] - fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) + fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) } #[inline(always)] - fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) + fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) } #[inline(always)] - fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) + fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { + ( + u32x8 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + u32x8 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] - fn not_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) + fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> u32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_u32x8( + token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) } #[inline(always)] - fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) + fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x16, dest: &mut [u32; 16usize]) -> () { + let (v01, v23) = token.split_u32x16(a); + let (v0, v1) = token.split_u32x8(v01); + let (v2, v3) = token.split_u32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); } #[inline(always)] - fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) + fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) } #[inline(always)] - fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) + fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) } #[inline(always)] - fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) + fn splat_mask32x16(self, val: bool) -> mask32x16 { + let half = self.splat_mask32x8(val); + self.combine_mask32x8(half, half) } #[inline(always)] - fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) + fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { + mask32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) + fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) } #[inline(always)] - fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) + fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { + let lo = self.from_bitmask_mask32x8(bits); + let hi = self.from_bitmask_mask32x8(bits >> 8usize); + self.combine_mask32x8(lo, hi) } #[inline(always)] - fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) + fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { + let (lo, hi) = self.split_mask32x16(a); + let lo = self.to_bitmask_mask32x8(lo); + let hi = self.to_bitmask_mask32x8(hi); + lo | (hi << 8usize) } #[inline(always)] - fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask32x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x16(lanes); } #[inline(always)] - fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, _) = self.split_i32x16(a); - let (b0, _) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) + fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) } #[inline(always)] - fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (_, a1) = self.split_i32x16(a); - let (_, b1) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) } #[inline(always)] - fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) } #[inline(always)] - fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + fn not_mask32x16(self, a: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) } #[inline(always)] - fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_lo = self.zip_low_i32x8(a0, b0); - let lo_hi = self.zip_high_i32x8(a0, b0); - let hi_lo = self.zip_low_i32x8(a1, b1); - let hi_hi = self.zip_high_i32x8(a1, b1); - ( - self.combine_i32x8(lo_lo, lo_hi), - self.combine_i32x8(hi_lo, hi_hi), + fn select_mask32x16( + self, + a: mask32x16, + b: mask32x16, + c: mask32x16, + ) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + let (c0, c1) = self.split_mask32x16(c); + self.combine_mask32x8( + self.select_mask32x8(a0, b0, c0), + self.select_mask32x8(a1, b1, c1), ) } #[inline(always)] - fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_even = self.unzip_low_i32x8(a0, a1); - let lo_odd = self.unzip_high_i32x8(a0, a1); - let hi_even = self.unzip_low_i32x8(b0, b1); - let hi_odd = self.unzip_high_i32x8(b0, b1); - ( - self.combine_i32x8(lo_even, hi_even), - self.combine_i32x8(lo_odd, hi_odd), - ) + fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) } #[inline(always)] - fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + fn any_true_mask32x16(self, a: mask32x16) -> bool { let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_i32x16(b); - let (c0, c1) = self.split_i32x16(c); - self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) + self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) } #[inline(always)] - fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) + fn all_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) } #[inline(always)] - fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + fn any_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) } #[inline(always)] - fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { + fn all_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) + } + #[inline(always)] + fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { ( - i32x8 { + mask32x8 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - i32x8 { + mask32x8 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn neg_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u32x8( - self.reinterpret_u32_i32x8(a0), - self.reinterpret_u32_i32x8(a1), - ) - } - #[inline(always)] - fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) - } - #[inline(always)] - fn splat_u32x16(self, val: u32) -> u32x16 { - let half = self.splat_u32x8(val); - self.combine_u32x8(half, half) + fn splat_f64x8(self, val: f64) -> f64x8 { + let half = self.splat_f64x4(val); + self.combine_f64x4(half, half) } #[inline(always)] - fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { - u32x16 { + fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0) + fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0) + fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { + crate::transmute::checked_cast_ref::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { + crate::transmute::checked_cast_mut::<[__m128d; 4usize], [f64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { - u32x16 { + fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { + fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - if SHIFT >= 16usize { + fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_alignr_128x4( self, - self.cvt_to_bytes_u32x16(b).val.0, - self.cvt_to_bytes_u32x16(a).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_f64x8(b).val.0, + self.cvt_to_bytes_f64x8(a).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_u32x16(u8x64 { + self.cvt_from_bytes_f64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u32x16( + fn slide_within_blocks_f64x8( self, - a: u32x16, - b: u32x16, - ) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8( - self.slide_within_blocks_u32x8::(a0, b0), - self.slide_within_blocks_u32x8::(a1, b1), - ) - } - #[inline(always)] - fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) - } - #[inline(always)] - fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) - } - #[inline(always)] - fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) - } - #[inline(always)] - fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) - } - #[inline(always)] - fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) - } - #[inline(always)] - fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) - } - #[inline(always)] - fn not_u32x16(self, a: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) - } - #[inline(always)] - fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) - } - #[inline(always)] - fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) - } - #[inline(always)] - fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) - } - #[inline(always)] - fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) + a: f64x8, + b: f64x8, + ) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.slide_within_blocks_f64x4::(a0, b0), + self.slide_within_blocks_f64x4::(a1, b1), + ) } #[inline(always)] - fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) + fn abs_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) } #[inline(always)] - fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) + fn neg_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) } #[inline(always)] - fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + fn sqrt_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) } #[inline(always)] - fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, _) = self.split_u32x16(a); - let (b0, _) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) + fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.approximate_recip_f64x4(a0), + self.approximate_recip_f64x4(a1), + ) } #[inline(always)] - fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (_, a1) = self.split_u32x16(a); - let (_, b1) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) + fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) } #[inline(always)] - fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) + fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) } #[inline(always)] - fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) } #[inline(always)] - fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_lo = self.zip_low_u32x8(a0, b0); - let lo_hi = self.zip_high_u32x8(a0, b0); - let hi_lo = self.zip_low_u32x8(a1, b1); - let hi_hi = self.zip_high_u32x8(a1, b1); - ( - self.combine_u32x8(lo_lo, lo_hi), - self.combine_u32x8(hi_lo, hi_hi), - ) + fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) } #[inline(always)] - fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_even = self.unzip_low_u32x8(a0, a1); - let lo_odd = self.unzip_high_u32x8(a0, a1); - let hi_even = self.unzip_low_u32x8(b0, b1); - let hi_odd = self.unzip_high_u32x8(b0, b1); - ( - self.combine_u32x8(lo_even, hi_even), - self.combine_u32x8(lo_odd, hi_odd), - ) + fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) } #[inline(always)] - fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_u32x16(b); - let (c0, c1) = self.split_u32x16(c); - self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) + fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) } #[inline(always)] - fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) + fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) } #[inline(always)] - fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) + fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) } #[inline(always)] - fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { - ( - u32x8 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - u32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) + fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) + fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) } #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, _) = self.split_f64x8(a); + let (b0, _) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) } #[inline(always)] - fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) + fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (_, a1) = self.split_f64x8(a); + let (_, b1) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) } #[inline(always)] - fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) + fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) } #[inline(always)] - fn splat_mask32x16(self, val: bool) -> mask32x16 { - let half = self.splat_mask32x8(val); - self.combine_mask32x8(half, half) + fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) } #[inline(always)] - fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { - mask32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_lo = self.zip_low_f64x4(a0, b0); + let lo_hi = self.zip_high_f64x4(a0, b0); + let hi_lo = self.zip_low_f64x4(a1, b1); + let hi_hi = self.zip_high_f64x4(a1, b1); + ( + self.combine_f64x4(lo_lo, lo_hi), + self.combine_f64x4(hi_lo, hi_hi), + ) } #[inline(always)] - fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0) + fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_even = self.unzip_low_f64x4(a0, a1); + let lo_odd = self.unzip_high_f64x4(a0, a1); + let hi_even = self.unzip_low_f64x4(b0, b1); + let hi_odd = self.unzip_high_f64x4(b0, b1); + ( + self.combine_f64x4(lo_even, hi_even), + self.combine_f64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { - let lo = self.from_bitmask_mask32x8(bits); - let hi = self.from_bitmask_mask32x8(bits >> 8usize); - self.combine_mask32x8(lo, hi) + fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) } #[inline(always)] - fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { - let (lo, hi) = self.split_mask32x16(a); - let lo = self.to_bitmask_mask32x8(lo); - let hi = self.to_bitmask_mask32x8(hi); - lo | (hi << 8usize) + fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) } #[inline(always)] - fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) + fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.max_precise_f64x4(a0, b0), + self.max_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) + fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.min_precise_f64x4(a0, b0), + self.min_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) + fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_add_f64x4(a0, b0, c0), + self.mul_add_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn not_mask32x16(self, a: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_sub_f64x4(a0, b0, c0), + self.mul_sub_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn select_mask32x16( - self, - a: mask32x16, - b: mask32x16, - c: mask32x16, - ) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - let (c0, c1) = self.split_mask32x16(c); - self.combine_mask32x8( - self.select_mask32x8(a0, b0, c0), - self.select_mask32x8(a1, b1, c1), - ) + fn floor_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) } #[inline(always)] - fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) + fn ceil_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) } #[inline(always)] - fn any_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) + fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.round_ties_even_f64x4(a0), + self.round_ties_even_f64x4(a1), + ) } #[inline(always)] - fn all_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) + fn fract_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) } #[inline(always)] - fn any_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) + fn trunc_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) } #[inline(always)] - fn all_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) + fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) } #[inline(always)] - fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { + fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { ( - mask32x8 { + f64x4 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - mask32x8 { + f64x4 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn splat_f64x8(self, val: f64) -> f64x8 { - let half = self.splat_f64x4(val); - self.combine_f64x4(half, half) + fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f32x8( + self.reinterpret_f32_f64x4(a0), + self.reinterpret_f32_f64x4(a1), + ) } #[inline(always)] - fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { - f64x8 { + fn splat_i64x8(self, val: i64) -> i64x8 { + let half = self.splat_i64x4(val); + self.combine_i64x4(half, half) + } + #[inline(always)] + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { - f64x8 { + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0) + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { - crate::transmute::checked_cast_ref::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0) + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { - crate::transmute::checked_cast_mut::<[__m128d; 4usize], [f64; 8usize]>(&mut a.val.0) + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { - f64x8 { + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { if SHIFT >= 8usize { return b; } let result = cross_block_alignr_128x4( self, - self.cvt_to_bytes_f64x8(b).val.0, - self.cvt_to_bytes_f64x8(a).val.0, + self.cvt_to_bytes_i64x8(b).val.0, + self.cvt_to_bytes_i64x8(a).val.0, SHIFT * 8usize, ); - self.cvt_from_bytes_f64x8(u8x64 { + self.cvt_from_bytes_i64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x8( + fn slide_within_blocks_i64x8( self, - a: f64x8, - b: f64x8, - ) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.slide_within_blocks_f64x4::(a0, b0), - self.slide_within_blocks_f64x4::(a1, b1), + a: i64x8, + b: i64x8, + ) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4( + self.slide_within_blocks_i64x4::(a0, b0), + self.slide_within_blocks_i64x4::(a1, b1), ) } #[inline(always)] - fn abs_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) } #[inline(always)] - fn neg_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) } #[inline(always)] - fn sqrt_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) } #[inline(always)] - fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.approximate_recip_f64x4(a0), - self.approximate_recip_f64x4(a1), + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), ) } #[inline(always)] - fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u64; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_u64x8(b).val.0, + self.cvt_to_bytes_u64x8(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), + ) } #[inline(always)] - fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.add_u64x4(a0, b0), self.add_u64x4(a1, b1)) } #[inline(always)] - fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.sub_u64x4(a0, b0), self.sub_u64x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1)) } #[inline(always)] - fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.and_u64x4(a0, b0), self.and_u64x4(a1, b1)) } #[inline(always)] - fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.or_u64x4(a0, b0), self.or_u64x4(a1, b1)) } #[inline(always)] - fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.xor_u64x4(a0, b0), self.xor_u64x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, _) = self.split_f64x8(a); - let (b0, _) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) + fn not_u64x8(self, a: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.not_u64x4(a0), self.not_u64x4(a1)) } #[inline(always)] - fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (_, a1) = self.split_f64x8(a); - let (_, b1) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shl_u64x4(a0, shift), self.shl_u64x4(a1, shift)) } #[inline(always)] - fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shlv_u64x4(a0, b0), self.shlv_u64x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shr_u64x4(a0, shift), self.shr_u64x4(a1, shift)) } #[inline(always)] - fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_lo = self.zip_low_f64x4(a0, b0); - let lo_hi = self.zip_high_f64x4(a0, b0); - let hi_lo = self.zip_low_f64x4(a1, b1); - let hi_hi = self.zip_high_f64x4(a1, b1); - ( - self.combine_f64x4(lo_lo, lo_hi), - self.combine_f64x4(hi_lo, hi_hi), - ) + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shrv_u64x4(a0, b0), self.shrv_u64x4(a1, b1)) } #[inline(always)] - fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_even = self.unzip_low_f64x4(a0, a1); - let lo_odd = self.unzip_high_f64x4(a0, a1); - let hi_even = self.unzip_low_f64x4(b0, b1); - let hi_odd = self.unzip_high_f64x4(b0, b1); - ( - self.combine_f64x4(lo_even, hi_even), - self.combine_f64x4(lo_odd, hi_odd), - ) + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_eq_u64x4(a0, b0), self.simd_eq_u64x4(a1, b1)) } #[inline(always)] - fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_lt_u64x4(a0, b0), self.simd_lt_u64x4(a1, b1)) } #[inline(always)] - fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_le_u64x4(a0, b0), self.simd_le_u64x4(a1, b1)) } #[inline(always)] - fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.max_precise_f64x4(a0, b0), - self.max_precise_f64x4(a1, b1), - ) + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_ge_u64x4(a0, b0), self.simd_ge_u64x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.min_precise_f64x4(a0, b0), - self.min_precise_f64x4(a1, b1), - ) + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_gt_u64x4(a0, b0), self.simd_gt_u64x4(a1, b1)) } #[inline(always)] - fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_add_f64x4(a0, b0, c0), - self.mul_add_f64x4(a1, b1, c1), - ) + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, _) = self.split_u64x8(a); + let (b0, _) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a0, b0), self.zip_high_u64x4(a0, b0)) } #[inline(always)] - fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_sub_f64x4(a0, b0, c0), - self.mul_sub_f64x4(a1, b1, c1), - ) + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (_, a1) = self.split_u64x8(a); + let (_, b1) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a1, b1), self.zip_high_u64x4(a1, b1)) } #[inline(always)] - fn floor_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_low_u64x4(a0, a1), self.unzip_low_u64x4(b0, b1)) } #[inline(always)] - fn ceil_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_high_u64x4(a0, a1), self.unzip_high_u64x4(b0, b1)) } #[inline(always)] - fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.round_ties_even_f64x4(a0), - self.round_ties_even_f64x4(a1), + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_lo = self.zip_low_u64x4(a0, b0); + let lo_hi = self.zip_high_u64x4(a0, b0); + let hi_lo = self.zip_low_u64x4(a1, b1); + let hi_hi = self.zip_high_u64x4(a1, b1); + ( + self.combine_u64x4(lo_lo, lo_hi), + self.combine_u64x4(hi_lo, hi_hi), ) } #[inline(always)] - fn fract_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_even = self.unzip_low_u64x4(a0, a1); + let lo_odd = self.unzip_high_u64x4(a0, a1); + let hi_even = self.unzip_low_u64x4(b0, b1); + let hi_odd = self.unzip_high_u64x4(b0, b1); + ( + self.combine_u64x4(lo_even, hi_even), + self.combine_u64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn trunc_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_u64x8(b); + let (c0, c1) = self.split_u64x8(c); + self.combine_u64x4(self.select_u64x4(a0, b0, c0), self.select_u64x4(a1, b1, c1)) } #[inline(always)] - fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.min_u64x4(a0, b0), self.min_u64x4(a1, b1)) } #[inline(always)] - fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.max_u64x4(a0, b0), self.max_u64x4(a1, b1)) + } + #[inline(always)] + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( - f64x4 { + u64x4 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - f64x4 { + u64x4 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f32x8( - self.reinterpret_f32_f64x4(a0), - self.reinterpret_f32_f64x4(a1), + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> u64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + token.combine_u64x4( + token.combine_u64x2(out0.simd_into(token), out1.simd_into(token)), + token.combine_u64x2(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x8, dest: &mut [u64; 8usize]) -> () { + let (v01, v23) = token.split_u64x8(a); + let (v0, v1) = token.split_u64x4(v01); + let (v2, v3) = token.split_u64x4(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u8x32(self.reinterpret_u8_u64x4(a0), self.reinterpret_u8_u64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u32x8( + self.reinterpret_u32_u64x4(a0), + self.reinterpret_u32_u64x4(a1), ) } #[inline(always)] @@ -9639,6 +11892,17 @@ impl Simd for Sse4_2 { lo | (hi << 4usize) } #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask64x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x8(lanes); + } + #[inline(always)] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { let (a0, a1) = self.split_mask64x8(a); let (b0, b1) = self.split_mask64x8(b); @@ -9764,16 +12028,15 @@ impl From> for __m128i { impl SimdFrom<__m128i, S> for mask8x16 { #[inline(always)] fn simd_from(simd: S, arch: __m128i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m128i { #[inline(always)] fn from(value: mask8x16) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i8; 16usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m128i, S> for i16x8 { @@ -9809,16 +12072,15 @@ impl From> for __m128i { impl SimdFrom<__m128i, S> for mask16x8 { #[inline(always)] fn simd_from(simd: S, arch: __m128i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m128i { #[inline(always)] fn from(value: mask16x8) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i16; 8usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m128i, S> for i32x4 { @@ -9854,16 +12116,15 @@ impl From> for __m128i { impl SimdFrom<__m128i, S> for mask32x4 { #[inline(always)] fn simd_from(simd: S, arch: __m128i) -> Self { - Self { - val: crate::transmute::checked_transmute_copy(&arch), - simd, - } + let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) } } impl From> for __m128i { #[inline(always)] fn from(value: mask32x4) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i32; 4usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } impl SimdFrom<__m128d, S> for f64x2 { @@ -9881,7 +12142,22 @@ impl From> for __m128d { crate::transmute::checked_transmute_copy(&value.val) } } -impl SimdFrom<__m128i, S> for mask64x2 { +impl SimdFrom<__m128i, S> for i64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: __m128i) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for __m128i { + #[inline(always)] + fn from(value: i64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m128i, S> for u64x2 { #[inline(always)] fn simd_from(simd: S, arch: __m128i) -> Self { Self { @@ -9890,10 +12166,24 @@ impl SimdFrom<__m128i, S> for mask64x2 { } } } +impl From> for __m128i { + #[inline(always)] + fn from(value: u64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom<__m128i, S> for mask64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: __m128i) -> Self { + let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) + } +} impl From> for __m128i { #[inline(always)] fn from(value: mask64x2) -> Self { - crate::transmute::checked_transmute_copy(&value.val) + let lanes: [i64; 2usize] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) } } crate::kernel!( diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 40c0acb2d..2cab703e4 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -6,9 +6,9 @@ use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal}; use crate::{ f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4, - i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4, - mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32, - u32x4, u32x8, u32x16, + i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, + mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, + u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8, }; use core::arch::wasm32::*; #[doc = "A token for WASM SIMD128, representing the \"wasm128\" level."] @@ -35,6 +35,8 @@ impl ArchTypes for WasmSimd128 { type u32x4 = crate::support::Aligned128; type mask32x4 = crate::support::Aligned128; type f64x2 = crate::support::Aligned128; + type i64x2 = crate::support::Aligned128; + type u64x2 = crate::support::Aligned128; type mask64x2 = crate::support::Aligned128; type f32x8 = crate::support::Aligned256<[v128; 2usize]>; type i8x32 = crate::support::Aligned256<[v128; 2usize]>; @@ -47,6 +49,8 @@ impl ArchTypes for WasmSimd128 { type u32x8 = crate::support::Aligned256<[v128; 2usize]>; type mask32x8 = crate::support::Aligned256<[v128; 2usize]>; type f64x4 = crate::support::Aligned256<[v128; 2usize]>; + type i64x4 = crate::support::Aligned256<[v128; 2usize]>; + type u64x4 = crate::support::Aligned256<[v128; 2usize]>; type mask64x4 = crate::support::Aligned256<[v128; 2usize]>; type f32x16 = crate::support::Aligned512<[v128; 4usize]>; type i8x64 = crate::support::Aligned512<[v128; 4usize]>; @@ -59,6 +63,8 @@ impl ArchTypes for WasmSimd128 { type u32x16 = crate::support::Aligned512<[v128; 4usize]>; type mask32x16 = crate::support::Aligned512<[v128; 4usize]>; type f64x8 = crate::support::Aligned512<[v128; 4usize]>; + type i64x8 = crate::support::Aligned512<[v128; 4usize]>; + type u64x8 = crate::support::Aligned512<[v128; 4usize]>; type mask64x8 = crate::support::Aligned512<[v128; 4usize]>; } impl Simd for WasmSimd128 { @@ -70,6 +76,8 @@ impl Simd for WasmSimd128 { type i16s = i16x8; type u32s = u32x4; type i32s = i32x4; + type u64s = u64x2; + type i64s = i64x2; type mask8s = mask8x16; type mask16s = mask16x8; type mask32s = mask32x4; @@ -487,7 +495,27 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i8x16(self, a: i8x16, shift: u32) -> i8x16 { @@ -495,7 +523,27 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i8; 16usize] = a.into(); + let b: [i8; 16usize] = b.into(); + let result: [i8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i8x16(self, a: i8x16, b: i8x16) -> mask8x16 { @@ -700,7 +748,27 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + core::ops::Shl::shl(a[8usize], b[8usize]), + core::ops::Shl::shl(a[9usize], b[9usize]), + core::ops::Shl::shl(a[10usize], b[10usize]), + core::ops::Shl::shl(a[11usize], b[11usize]), + core::ops::Shl::shl(a[12usize], b[12usize]), + core::ops::Shl::shl(a[13usize], b[13usize]), + core::ops::Shl::shl(a[14usize], b[14usize]), + core::ops::Shl::shl(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u8x16(self, a: u8x16, shift: u32) -> u8x16 { @@ -708,7 +776,27 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u8; 16usize] = a.into(); + let b: [u8; 16usize] = b.into(); + let result: [u8; 16usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + core::ops::Shr::shr(a[8usize], b[8usize]), + core::ops::Shr::shr(a[9usize], b[9usize]), + core::ops::Shr::shr(a[10usize], b[10usize]), + core::ops::Shr::shr(a[11usize], b[11usize]), + core::ops::Shr::shr(a[12usize], b[12usize]), + core::ops::Shr::shr(a[13usize], b[13usize]), + core::ops::Shr::shr(a[14usize], b[14usize]), + core::ops::Shr::shr(a[15usize], b[15usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u8x16(self, a: u8x16, b: u8x16) -> mask8x16 { @@ -833,6 +921,17 @@ impl Simd for WasmSimd128 { i8x16_bitmask(a.into()) as u64 } #[inline(always)] + fn set_mask8x16(self, a: &mut mask8x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask8x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x16(lanes); + } + #[inline(always)] fn and_mask8x16(self, a: mask8x16, b: mask8x16) -> mask8x16 { v128_and(a.into(), b.into()).simd_into(self) } @@ -996,7 +1095,19 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i16x8(self, a: i16x8, shift: u32) -> i16x8 { @@ -1004,7 +1115,19 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i16; 8usize] = a.into(); + let b: [i16; 8usize] = b.into(); + let result: [i16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i16x8(self, a: i16x8, b: i16x8) -> mask16x8 { @@ -1193,7 +1316,19 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + core::ops::Shl::shl(a[4usize], b[4usize]), + core::ops::Shl::shl(a[5usize], b[5usize]), + core::ops::Shl::shl(a[6usize], b[6usize]), + core::ops::Shl::shl(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u16x8(self, a: u16x8, shift: u32) -> u16x8 { @@ -1201,7 +1336,19 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u16; 8usize] = a.into(); + let b: [u16; 8usize] = b.into(); + let result: [u16; 8usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + core::ops::Shr::shr(a[4usize], b[4usize]), + core::ops::Shr::shr(a[5usize], b[5usize]), + core::ops::Shr::shr(a[6usize], b[6usize]), + core::ops::Shr::shr(a[7usize], b[7usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u16x8(self, a: u16x8, b: u16x8) -> mask16x8 { @@ -1309,6 +1456,17 @@ impl Simd for WasmSimd128 { i16x8_bitmask(a.into()) as u64 } #[inline(always)] + fn set_mask16x8(self, a: &mut mask16x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask16x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x8(lanes); + } + #[inline(always)] fn and_mask16x8(self, a: mask16x8, b: mask16x8) -> mask16x8 { v128_and(a.into(), b.into()).simd_into(self) } @@ -1472,7 +1630,15 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [i32; 4usize] = a.into(); + let b: [i32; 4usize] = b.into(); + let result: [i32; 4usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_i32x4(self, a: i32x4, shift: u32) -> i32x4 { @@ -1480,7 +1646,15 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [i32; 4usize] = a.into(); + let b: [i32; 4usize] = b.into(); + let result: [i32; 4usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_i32x4(self, a: i32x4, b: i32x4) -> mask32x4 { @@ -1673,7 +1847,15 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shlv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { - core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self) + let a: [u32; 4usize] = a.into(); + let b: [u32; 4usize] = b.into(); + let result: [u32; 4usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + core::ops::Shl::shl(a[2usize], b[2usize]), + core::ops::Shl::shl(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn shr_u32x4(self, a: u32x4, shift: u32) -> u32x4 { @@ -1681,7 +1863,15 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn shrv_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { - core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self) + let a: [u32; 4usize] = a.into(); + let b: [u32; 4usize] = b.into(); + let result: [u32; 4usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + core::ops::Shr::shr(a[2usize], b[2usize]), + core::ops::Shr::shr(a[3usize], b[3usize]), + ]; + result.simd_into(self) } #[inline(always)] fn simd_eq_u32x4(self, a: u32x4, b: u32x4) -> mask32x4 { @@ -1789,6 +1979,17 @@ impl Simd for WasmSimd128 { i32x4_bitmask(a.into()) as u64 } #[inline(always)] + fn set_mask32x4(self, a: &mut mask32x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize + ); + let mut lanes = self.as_array_mask32x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x4(lanes); + } + #[inline(always)] fn and_mask32x4(self, a: mask32x4, b: mask32x4) -> mask32x4 { v128_and(a.into(), b.into()).simd_into(self) } @@ -2100,55 +2301,178 @@ impl Simd for WasmSimd128 { ::from(a).simd_into(self) } #[inline(always)] - fn splat_mask64x2(self, val: bool) -> mask64x2 { - let val: i64 = if val { !0 } else { 0 }; + fn splat_i64x2(self, val: i64) -> i64x2 { i64x2_splat(val).simd_into(self) } #[inline(always)] - fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { - mask64x2 { + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { - let bitset = i64x2_splat(bits as i64); - let powers = u64x2(1, 2); - let selected = v128_and(bitset, powers); - i64x2_ne(selected, i64x2_splat(0)).simd_into(self) + fn as_array_ref_i64x2(self, a: &i64x2) -> &[i64; 2usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { - i64x2_bitmask(a.into()) as u64 + fn as_array_mut_i64x2(self, a: &mut i64x2) -> &mut [i64; 2usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn store_array_i64x2(self, a: i64x2, dest: &mut [i64; 2usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i64x2(self, a: u8x16) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i64x2(self, a: i64x2) -> u8x16 { + u8x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + if SHIFT >= 2usize { + return b; + } + let result = dyn_slide_128( + self.cvt_to_bytes_i64x2(a).val.0, + self.cvt_to_bytes_i64x2(b).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i64x2( + self, + a: i64x2, + b: i64x2, + ) -> i64x2 { + self.slide_i64x2::(a, b) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + i64x2_add(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn sub_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + i64x2_sub(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn mul_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + i64x2_mul(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn and_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { v128_and(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn or_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { v128_or(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + fn xor_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { v128_xor(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + fn not_i64x2(self, a: i64x2) -> i64x2 { v128_not(a.into()).simd_into(self) } #[inline(always)] - fn select_mask64x2( - self, - a: mask64x2, - b: mask64x2, - c: mask64x2, - ) -> mask64x2 { + fn shl_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + i64x2_shl(a.into(), shift).simd_into(self) + } + #[inline(always)] + fn shlv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn shr_i64x2(self, a: i64x2, shift: u32) -> i64x2 { + i64x2_shr(a.into(), shift).simd_into(self) + } + #[inline(always)] + fn shrv_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + ]; + result.simd_into(self) + } + #[inline(always)] + fn simd_eq_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + i64x2_eq(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn simd_lt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + i64x2_lt(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn simd_le_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + i64x2_le(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn simd_ge_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + i64x2_ge(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn simd_gt_i64x2(self, a: i64x2, b: i64x2) -> mask64x2 { + i64x2_gt(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn zip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn zip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn unzip_low_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn unzip_high_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn interleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b)) + } + #[inline(always)] + fn deinterleave_i64x2(self, a: i64x2, b: i64x2) -> (i64x2, i64x2) { + (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b)) + } + #[inline(always)] + fn select_i64x2(self, a: mask64x2, b: i64x2, c: i64x2) -> i64x2 { #[cfg(target_feature = "relaxed-simd")] { i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self) @@ -2159,5666 +2483,7259 @@ impl Simd for WasmSimd128 { } } #[inline(always)] - fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { - i64x2_eq(a.into(), b.into()).simd_into(self) + fn min_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn any_true_mask64x2(self, a: mask64x2) -> bool { - v128_any_true(a.into()) + fn max_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + let a: [i64; 2usize] = a.into(); + let b: [i64; 2usize] = b.into(); + let result: [i64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn all_true_mask64x2(self, a: mask64x2) -> bool { - i64x2_all_true(a.into()) + fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4 { + i64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } } #[inline(always)] - fn any_false_mask64x2(self, a: mask64x2) -> bool { - !i64x2_all_true(a.into()) + fn neg_i64x2(self, a: i64x2) -> i64x2 { + i64x2_neg(a.into()).simd_into(self) } #[inline(always)] - fn all_false_mask64x2(self, a: mask64x2) -> bool { - !v128_any_true(a.into()) + fn reinterpret_u8_i64x2(self, a: i64x2) -> u8x16 { + ::from(a).simd_into(self) } #[inline(always)] - fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { - mask64x4 { - val: crate::support::Aligned256([a.val.0, b.val.0]), - simd: self, - } + fn reinterpret_u32_i64x2(self, a: i64x2) -> u32x4 { + ::from(a).simd_into(self) } #[inline(always)] - fn splat_f32x8(self, val: f32) -> f32x8 { - let half = self.splat_f32x4(val); - self.combine_f32x4(half, half) + fn splat_u64x2(self, val: u64) -> u64x2 { + u64x2_splat(val).simd_into(self) } #[inline(always)] - fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { - f32x8 { + fn load_array_u64x2(self, val: [u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { - f32x8 { + fn load_array_ref_u64x2(self, val: &[u64; 2usize]) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [f32; 8usize]>(&a.val.0) + fn as_array_u64x2(self, a: u64x2) -> [u64; 2usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [f32; 8usize]>(&a.val.0) + fn as_array_ref_u64x2(self, a: &u64x2) -> &[u64; 2usize] { + crate::transmute::checked_cast_ref::(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [f32; 8usize]>(&mut a.val.0) + fn as_array_mut_u64x2(self, a: &mut u64x2) -> &mut [u64; 2usize] { + crate::transmute::checked_cast_mut::(&mut a.val.0) } #[inline(always)] - fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { + fn store_array_u64x2(self, a: u64x2, dest: &mut [u64; 2usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { - f32x8 { + fn cvt_from_bytes_u64x2(self, a: u8x16) -> u64x2 { + u64x2 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { - u8x32 { + fn cvt_to_bytes_u64x2(self, a: u64x2) -> u8x16 { + u8x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - if SHIFT >= 8usize { + fn slide_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + if SHIFT >= 2usize { return b; } - let result = cross_block_slide_128x2( - self.cvt_to_bytes_f32x8(a).val.0, - self.cvt_to_bytes_f32x8(b).val.0, - SHIFT * 4usize, + let result = dyn_slide_128( + self.cvt_to_bytes_u64x2(a).val.0, + self.cvt_to_bytes_u64x2(b).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_f32x8(u8x32 { - val: crate::support::Aligned256(result), + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x8( + fn slide_within_blocks_u64x2( self, - a: f32x8, - b: f32x8, - ) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.slide_within_blocks_f32x4::(a0, b0), - self.slide_within_blocks_f32x4::(a1, b1), - ) + a: u64x2, + b: u64x2, + ) -> u64x2 { + self.slide_u64x2::(a, b) } #[inline(always)] - fn abs_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) + fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_add(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn neg_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) + fn sub_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_sub(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn sqrt_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) + fn mul_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_mul(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.approximate_recip_f32x4(a0), - self.approximate_recip_f32x4(a1), - ) + fn and_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + v128_and(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) + fn or_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + v128_or(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) + fn xor_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + v128_xor(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) + fn not_u64x2(self, a: u64x2) -> u64x2 { + v128_not(a.into()).simd_into(self) } #[inline(always)] - fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) + fn shl_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + u64x2_shl(a.into(), shift).simd_into(self) } #[inline(always)] - fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) + fn shlv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + core::ops::Shl::shl(a[0usize], b[0usize]), + core::ops::Shl::shl(a[1usize], b[1usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) + fn shr_u64x2(self, a: u64x2, shift: u32) -> u64x2 { + u64x2_shr(a.into(), shift).simd_into(self) } #[inline(always)] - fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) + fn shrv_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [ + core::ops::Shr::shr(a[0usize], b[0usize]), + core::ops::Shr::shr(a[1usize], b[1usize]), + ]; + result.simd_into(self) } #[inline(always)] - fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) + fn simd_eq_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] == b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] == b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(self) } #[inline(always)] - fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) + fn simd_lt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] < b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] < b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(self) } #[inline(always)] - fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) + fn simd_le_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] <= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] <= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(self) } #[inline(always)] - fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, _) = self.split_f32x8(a); - let (b0, _) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) + fn simd_ge_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] >= b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] >= b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(self) } #[inline(always)] - fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (_, a1) = self.split_f32x8(a); - let (_, b1) = self.split_f32x8(b); - self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) + fn simd_gt_u64x2(self, a: u64x2, b: u64x2) -> mask64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let true_lane: i64 = !0; + let false_lane: i64 = 0; + let result: [i64; 2usize] = [ + if a[0usize] > b[0usize] { + true_lane + } else { + false_lane + }, + if a[1usize] > b[1usize] { + true_lane + } else { + false_lane + }, + ]; + result.simd_into(self) } #[inline(always)] - fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) + fn zip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) + fn zip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_lo = self.zip_low_f32x4(a0, b0); - let lo_hi = self.zip_high_f32x4(a0, b0); - let hi_lo = self.zip_low_f32x4(a1, b1); - let hi_hi = self.zip_high_f32x4(a1, b1); - ( - self.combine_f32x4(lo_lo, lo_hi), - self.combine_f32x4(hi_lo, hi_hi), - ) + fn unzip_low_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let lo_even = self.unzip_low_f32x4(a0, a1); - let lo_odd = self.unzip_high_f32x4(a0, a1); - let hi_even = self.unzip_low_f32x4(b0, b1); - let hi_odd = self.unzip_high_f32x4(b0, b1); - ( - self.combine_f32x4(lo_even, hi_even), - self.combine_f32x4(lo_odd, hi_odd), - ) + fn unzip_high_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) + fn interleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b)) } #[inline(always)] - fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) + fn deinterleave_u64x2(self, a: u64x2, b: u64x2) -> (u64x2, u64x2) { + (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b)) } #[inline(always)] - fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.max_precise_f32x4(a0, b0), - self.max_precise_f32x4(a1, b1), - ) + fn select_u64x2(self, a: mask64x2, b: u64x2, c: u64x2) -> u64x2 { + #[cfg(target_feature = "relaxed-simd")] + { + i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self) + } + #[cfg(not(target_feature = "relaxed-simd"))] + { + v128_bitselect(b.into(), c.into(), a.into()).simd_into(self) + } } #[inline(always)] - fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - self.combine_f32x4( - self.min_precise_f32x4(a0, b0), - self.min_precise_f32x4(a1, b1), - ) + fn min_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].min(b[0usize]), a[1usize].min(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_add_f32x4(a0, b0, c0), - self.mul_add_f32x4(a1, b1, c1), - ) + fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { + let a: [u64; 2usize] = a.into(); + let b: [u64; 2usize] = b.into(); + let result: [u64; 2usize] = [a[0usize].max(b[0usize]), a[1usize].max(b[1usize])]; + result.simd_into(self) } #[inline(always)] - fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4( - self.mul_sub_f32x4(a0, b0, c0), - self.mul_sub_f32x4(a1, b1, c1), - ) + fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4 { + u64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } } #[inline(always)] - fn floor_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) + fn reinterpret_u8_u64x2(self, a: u64x2) -> u8x16 { + ::from(a).simd_into(self) } #[inline(always)] - fn ceil_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + fn reinterpret_u32_u64x2(self, a: u64x2) -> u32x4 { + ::from(a).simd_into(self) } #[inline(always)] - fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4( - self.round_ties_even_f32x4(a0), - self.round_ties_even_f32x4(a1), - ) + fn splat_mask64x2(self, val: bool) -> mask64x2 { + let val: i64 = if val { !0 } else { 0 }; + i64x2_splat(val).simd_into(self) } #[inline(always)] - fn fract_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2 { + mask64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn trunc_f32x8(self, a: f32x8) -> f32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + fn as_array_mask64x2(self, a: mask64x2) -> [i64; 2usize] { + crate::transmute::checked_transmute_copy::(&a.val.0) } #[inline(always)] - fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_f32x8(b); - let (c0, c1) = self.split_f32x8(c); - self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2 { + let bitset = i64x2_splat(bits as i64); + let powers = u64x2(1, 2); + let selected = v128_and(bitset, powers); + i64x2_ne(selected, i64x2_splat(0)).simd_into(self) } #[inline(always)] - fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { - f32x16 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn to_bitmask_mask64x2(self, a: mask64x2) -> u64 { + i64x2_bitmask(a.into()) as u64 } #[inline(always)] - fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { - ( - f32x4 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - f32x4 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn set_mask64x2(self, a: &mut mask64x2, index: usize, value: bool) -> () { + assert!( + index < 2usize, + "mask lane index {index} is out of bounds for {} lanes", + 2usize + ); + let mut lanes = self.as_array_mask64x2(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x2(lanes); } #[inline(always)] - fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { - let (a0, a1) = self.split_f32x8(a); - self.combine_f64x2( - self.reinterpret_f64_f32x4(a0), - self.reinterpret_f64_f32x4(a1), - ) + fn and_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + v128_and(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.reinterpret_i32_f32x4(a0), - self.reinterpret_i32_f32x4(a1), - ) + fn or_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + v128_or(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + fn xor_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + v128_xor(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.reinterpret_u32_f32x4(a0), - self.reinterpret_u32_f32x4(a1), - ) + fn not_mask64x2(self, a: mask64x2) -> mask64x2 { + v128_not(a.into()).simd_into(self) } #[inline(always)] - fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) + fn select_mask64x2( + self, + a: mask64x2, + b: mask64x2, + c: mask64x2, + ) -> mask64x2 { + #[cfg(target_feature = "relaxed-simd")] + { + i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self) + } + #[cfg(not(target_feature = "relaxed-simd"))] + { + v128_bitselect(b.into(), c.into(), a.into()).simd_into(self) + } } #[inline(always)] - fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_u32x4( - self.cvt_u32_precise_f32x4(a0), - self.cvt_u32_precise_f32x4(a1), - ) + fn simd_eq_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x2 { + i64x2_eq(a.into(), b.into()).simd_into(self) } #[inline(always)] - fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + fn any_true_mask64x2(self, a: mask64x2) -> bool { + v128_any_true(a.into()) } #[inline(always)] - fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { - let (a0, a1) = self.split_f32x8(a); - self.combine_i32x4( - self.cvt_i32_precise_f32x4(a0), - self.cvt_i32_precise_f32x4(a1), - ) + fn all_true_mask64x2(self, a: mask64x2) -> bool { + i64x2_all_true(a.into()) } #[inline(always)] - fn splat_i8x32(self, val: i8) -> i8x32 { - let half = self.splat_i8x16(val); - self.combine_i8x16(half, half) + fn any_false_mask64x2(self, a: mask64x2) -> bool { + !i64x2_all_true(a.into()) } #[inline(always)] - fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { - i8x32 { + fn all_false_mask64x2(self, a: mask64x2) -> bool { + !v128_any_true(a.into()) + } + #[inline(always)] + fn combine_mask64x2(self, a: mask64x2, b: mask64x2) -> mask64x4 { + mask64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn splat_f32x8(self, val: f32) -> f32x8 { + let half = self.splat_f32x4(val); + self.combine_f32x4(half, half) + } + #[inline(always)] + fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { - i8x32 { + fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i8; 32usize]>(&a.val.0) + fn as_array_f32x8(self, a: f32x8) -> [f32; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [i8; 32usize]>(&a.val.0) + fn as_array_ref_f32x8(self, a: &f32x8) -> &[f32; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [f32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [i8; 32usize]>(&mut a.val.0) + fn as_array_mut_f32x8(self, a: &mut f32x8) -> &mut [f32; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [f32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { + fn store_array_f32x8(self, a: f32x8, dest: &mut [f32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { - i8x32 { + fn cvt_from_bytes_f32x8(self, a: u8x32) -> f32x8 { + f32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { + fn cvt_to_bytes_f32x8(self, a: f32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - if SHIFT >= 32usize { + fn slide_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_slide_128x2( - self.cvt_to_bytes_i8x32(a).val.0, - self.cvt_to_bytes_i8x32(b).val.0, - SHIFT, + self.cvt_to_bytes_f32x8(a).val.0, + self.cvt_to_bytes_f32x8(b).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i8x32(u8x32 { + self.cvt_from_bytes_f32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x32( + fn slide_within_blocks_f32x8( self, - a: i8x32, - b: i8x32, - ) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16( - self.slide_within_blocks_i8x16::(a0, b0), - self.slide_within_blocks_i8x16::(a1, b1), + a: f32x8, + b: f32x8, + ) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.slide_within_blocks_f32x4::(a0, b0), + self.slide_within_blocks_f32x4::(a1, b1), ) } #[inline(always)] - fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) - } - #[inline(always)] - fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) - } - #[inline(always)] - fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) + fn abs_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) } #[inline(always)] - fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) + fn neg_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1)) } #[inline(always)] - fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) + fn sqrt_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1)) } #[inline(always)] - fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) + fn approximate_recip_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.approximate_recip_f32x4(a0), + self.approximate_recip_f32x4(a1), + ) } #[inline(always)] - fn not_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) + fn add_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1)) } #[inline(always)] - fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) + fn sub_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1)) } #[inline(always)] - fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) + fn mul_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1)) } #[inline(always)] - fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) + fn div_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1)) } #[inline(always)] - fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) + fn copysign_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1)) } #[inline(always)] - fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) + fn simd_eq_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1)) } #[inline(always)] - fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) + fn simd_lt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1)) } #[inline(always)] - fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) + fn simd_le_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1)) } #[inline(always)] - fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) + fn simd_ge_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1)) } #[inline(always)] - fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) + fn simd_gt_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1)) } #[inline(always)] - fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, _) = self.split_i8x32(a); - let (b0, _) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) + fn zip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, _) = self.split_f32x8(a); + let (b0, _) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0)) } #[inline(always)] - fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (_, a1) = self.split_i8x32(a); - let (_, b1) = self.split_i8x32(b); - self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) + fn zip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (_, a1) = self.split_f32x8(a); + let (_, b1) = self.split_f32x8(b); + self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1)) } #[inline(always)] - fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) + fn unzip_low_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1)) } #[inline(always)] - fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) + fn unzip_high_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1)) } #[inline(always)] - fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_lo = self.zip_low_i8x16(a0, b0); - let lo_hi = self.zip_high_i8x16(a0, b0); - let hi_lo = self.zip_low_i8x16(a1, b1); - let hi_hi = self.zip_high_i8x16(a1, b1); + fn interleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_lo = self.zip_low_f32x4(a0, b0); + let lo_hi = self.zip_high_f32x4(a0, b0); + let hi_lo = self.zip_low_f32x4(a1, b1); + let hi_hi = self.zip_high_f32x4(a1, b1); ( - self.combine_i8x16(lo_lo, lo_hi), - self.combine_i8x16(hi_lo, hi_hi), + self.combine_f32x4(lo_lo, lo_hi), + self.combine_f32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - let lo_even = self.unzip_low_i8x16(a0, a1); - let lo_odd = self.unzip_high_i8x16(a0, a1); - let hi_even = self.unzip_low_i8x16(b0, b1); - let hi_odd = self.unzip_high_i8x16(b0, b1); + fn deinterleave_f32x8(self, a: f32x8, b: f32x8) -> (f32x8, f32x8) { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let lo_even = self.unzip_low_f32x4(a0, a1); + let lo_odd = self.unzip_high_f32x4(a0, a1); + let hi_even = self.unzip_low_f32x4(b0, b1); + let hi_odd = self.unzip_high_f32x4(b0, b1); ( - self.combine_i8x16(lo_even, hi_even), - self.combine_i8x16(lo_odd, hi_odd), + self.combine_f32x4(lo_even, hi_even), + self.combine_f32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_i8x32(b); - let (c0, c1) = self.split_i8x32(c); - self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) - } - #[inline(always)] - fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) + fn max_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1)) } #[inline(always)] - fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - let (b0, b1) = self.split_i8x32(b); - self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) + fn min_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1)) } #[inline(always)] - fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { - i8x64 { + fn max_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.max_precise_f32x4(a0, b0), + self.max_precise_f32x4(a1, b1), + ) + } + #[inline(always)] + fn min_precise_f32x8(self, a: f32x8, b: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + self.combine_f32x4( + self.min_precise_f32x4(a0, b0), + self.min_precise_f32x4(a1, b1), + ) + } + #[inline(always)] + fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_add_f32x4(a0, b0, c0), + self.mul_add_f32x4(a1, b1, c1), + ) + } + #[inline(always)] + fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4( + self.mul_sub_f32x4(a0, b0, c0), + self.mul_sub_f32x4(a1, b1, c1), + ) + } + #[inline(always)] + fn floor_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1)) + } + #[inline(always)] + fn ceil_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1)) + } + #[inline(always)] + fn round_ties_even_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4( + self.round_ties_even_f32x4(a0), + self.round_ties_even_f32x4(a1), + ) + } + #[inline(always)] + fn fract_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1)) + } + #[inline(always)] + fn trunc_f32x8(self, a: f32x8) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1)) + } + #[inline(always)] + fn select_f32x8(self, a: mask32x8, b: f32x8, c: f32x8) -> f32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_f32x8(b); + let (c0, c1) = self.split_f32x8(c); + self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1)) + } + #[inline(always)] + fn combine_f32x8(self, a: f32x8, b: f32x8) -> f32x16 { + f32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { + fn split_f32x8(self, a: f32x8) -> (f32x4, f32x4) { ( - i8x16 { + f32x4 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i8x16 { + f32x4 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i8x32(self, a: i8x32) -> i8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) + fn reinterpret_f64_f32x8(self, a: f32x8) -> f64x4 { + let (a0, a1) = self.split_f32x8(a); + self.combine_f64x2( + self.reinterpret_f64_f32x4(a0), + self.reinterpret_f64_f32x4(a1), + ) } #[inline(always)] - fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { - let (a0, a1) = self.split_i8x32(a); - self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + fn reinterpret_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.reinterpret_i32_f32x4(a0), + self.reinterpret_i32_f32x4(a1), + ) } #[inline(always)] - fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { - let (a0, a1) = self.split_i8x32(a); + fn reinterpret_u8_f32x8(self, a: f32x8) -> u8x32 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); self.combine_u32x4( - self.reinterpret_u32_i8x16(a0), - self.reinterpret_u32_i8x16(a1), + self.reinterpret_u32_f32x4(a0), + self.reinterpret_u32_f32x4(a1), ) } #[inline(always)] - fn splat_u8x32(self, val: u8) -> u8x32 { - let half = self.splat_u8x16(val); - self.combine_u8x16(half, half) + fn cvt_u32_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1)) } #[inline(always)] - fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { - u8x32 { + fn cvt_u32_precise_f32x8(self, a: f32x8) -> u32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_u32x4( + self.cvt_u32_precise_f32x4(a0), + self.cvt_u32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn cvt_i32_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1)) + } + #[inline(always)] + fn cvt_i32_precise_f32x8(self, a: f32x8) -> i32x8 { + let (a0, a1) = self.split_f32x8(a); + self.combine_i32x4( + self.cvt_i32_precise_f32x4(a0), + self.cvt_i32_precise_f32x4(a1), + ) + } + #[inline(always)] + fn splat_i8x32(self, val: i8) -> i8x32 { + let half = self.splat_i8x16(val); + self.combine_i8x16(half, half) + } + #[inline(always)] + fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { - u8x32 { + fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [u8; 32usize]>(&a.val.0) + fn as_array_i8x32(self, a: i8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [u8; 32usize]>(&a.val.0) + fn as_array_ref_i8x32(self, a: &i8x32) -> &[i8; 32usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [u8; 32usize]>(&mut a.val.0) + fn as_array_mut_i8x32(self, a: &mut i8x32) -> &mut [i8; 32usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [i8; 32usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + fn store_array_i8x32(self, a: i8x32, dest: &mut [i8; 32usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { - u8x32 { + fn cvt_from_bytes_i8x32(self, a: u8x32) -> i8x32 { + i8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + fn cvt_to_bytes_i8x32(self, a: i8x32) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn slide_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { if SHIFT >= 32usize { return b; } let result = cross_block_slide_128x2( - self.cvt_to_bytes_u8x32(a).val.0, - self.cvt_to_bytes_u8x32(b).val.0, + self.cvt_to_bytes_i8x32(a).val.0, + self.cvt_to_bytes_i8x32(b).val.0, SHIFT, ); - self.cvt_from_bytes_u8x32(u8x32 { + self.cvt_from_bytes_i8x32(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x32( + fn slide_within_blocks_i8x32( self, - a: u8x32, - b: u8x32, - ) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16( - self.slide_within_blocks_u8x16::(a0, b0), - self.slide_within_blocks_u8x16::(a1, b1), + a: i8x32, + b: i8x32, + ) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16( + self.slide_within_blocks_i8x16::(a0, b0), + self.slide_within_blocks_i8x16::(a1, b1), ) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) + fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1)) } #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) + fn sub_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1)) } #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) + fn mul_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1)) } #[inline(always)] - fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) + fn and_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1)) } #[inline(always)] - fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) + fn or_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1)) } #[inline(always)] - fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) + fn xor_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1)) } #[inline(always)] - fn not_u8x32(self, a: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) + fn not_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1)) } #[inline(always)] - fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) + fn shl_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift)) } #[inline(always)] - fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) + fn shlv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1)) } #[inline(always)] - fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) + fn shr_i8x32(self, a: i8x32, shift: u32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift)) } #[inline(always)] - fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) + fn shrv_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1)) } #[inline(always)] - fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) + fn simd_eq_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1)) } #[inline(always)] - fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) + fn simd_lt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1)) } #[inline(always)] - fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) + fn simd_le_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1)) } #[inline(always)] - fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) + fn simd_ge_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1)) } #[inline(always)] - fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) + fn simd_gt_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1)) } #[inline(always)] - fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, _) = self.split_u8x32(a); - let (b0, _) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) + fn zip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, _) = self.split_i8x32(a); + let (b0, _) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0)) } #[inline(always)] - fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (_, a1) = self.split_u8x32(a); - let (_, b1) = self.split_u8x32(b); - self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) + fn zip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (_, a1) = self.split_i8x32(a); + let (_, b1) = self.split_i8x32(b); + self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1)) } #[inline(always)] - fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) + fn unzip_low_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1)) } #[inline(always)] - fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) + fn unzip_high_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1)) } #[inline(always)] - fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_lo = self.zip_low_u8x16(a0, b0); - let lo_hi = self.zip_high_u8x16(a0, b0); - let hi_lo = self.zip_low_u8x16(a1, b1); - let hi_hi = self.zip_high_u8x16(a1, b1); + fn interleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_lo = self.zip_low_i8x16(a0, b0); + let lo_hi = self.zip_high_i8x16(a0, b0); + let hi_lo = self.zip_low_i8x16(a1, b1); + let hi_hi = self.zip_high_i8x16(a1, b1); ( - self.combine_u8x16(lo_lo, lo_hi), - self.combine_u8x16(hi_lo, hi_hi), + self.combine_i8x16(lo_lo, lo_hi), + self.combine_i8x16(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - let lo_even = self.unzip_low_u8x16(a0, a1); - let lo_odd = self.unzip_high_u8x16(a0, a1); - let hi_even = self.unzip_low_u8x16(b0, b1); - let hi_odd = self.unzip_high_u8x16(b0, b1); + fn deinterleave_i8x32(self, a: i8x32, b: i8x32) -> (i8x32, i8x32) { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + let lo_even = self.unzip_low_i8x16(a0, a1); + let lo_odd = self.unzip_high_i8x16(a0, a1); + let hi_even = self.unzip_low_i8x16(b0, b1); + let hi_odd = self.unzip_high_i8x16(b0, b1); ( - self.combine_u8x16(lo_even, hi_even), - self.combine_u8x16(lo_odd, hi_odd), + self.combine_i8x16(lo_even, hi_even), + self.combine_i8x16(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + fn select_i8x32(self, a: mask8x32, b: i8x32, c: i8x32) -> i8x32 { let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_u8x32(b); - let (c0, c1) = self.split_u8x32(c); - self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) + let (b0, b1) = self.split_i8x32(b); + let (c0, c1) = self.split_i8x32(c); + self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1)) } #[inline(always)] - fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) + fn min_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1)) } #[inline(always)] - fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) + fn max_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (b0, b1) = self.split_i8x32(b); + self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1)) } #[inline(always)] - fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { - u8x64 { + fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { + i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + fn split_i8x32(self, a: i8x32) -> (i8x16, i8x16) { ( - u8x16 { + i8x16 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u8x16 { + i8x16 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn widen_u8x32(self, a: u8x32) -> u16x32 { - let (a0, a1) = self.split_u8x32(a); - self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) + fn neg_i8x32(self, a: i8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1)) } #[inline(always)] - fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { - let (a0, a1) = self.split_u8x32(a); + fn reinterpret_u8_i8x32(self, a: i8x32) -> u8x32 { + let (a0, a1) = self.split_i8x32(a); + self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1)) + } + #[inline(always)] + fn reinterpret_u32_i8x32(self, a: i8x32) -> u32x8 { + let (a0, a1) = self.split_i8x32(a); self.combine_u32x4( - self.reinterpret_u32_u8x16(a0), - self.reinterpret_u32_u8x16(a1), + self.reinterpret_u32_i8x16(a0), + self.reinterpret_u32_i8x16(a1), ) } #[inline(always)] - fn splat_mask8x32(self, val: bool) -> mask8x32 { - let half = self.splat_mask8x16(val); - self.combine_mask8x16(half, half) + fn splat_u8x32(self, val: u8) -> u8x32 { + let half = self.splat_u8x16(val); + self.combine_u8x16(half, half) } #[inline(always)] - fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { - mask8x32 { + fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i8; 32usize]>(&a.val.0) + fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { - let lo = self.from_bitmask_mask8x16(bits); - let hi = self.from_bitmask_mask8x16(bits >> 16usize); - self.combine_mask8x16(lo, hi) + fn as_array_u8x32(self, a: u8x32) -> [u8; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [u8; 32usize]>(&a.val.0) } #[inline(always)] - fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { - let (lo, hi) = self.split_mask8x32(a); - let lo = self.to_bitmask_mask8x16(lo); - let hi = self.to_bitmask_mask8x16(hi); - lo | (hi << 16usize) + fn as_array_ref_u8x32(self, a: &u8x32) -> &[u8; 32usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [u8; 32usize]>(&a.val.0) } #[inline(always)] - fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) + fn as_array_mut_u8x32(self, a: &mut u8x32) -> &mut [u8; 32usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [u8; 32usize]>(&mut a.val.0) } #[inline(always)] - fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) + fn store_array_u8x32(self, a: u8x32, dest: &mut [u8; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) + fn cvt_from_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn not_mask8x32(self, a: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) + fn cvt_to_bytes_u8x32(self, a: u8x32) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn select_mask8x32( - self, - a: mask8x32, - b: mask8x32, - c: mask8x32, - ) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - let (c0, c1) = self.split_mask8x32(c); - self.combine_mask8x16( - self.select_mask8x16(a0, b0, c0), - self.select_mask8x16(a1, b1, c1), - ) + fn slide_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_slide_128x2( + self.cvt_to_bytes_u8x32(a).val.0, + self.cvt_to_bytes_u8x32(b).val.0, + SHIFT, + ); + self.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { - let (a0, a1) = self.split_mask8x32(a); - let (b0, b1) = self.split_mask8x32(b); - self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) + fn slide_within_blocks_u8x32( + self, + a: u8x32, + b: u8x32, + ) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16( + self.slide_within_blocks_u8x16::(a0, b0), + self.slide_within_blocks_u8x16::(a1, b1), + ) } #[inline(always)] - fn any_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) } #[inline(always)] - fn all_true_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) } #[inline(always)] - fn any_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) } #[inline(always)] - fn all_false_mask8x32(self, a: mask8x32) -> bool { - let (a0, a1) = self.split_mask8x32(a); - self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) } #[inline(always)] - fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { - mask8x64 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) } #[inline(always)] - fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { - ( - mask8x16 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - mask8x16 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) } #[inline(always)] - fn splat_i16x16(self, val: i16) -> i16x16 { - let half = self.splat_i16x8(val); - self.combine_i16x8(half, half) + fn not_u8x32(self, a: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1)) } #[inline(always)] - fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn shl_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift)) } #[inline(always)] - fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn shlv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1)) } #[inline(always)] - fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i16; 16usize]>(&a.val.0) + fn shr_u8x32(self, a: u8x32, shift: u32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift)) } #[inline(always)] - fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [i16; 16usize]>(&a.val.0) + fn shrv_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1)) } #[inline(always)] - fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [i16; 16usize]>(&mut a.val.0) + fn simd_eq_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1)) } #[inline(always)] - fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn simd_lt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1)) } #[inline(always)] - fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { - i16x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn simd_le_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn simd_ge_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1)) } #[inline(always)] - fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - if SHIFT >= 16usize { - return b; - } - let result = cross_block_slide_128x2( - self.cvt_to_bytes_i16x16(a).val.0, - self.cvt_to_bytes_i16x16(b).val.0, - SHIFT * 2usize, - ); - self.cvt_from_bytes_i16x16(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn simd_gt_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1)) } #[inline(always)] - fn slide_within_blocks_i16x16( - self, - a: i16x16, - b: i16x16, - ) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8( - self.slide_within_blocks_i16x8::(a0, b0), - self.slide_within_blocks_i16x8::(a1, b1), - ) + fn zip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, _) = self.split_u8x32(a); + let (b0, _) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0)) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) + fn zip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (_, a1) = self.split_u8x32(a); + let (_, b1) = self.split_u8x32(b); + self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1)) } #[inline(always)] - fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1)) } #[inline(always)] - fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1)) } #[inline(always)] - fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_lo = self.zip_low_u8x16(a0, b0); + let lo_hi = self.zip_high_u8x16(a0, b0); + let hi_lo = self.zip_low_u8x16(a1, b1); + let hi_hi = self.zip_high_u8x16(a1, b1); + ( + self.combine_u8x16(lo_lo, lo_hi), + self.combine_u8x16(hi_lo, hi_hi), + ) } #[inline(always)] - fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + let lo_even = self.unzip_low_u8x16(a0, a1); + let lo_odd = self.unzip_high_u8x16(a0, a1); + let hi_even = self.unzip_low_u8x16(b0, b1); + let hi_odd = self.unzip_high_u8x16(b0, b1); + ( + self.combine_u8x16(lo_even, hi_even), + self.combine_u8x16(lo_odd, hi_odd), + ) } #[inline(always)] - fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_u8x32(b); + let (c0, c1) = self.split_u8x32(c); + self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) } #[inline(always)] - fn not_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) + fn min_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1)) } #[inline(always)] - fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) + fn max_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1)) } #[inline(always)] - fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) + fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { + u8x64 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) + fn split_u8x32(self, a: u8x32) -> (u8x16, u8x16) { + ( + u8x16 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + u8x16 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] - fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) + fn widen_u8x32(self, a: u8x32) -> u16x32 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1)) } #[inline(always)] - fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) + fn reinterpret_u32_u8x32(self, a: u8x32) -> u32x8 { + let (a0, a1) = self.split_u8x32(a); + self.combine_u32x4( + self.reinterpret_u32_u8x16(a0), + self.reinterpret_u32_u8x16(a1), + ) } #[inline(always)] - fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) + fn splat_mask8x32(self, val: bool) -> mask8x32 { + let half = self.splat_mask8x16(val); + self.combine_mask8x16(half, half) } #[inline(always)] - fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) + fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32 { + mask8x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) + fn as_array_mask8x32(self, a: mask8x32) -> [i8; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i8; 32usize]>(&a.val.0) } #[inline(always)] - fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) + fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32 { + let lo = self.from_bitmask_mask8x16(bits); + let hi = self.from_bitmask_mask8x16(bits >> 16usize); + self.combine_mask8x16(lo, hi) } #[inline(always)] - fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, _) = self.split_i16x16(a); - let (b0, _) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) + fn to_bitmask_mask8x32(self, a: mask8x32) -> u64 { + let (lo, hi) = self.split_mask8x32(a); + let lo = self.to_bitmask_mask8x16(lo); + let hi = self.to_bitmask_mask8x16(hi); + lo | (hi << 16usize) } #[inline(always)] - fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (_, a1) = self.split_i16x16(a); - let (_, b1) = self.split_i16x16(b); - self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) + fn set_mask8x32(self, a: &mut mask8x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask8x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x32(lanes); } #[inline(always)] - fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) + fn and_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1)) } #[inline(always)] - fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) + fn or_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1)) } #[inline(always)] - fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_lo = self.zip_low_i16x8(a0, b0); - let lo_hi = self.zip_high_i16x8(a0, b0); - let hi_lo = self.zip_low_i16x8(a1, b1); - let hi_hi = self.zip_high_i16x8(a1, b1); - ( - self.combine_i16x8(lo_lo, lo_hi), - self.combine_i16x8(hi_lo, hi_hi), - ) + fn xor_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1)) } #[inline(always)] - fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - let lo_even = self.unzip_low_i16x8(a0, a1); - let lo_odd = self.unzip_high_i16x8(a0, a1); - let hi_even = self.unzip_low_i16x8(b0, b1); - let hi_odd = self.unzip_high_i16x8(b0, b1); - ( - self.combine_i16x8(lo_even, hi_even), - self.combine_i16x8(lo_odd, hi_odd), + fn not_mask8x32(self, a: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1)) + } + #[inline(always)] + fn select_mask8x32( + self, + a: mask8x32, + b: mask8x32, + c: mask8x32, + ) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + let (c0, c1) = self.split_mask8x32(c); + self.combine_mask8x16( + self.select_mask8x16(a0, b0, c0), + self.select_mask8x16(a1, b1, c1), ) } #[inline(always)] - fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_i16x16(b); - let (c0, c1) = self.split_i16x16(c); - self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) + fn simd_eq_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x32 { + let (a0, a1) = self.split_mask8x32(a); + let (b0, b1) = self.split_mask8x32(b); + self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1)) } #[inline(always)] - fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) + fn any_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1) } #[inline(always)] - fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) + fn all_true_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1) } #[inline(always)] - fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { - i16x32 { + fn any_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1) + } + #[inline(always)] + fn all_false_mask8x32(self, a: mask8x32) -> bool { + let (a0, a1) = self.split_mask8x32(a); + self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1) + } + #[inline(always)] + fn combine_mask8x32(self, a: mask8x32, b: mask8x32) -> mask8x64 { + mask8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { + fn split_mask8x32(self, a: mask8x32) -> (mask8x16, mask8x16) { ( - i16x8 { + mask8x16 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i16x8 { + mask8x16 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i16x16(self, a: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) - } - #[inline(always)] - fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { - let (a0, a1) = self.split_i16x16(a); - self.combine_u32x4( - self.reinterpret_u32_i16x8(a0), - self.reinterpret_u32_i16x8(a1), - ) - } - #[inline(always)] - fn splat_u16x16(self, val: u16) -> u16x16 { - let half = self.splat_u16x8(val); - self.combine_u16x8(half, half) + fn splat_i16x16(self, val: i16) -> i16x16 { + let half = self.splat_i16x8(val); + self.combine_i16x8(half, half) } #[inline(always)] - fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { - u16x16 { + fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { - u16x16 { + fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [u16; 16usize]>(&a.val.0) + fn as_array_i16x16(self, a: i16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [u16; 16usize]>(&a.val.0) + fn as_array_ref_i16x16(self, a: &i16x16) -> &[i16; 16usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [u16; 16usize]>(&mut a.val.0) + fn as_array_mut_i16x16(self, a: &mut i16x16) -> &mut [i16; 16usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [i16; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { + fn store_array_i16x16(self, a: i16x16, dest: &mut [i16; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { - u16x16 { + fn cvt_from_bytes_i16x16(self, a: u8x32) -> i16x16 { + i16x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { + fn cvt_to_bytes_i16x16(self, a: i16x16) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + fn slide_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { if SHIFT >= 16usize { return b; } let result = cross_block_slide_128x2( - self.cvt_to_bytes_u16x16(a).val.0, - self.cvt_to_bytes_u16x16(b).val.0, + self.cvt_to_bytes_i16x16(a).val.0, + self.cvt_to_bytes_i16x16(b).val.0, SHIFT * 2usize, ); - self.cvt_from_bytes_u16x16(u8x32 { + self.cvt_from_bytes_i16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u16x16( + fn slide_within_blocks_i16x16( self, - a: u16x16, - b: u16x16, - ) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8( - self.slide_within_blocks_u16x8::(a0, b0), - self.slide_within_blocks_u16x8::(a1, b1), - ) - } + a: i16x16, + b: i16x16, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8( + self.slide_within_blocks_i16x8::(a0, b0), + self.slide_within_blocks_i16x8::(a1, b1), + ) + } #[inline(always)] - fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) } #[inline(always)] - fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) + fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1)) } #[inline(always)] - fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) + fn mul_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1)) } #[inline(always)] - fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) + fn and_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1)) } #[inline(always)] - fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) + fn or_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1)) } #[inline(always)] - fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) + fn xor_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1)) } #[inline(always)] - fn not_u16x16(self, a: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) + fn not_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1)) } #[inline(always)] - fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) + fn shl_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift)) } #[inline(always)] - fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) + fn shlv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1)) } #[inline(always)] - fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) + fn shr_i16x16(self, a: i16x16, shift: u32) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift)) } #[inline(always)] - fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) + fn shrv_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1)) } #[inline(always)] - fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) + fn simd_eq_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1)) } #[inline(always)] - fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) + fn simd_lt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1)) } #[inline(always)] - fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) + fn simd_le_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1)) } #[inline(always)] - fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) + fn simd_ge_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1)) } #[inline(always)] - fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) + fn simd_gt_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1)) } #[inline(always)] - fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, _) = self.split_u16x16(a); - let (b0, _) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) + fn zip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, _) = self.split_i16x16(a); + let (b0, _) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0)) } #[inline(always)] - fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (_, a1) = self.split_u16x16(a); - let (_, b1) = self.split_u16x16(b); - self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) + fn zip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (_, a1) = self.split_i16x16(a); + let (_, b1) = self.split_i16x16(b); + self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1)) } #[inline(always)] - fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) + fn unzip_low_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1)) } #[inline(always)] - fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) + fn unzip_high_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1)) } #[inline(always)] - fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_lo = self.zip_low_u16x8(a0, b0); - let lo_hi = self.zip_high_u16x8(a0, b0); - let hi_lo = self.zip_low_u16x8(a1, b1); - let hi_hi = self.zip_high_u16x8(a1, b1); + fn interleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_lo = self.zip_low_i16x8(a0, b0); + let lo_hi = self.zip_high_i16x8(a0, b0); + let hi_lo = self.zip_low_i16x8(a1, b1); + let hi_hi = self.zip_high_i16x8(a1, b1); ( - self.combine_u16x8(lo_lo, lo_hi), - self.combine_u16x8(hi_lo, hi_hi), + self.combine_i16x8(lo_lo, lo_hi), + self.combine_i16x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - let lo_even = self.unzip_low_u16x8(a0, a1); - let lo_odd = self.unzip_high_u16x8(a0, a1); - let hi_even = self.unzip_low_u16x8(b0, b1); - let hi_odd = self.unzip_high_u16x8(b0, b1); + fn deinterleave_i16x16(self, a: i16x16, b: i16x16) -> (i16x16, i16x16) { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + let lo_even = self.unzip_low_i16x8(a0, a1); + let lo_odd = self.unzip_high_i16x8(a0, a1); + let hi_even = self.unzip_low_i16x8(b0, b1); + let hi_odd = self.unzip_high_i16x8(b0, b1); ( - self.combine_u16x8(lo_even, hi_even), - self.combine_u16x8(lo_odd, hi_odd), + self.combine_i16x8(lo_even, hi_even), + self.combine_i16x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + fn select_i16x16(self, a: mask16x16, b: i16x16, c: i16x16) -> i16x16 { let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_u16x16(b); - let (c0, c1) = self.split_u16x16(c); - self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) + let (b0, b1) = self.split_i16x16(b); + let (c0, c1) = self.split_i16x16(c); + self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1)) } #[inline(always)] - fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) + fn min_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1)) } #[inline(always)] - fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { - let (a0, a1) = self.split_u16x16(a); - let (b0, b1) = self.split_u16x16(b); - self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) + fn max_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1)) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { + fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { + i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + fn split_i16x16(self, a: i16x16) -> (i16x8, i16x8) { ( - u16x8 { + i16x8 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u16x8 { + i16x8 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn narrow_u16x16(self, a: u16x16) -> u8x16 { - let mask = u16x8_splat(0xFF); - let (low, high) = self.split_u16x16(a); - let low_masked = v128_and(low.into(), mask); - let high_masked = v128_and(high.into(), mask); - let result = u8x16_narrow_i16x8(low_masked, high_masked); - result.simd_into(self) + fn neg_i16x16(self, a: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1)) } #[inline(always)] - fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { - let (a0, a1) = self.split_u16x16(a); - self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) + fn reinterpret_u8_i16x16(self, a: i16x16) -> u8x32 { + let (a0, a1) = self.split_i16x16(a); + self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1)) } #[inline(always)] - fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { - let (a0, a1) = self.split_u16x16(a); + fn reinterpret_u32_i16x16(self, a: i16x16) -> u32x8 { + let (a0, a1) = self.split_i16x16(a); self.combine_u32x4( - self.reinterpret_u32_u16x8(a0), - self.reinterpret_u32_u16x8(a1), - ) - } - #[inline(always)] - fn splat_mask16x16(self, val: bool) -> mask16x16 { - let half = self.splat_mask16x8(val); - self.combine_mask16x8(half, half) - } - #[inline(always)] - fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { - mask16x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i16; 16usize]>(&a.val.0) - } - #[inline(always)] - fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { - let lo = self.from_bitmask_mask16x8(bits); - let hi = self.from_bitmask_mask16x8(bits >> 8usize); - self.combine_mask16x8(lo, hi) - } - #[inline(always)] - fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { - let (lo, hi) = self.split_mask16x16(a); - let lo = self.to_bitmask_mask16x8(lo); - let hi = self.to_bitmask_mask16x8(hi); - lo | (hi << 8usize) - } - #[inline(always)] - fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) - } - #[inline(always)] - fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) - } - #[inline(always)] - fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) - } - #[inline(always)] - fn not_mask16x16(self, a: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) - } - #[inline(always)] - fn select_mask16x16( - self, - a: mask16x16, - b: mask16x16, - c: mask16x16, - ) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - let (c0, c1) = self.split_mask16x16(c); - self.combine_mask16x8( - self.select_mask16x8(a0, b0, c0), - self.select_mask16x8(a1, b1, c1), - ) - } - #[inline(always)] - fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { - let (a0, a1) = self.split_mask16x16(a); - let (b0, b1) = self.split_mask16x16(b); - self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) - } - #[inline(always)] - fn any_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) - } - #[inline(always)] - fn all_true_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) - } - #[inline(always)] - fn any_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) - } - #[inline(always)] - fn all_false_mask16x16(self, a: mask16x16) -> bool { - let (a0, a1) = self.split_mask16x16(a); - self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) - } - #[inline(always)] - fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { - mask16x32 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } - } - #[inline(always)] - fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { - ( - mask16x8 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - mask16x8 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, + self.reinterpret_u32_i16x8(a0), + self.reinterpret_u32_i16x8(a1), ) } #[inline(always)] - fn splat_i32x8(self, val: i32) -> i32x8 { - let half = self.splat_i32x4(val); - self.combine_i32x4(half, half) + fn splat_u16x16(self, val: u16) -> u16x16 { + let half = self.splat_u16x8(val); + self.combine_u16x8(half, half) } #[inline(always)] - fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { - i32x8 { + fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { - i32x8 { + fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i32; 8usize]>(&a.val.0) + fn as_array_u16x16(self, a: u16x16) -> [u16; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [u16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [i32; 8usize]>(&a.val.0) + fn as_array_ref_u16x16(self, a: &u16x16) -> &[u16; 16usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [u16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [i32; 8usize]>(&mut a.val.0) + fn as_array_mut_u16x16(self, a: &mut u16x16) -> &mut [u16; 16usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [u16; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + fn store_array_u16x16(self, a: u16x16, dest: &mut [u16; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { - i32x8 { + fn cvt_from_bytes_u16x16(self, a: u8x32) -> u16x16 { + u16x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + fn cvt_to_bytes_u16x16(self, a: u16x16) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - if SHIFT >= 8usize { + fn slide_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + if SHIFT >= 16usize { return b; } let result = cross_block_slide_128x2( - self.cvt_to_bytes_i32x8(a).val.0, - self.cvt_to_bytes_i32x8(b).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_u16x16(a).val.0, + self.cvt_to_bytes_u16x16(b).val.0, + SHIFT * 2usize, ); - self.cvt_from_bytes_i32x8(u8x32 { + self.cvt_from_bytes_u16x16(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x8( + fn slide_within_blocks_u16x16( self, - a: i32x8, - b: i32x8, - ) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4( - self.slide_within_blocks_i32x4::(a0, b0), - self.slide_within_blocks_i32x4::(a1, b1), + a: u16x16, + b: u16x16, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8( + self.slide_within_blocks_u16x8::(a0, b0), + self.slide_within_blocks_u16x8::(a1, b1), ) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) + fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1)) } #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) + fn sub_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1)) } #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) + fn mul_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1)) } #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) + fn and_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1)) } #[inline(always)] - fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) + fn or_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1)) } #[inline(always)] - fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) + fn xor_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1)) } #[inline(always)] - fn not_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) + fn not_u16x16(self, a: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1)) } #[inline(always)] - fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) + fn shl_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift)) } #[inline(always)] - fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) + fn shlv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1)) } #[inline(always)] - fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) + fn shr_u16x16(self, a: u16x16, shift: u32) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift)) } #[inline(always)] - fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) + fn shrv_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1)) } #[inline(always)] - fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) + fn simd_eq_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1)) } #[inline(always)] - fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) + fn simd_lt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1)) } #[inline(always)] - fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) + fn simd_le_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1)) } #[inline(always)] - fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) + fn simd_ge_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1)) } #[inline(always)] - fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) + fn simd_gt_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1)) } #[inline(always)] - fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, _) = self.split_i32x8(a); - let (b0, _) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) + fn zip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, _) = self.split_u16x16(a); + let (b0, _) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0)) } #[inline(always)] - fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (_, a1) = self.split_i32x8(a); - let (_, b1) = self.split_i32x8(b); - self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) + fn zip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (_, a1) = self.split_u16x16(a); + let (_, b1) = self.split_u16x16(b); + self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1)) } #[inline(always)] - fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) + fn unzip_low_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1)) } #[inline(always)] - fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) + fn unzip_high_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1)) } #[inline(always)] - fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_lo = self.zip_low_i32x4(a0, b0); - let lo_hi = self.zip_high_i32x4(a0, b0); - let hi_lo = self.zip_low_i32x4(a1, b1); - let hi_hi = self.zip_high_i32x4(a1, b1); + fn interleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_lo = self.zip_low_u16x8(a0, b0); + let lo_hi = self.zip_high_u16x8(a0, b0); + let hi_lo = self.zip_low_u16x8(a1, b1); + let hi_hi = self.zip_high_u16x8(a1, b1); ( - self.combine_i32x4(lo_lo, lo_hi), - self.combine_i32x4(hi_lo, hi_hi), + self.combine_u16x8(lo_lo, lo_hi), + self.combine_u16x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - let lo_even = self.unzip_low_i32x4(a0, a1); - let lo_odd = self.unzip_high_i32x4(a0, a1); - let hi_even = self.unzip_low_i32x4(b0, b1); - let hi_odd = self.unzip_high_i32x4(b0, b1); + fn deinterleave_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + let lo_even = self.unzip_low_u16x8(a0, a1); + let lo_odd = self.unzip_high_u16x8(a0, a1); + let hi_even = self.unzip_low_u16x8(b0, b1); + let hi_odd = self.unzip_high_u16x8(b0, b1); ( - self.combine_i32x4(lo_even, hi_even), - self.combine_i32x4(lo_odd, hi_odd), + self.combine_u16x8(lo_even, hi_even), + self.combine_u16x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_i32x8(b); - let (c0, c1) = self.split_i32x8(c); - self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) + fn select_u16x16(self, a: mask16x16, b: u16x16, c: u16x16) -> u16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_u16x16(b); + let (c0, c1) = self.split_u16x16(c); + self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1)) } #[inline(always)] - fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) + fn min_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1)) } #[inline(always)] - fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) + fn max_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (b0, b1) = self.split_u16x16(b); + self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1)) } #[inline(always)] - fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { - i32x16 { + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { ( - i32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i32x4 { + u16x8 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i32x8(self, a: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) + fn narrow_u16x16(self, a: u16x16) -> u8x16 { + let mask = u16x8_splat(0xFF); + let (low, high) = self.split_u16x16(a); + let low_masked = v128_and(low.into(), mask); + let high_masked = v128_and(high.into(), mask); + let result = u8x16_narrow_i16x8(low_masked, high_masked); + result.simd_into(self) } #[inline(always)] - fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { - let (a0, a1) = self.split_i32x8(a); - self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) + fn reinterpret_u8_u16x16(self, a: u16x16) -> u8x32 { + let (a0, a1) = self.split_u16x16(a); + self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1)) } #[inline(always)] - fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { - let (a0, a1) = self.split_i32x8(a); + fn reinterpret_u32_u16x16(self, a: u16x16) -> u32x8 { + let (a0, a1) = self.split_u16x16(a); self.combine_u32x4( - self.reinterpret_u32_i32x4(a0), - self.reinterpret_u32_i32x4(a1), + self.reinterpret_u32_u16x8(a0), + self.reinterpret_u32_u16x8(a1), ) } #[inline(always)] - fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { - let (a0, a1) = self.split_i32x8(a); - self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) - } - #[inline(always)] - fn splat_u32x8(self, val: u32) -> u32x8 { - let half = self.splat_u32x4(val); - self.combine_u32x4(half, half) + fn splat_mask16x16(self, val: bool) -> mask16x16 { + let half = self.splat_mask16x8(val); + self.combine_mask16x8(half, half) } #[inline(always)] - fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { - u32x8 { + fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16 { + mask16x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { - u32x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn as_array_mask16x16(self, a: mask16x16) -> [i16; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i16; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [u32; 8usize]>(&a.val.0) + fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16 { + let lo = self.from_bitmask_mask16x8(bits); + let hi = self.from_bitmask_mask16x8(bits >> 8usize); + self.combine_mask16x8(lo, hi) } #[inline(always)] - fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [u32; 8usize]>(&a.val.0) + fn to_bitmask_mask16x16(self, a: mask16x16) -> u64 { + let (lo, hi) = self.split_mask16x16(a); + let lo = self.to_bitmask_mask16x8(lo); + let hi = self.to_bitmask_mask16x8(hi); + lo | (hi << 8usize) } #[inline(always)] - fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [u32; 8usize]>(&mut a.val.0) + fn set_mask16x16(self, a: &mut mask16x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask16x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x16(lanes); } #[inline(always)] - fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn and_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1)) } #[inline(always)] - fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { - u32x8 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn or_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { - u8x32 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn xor_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1)) } #[inline(always)] - fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - if SHIFT >= 8usize { - return b; - } - let result = cross_block_slide_128x2( - self.cvt_to_bytes_u32x8(a).val.0, - self.cvt_to_bytes_u32x8(b).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_u32x8(u8x32 { - val: crate::support::Aligned256(result), - simd: self, - }) + fn not_mask16x16(self, a: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1)) } #[inline(always)] - fn slide_within_blocks_u32x8( + fn select_mask16x16( self, - a: u32x8, - b: u32x8, - ) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4( - self.slide_within_blocks_u32x4::(a0, b0), - self.slide_within_blocks_u32x4::(a1, b1), + a: mask16x16, + b: mask16x16, + c: mask16x16, + ) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + let (c0, c1) = self.split_mask16x16(c); + self.combine_mask16x8( + self.select_mask16x8(a0, b0, c0), + self.select_mask16x8(a1, b1, c1), ) } #[inline(always)] - fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) + fn simd_eq_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x16 { + let (a0, a1) = self.split_mask16x16(a); + let (b0, b1) = self.split_mask16x16(b); + self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1)) } #[inline(always)] - fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) + fn any_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1) } #[inline(always)] - fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) + fn all_true_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1) } #[inline(always)] - fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) + fn any_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1) } #[inline(always)] - fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) + fn all_false_mask16x16(self, a: mask16x16) -> bool { + let (a0, a1) = self.split_mask16x16(a); + self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1) } #[inline(always)] - fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) + fn combine_mask16x16(self, a: mask16x16, b: mask16x16) -> mask16x32 { + mask16x32 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn not_u32x8(self, a: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) + fn split_mask16x16(self, a: mask16x16) -> (mask16x8, mask16x8) { + ( + mask16x8 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + mask16x8 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] - fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) + fn splat_i32x8(self, val: i32) -> i32x8 { + let half = self.splat_i32x4(val); + self.combine_i32x4(half, half) } #[inline(always)] - fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) + fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) + fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) + fn as_array_i32x8(self, a: i32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) + fn as_array_ref_i32x8(self, a: &i32x8) -> &[i32; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) + fn as_array_mut_i32x8(self, a: &mut i32x8) -> &mut [i32; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [i32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) + fn store_array_i32x8(self, a: i32x8, dest: &mut [i32; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) + fn cvt_from_bytes_i32x8(self, a: u8x32) -> i32x8 { + i32x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) + fn cvt_to_bytes_i32x8(self, a: i32x8) -> u8x32 { + u8x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, _) = self.split_u32x8(a); - let (b0, _) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) + fn slide_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_slide_128x2( + self.cvt_to_bytes_i32x8(a).val.0, + self.cvt_to_bytes_i32x8(b).val.0, + SHIFT * 4usize, + ); + self.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: self, + }) } #[inline(always)] - fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (_, a1) = self.split_u32x8(a); - let (_, b1) = self.split_u32x8(b); - self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) + fn slide_within_blocks_i32x8( + self, + a: i32x8, + b: i32x8, + ) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4( + self.slide_within_blocks_i32x4::(a0, b0), + self.slide_within_blocks_i32x4::(a1, b1), + ) } #[inline(always)] - fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) } #[inline(always)] - fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) } #[inline(always)] - fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_lo = self.zip_low_u32x4(a0, b0); - let lo_hi = self.zip_high_u32x4(a0, b0); - let hi_lo = self.zip_low_u32x4(a1, b1); - let hi_hi = self.zip_high_u32x4(a1, b1); - ( - self.combine_u32x4(lo_lo, lo_hi), - self.combine_u32x4(hi_lo, hi_hi), - ) + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) } #[inline(always)] - fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - let lo_even = self.unzip_low_u32x4(a0, a1); - let lo_odd = self.unzip_high_u32x4(a0, a1); - let hi_even = self.unzip_low_u32x4(b0, b1); - let hi_odd = self.unzip_high_u32x4(b0, b1); - ( - self.combine_u32x4(lo_even, hi_even), - self.combine_u32x4(lo_odd, hi_odd), - ) + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) } #[inline(always)] - fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_u32x8(b); - let (c0, c1) = self.split_u32x8(c); - self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) } #[inline(always)] - fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) + fn xor_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1)) } #[inline(always)] - fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { - let (a0, a1) = self.split_u32x8(a); - let (b0, b1) = self.split_u32x8(b); - self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) + fn not_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1)) } #[inline(always)] - fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { - u32x16 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), - simd: self, - } + fn shl_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift)) } #[inline(always)] - fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { - ( - u32x4 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - u32x4 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) + fn shlv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1)) } #[inline(always)] - fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { - let (a0, a1) = self.split_u32x8(a); - self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) + fn shr_i32x8(self, a: i32x8, shift: u32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift)) } #[inline(always)] - fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { - let (a0, a1) = self.split_u32x8(a); - self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) + fn shrv_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1)) } #[inline(always)] - fn splat_mask32x8(self, val: bool) -> mask32x8 { - let half = self.splat_mask32x4(val); - self.combine_mask32x4(half, half) + fn simd_eq_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1)) } #[inline(always)] - fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { - mask32x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn simd_lt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1)) } #[inline(always)] - fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i32; 8usize]>(&a.val.0) + fn simd_le_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1)) } #[inline(always)] - fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { - let lo = self.from_bitmask_mask32x4(bits); - let hi = self.from_bitmask_mask32x4(bits >> 4usize); - self.combine_mask32x4(lo, hi) + fn simd_ge_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1)) } #[inline(always)] - fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { - let (lo, hi) = self.split_mask32x8(a); - let lo = self.to_bitmask_mask32x4(lo); - let hi = self.to_bitmask_mask32x4(hi); - lo | (hi << 4usize) + fn simd_gt_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1)) } #[inline(always)] - fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) + fn zip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, _) = self.split_i32x8(a); + let (b0, _) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0)) } #[inline(always)] - fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) + fn zip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (_, a1) = self.split_i32x8(a); + let (_, b1) = self.split_i32x8(b); + self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1)) } #[inline(always)] - fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) + fn unzip_low_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1)) } #[inline(always)] - fn not_mask32x8(self, a: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) + fn unzip_high_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1)) } #[inline(always)] - fn select_mask32x8( - self, - a: mask32x8, - b: mask32x8, - c: mask32x8, - ) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - let (c0, c1) = self.split_mask32x8(c); - self.combine_mask32x4( - self.select_mask32x4(a0, b0, c0), - self.select_mask32x4(a1, b1, c1), + fn interleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_lo = self.zip_low_i32x4(a0, b0); + let lo_hi = self.zip_high_i32x4(a0, b0); + let hi_lo = self.zip_low_i32x4(a1, b1); + let hi_hi = self.zip_high_i32x4(a1, b1); + ( + self.combine_i32x4(lo_lo, lo_hi), + self.combine_i32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { - let (a0, a1) = self.split_mask32x8(a); - let (b0, b1) = self.split_mask32x8(b); - self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) - } - #[inline(always)] - fn any_true_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) + fn deinterleave_i32x8(self, a: i32x8, b: i32x8) -> (i32x8, i32x8) { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + let lo_even = self.unzip_low_i32x4(a0, a1); + let lo_odd = self.unzip_high_i32x4(a0, a1); + let hi_even = self.unzip_low_i32x4(b0, b1); + let hi_odd = self.unzip_high_i32x4(b0, b1); + ( + self.combine_i32x4(lo_even, hi_even), + self.combine_i32x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn all_true_mask32x8(self, a: mask32x8) -> bool { + fn select_i32x8(self, a: mask32x8, b: i32x8, c: i32x8) -> i32x8 { let (a0, a1) = self.split_mask32x8(a); - self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) + let (b0, b1) = self.split_i32x8(b); + let (c0, c1) = self.split_i32x8(c); + self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1)) } #[inline(always)] - fn any_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) + fn min_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1)) } #[inline(always)] - fn all_false_mask32x8(self, a: mask32x8) -> bool { - let (a0, a1) = self.split_mask32x8(a); - self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) + fn max_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1)) } #[inline(always)] - fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { - mask32x16 { + fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { + i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { + fn split_i32x8(self, a: i32x8) -> (i32x4, i32x4) { ( - mask32x4 { + i32x4 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - mask32x4 { + i32x4 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_f64x4(self, val: f64) -> f64x4 { - let half = self.splat_f64x2(val); - self.combine_f64x2(half, half) + fn neg_i32x8(self, a: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1)) } #[inline(always)] - fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { - f64x4 { + fn reinterpret_u8_i32x8(self, a: i32x8) -> u8x32 { + let (a0, a1) = self.split_i32x8(a); + self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i32x8(self, a: i32x8) -> u32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_u32x4( + self.reinterpret_u32_i32x4(a0), + self.reinterpret_u32_i32x4(a1), + ) + } + #[inline(always)] + fn cvt_f32_i32x8(self, a: i32x8) -> f32x8 { + let (a0, a1) = self.split_i32x8(a); + self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1)) + } + #[inline(always)] + fn splat_u32x8(self, val: u32) -> u32x8 { + let half = self.splat_u32x4(val); + self.combine_u32x4(half, half) + } + #[inline(always)] + fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { - f64x4 { + fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [f64; 4usize]>(&a.val.0) + fn as_array_u32x8(self, a: u32x8) -> [u32; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { - crate::transmute::checked_cast_ref::<[v128; 2usize], [f64; 4usize]>(&a.val.0) + fn as_array_ref_u32x8(self, a: &u32x8) -> &[u32; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [u32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { - crate::transmute::checked_cast_mut::<[v128; 2usize], [f64; 4usize]>(&mut a.val.0) + fn as_array_mut_u32x8(self, a: &mut u32x8) -> &mut [u32; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [u32; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { + fn store_array_u32x8(self, a: u32x8, dest: &mut [u32; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { - f64x4 { + fn cvt_from_bytes_u32x8(self, a: u8x32) -> u32x8 { + u32x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + fn cvt_to_bytes_u32x8(self, a: u32x8) -> u8x32 { u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - if SHIFT >= 4usize { + fn slide_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_slide_128x2( - self.cvt_to_bytes_f64x4(a).val.0, - self.cvt_to_bytes_f64x4(b).val.0, - SHIFT * 8usize, + self.cvt_to_bytes_u32x8(a).val.0, + self.cvt_to_bytes_u32x8(b).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_f64x4(u8x32 { + self.cvt_from_bytes_u32x8(u8x32 { val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x4( + fn slide_within_blocks_u32x8( self, - a: f64x4, - b: f64x4, - ) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.slide_within_blocks_f64x2::(a0, b0), - self.slide_within_blocks_f64x2::(a1, b1), + a: u32x8, + b: u32x8, + ) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4( + self.slide_within_blocks_u32x4::(a0, b0), + self.slide_within_blocks_u32x4::(a1, b1), ) } #[inline(always)] - fn abs_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) - } - #[inline(always)] - fn neg_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) + fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1)) } #[inline(always)] - fn sqrt_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) + fn sub_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1)) } #[inline(always)] - fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.approximate_recip_f64x2(a0), - self.approximate_recip_f64x2(a1), - ) + fn mul_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1)) } #[inline(always)] - fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) + fn and_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1)) } #[inline(always)] - fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) + fn or_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1)) } #[inline(always)] - fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) + fn xor_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1)) } #[inline(always)] - fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) + fn not_u32x8(self, a: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1)) } #[inline(always)] - fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) + fn shl_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift)) } #[inline(always)] - fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) + fn shlv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) + fn shr_u32x8(self, a: u32x8, shift: u32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift)) } #[inline(always)] - fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) + fn shrv_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1)) } #[inline(always)] - fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) + fn simd_eq_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1)) } #[inline(always)] - fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) + fn simd_lt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, _) = self.split_f64x4(a); - let (b0, _) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) + fn simd_le_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1)) } #[inline(always)] - fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (_, a1) = self.split_f64x4(a); - let (_, b1) = self.split_f64x4(b); - self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) + fn simd_ge_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1)) } #[inline(always)] - fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) + fn simd_gt_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) + fn zip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, _) = self.split_u32x8(a); + let (b0, _) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0)) } #[inline(always)] - fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_lo = self.zip_low_f64x2(a0, b0); - let lo_hi = self.zip_high_f64x2(a0, b0); - let hi_lo = self.zip_low_f64x2(a1, b1); - let hi_hi = self.zip_high_f64x2(a1, b1); + fn zip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (_, a1) = self.split_u32x8(a); + let (_, b1) = self.split_u32x8(b); + self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1)) + } + #[inline(always)] + fn interleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_lo = self.zip_low_u32x4(a0, b0); + let lo_hi = self.zip_high_u32x4(a0, b0); + let hi_lo = self.zip_low_u32x4(a1, b1); + let hi_hi = self.zip_high_u32x4(a1, b1); ( - self.combine_f64x2(lo_lo, lo_hi), - self.combine_f64x2(hi_lo, hi_hi), + self.combine_u32x4(lo_lo, lo_hi), + self.combine_u32x4(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let lo_even = self.unzip_low_f64x2(a0, a1); - let lo_odd = self.unzip_high_f64x2(a0, a1); - let hi_even = self.unzip_low_f64x2(b0, b1); - let hi_odd = self.unzip_high_f64x2(b0, b1); + fn deinterleave_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + let lo_even = self.unzip_low_u32x4(a0, a1); + let lo_odd = self.unzip_high_u32x4(a0, a1); + let hi_even = self.unzip_low_u32x4(b0, b1); + let hi_odd = self.unzip_high_u32x4(b0, b1); ( - self.combine_f64x2(lo_even, hi_even), - self.combine_f64x2(lo_odd, hi_odd), + self.combine_u32x4(lo_even, hi_even), + self.combine_u32x4(lo_odd, hi_odd), ) } #[inline(always)] - fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) + fn select_u32x8(self, a: mask32x8, b: u32x8, c: u32x8) -> u32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_u32x8(b); + let (c0, c1) = self.split_u32x8(c); + self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1)) } #[inline(always)] - fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) + fn min_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1)) } #[inline(always)] - fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.max_precise_f64x2(a0, b0), - self.max_precise_f64x2(a1, b1), - ) + fn max_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (b0, b1) = self.split_u32x8(b); + self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - self.combine_f64x2( - self.min_precise_f64x2(a0, b0), - self.min_precise_f64x2(a1, b1), - ) + fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { + u32x16 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } } #[inline(always)] - fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_add_f64x2(a0, b0, c0), - self.mul_add_f64x2(a1, b1, c1), + fn split_u32x8(self, a: u32x8) -> (u32x4, u32x4) { + ( + u32x4 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + u32x4 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, ) } #[inline(always)] - fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2( - self.mul_sub_f64x2(a0, b0, c0), - self.mul_sub_f64x2(a1, b1, c1), - ) + fn reinterpret_u8_u32x8(self, a: u32x8) -> u8x32 { + let (a0, a1) = self.split_u32x8(a); + self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1)) } #[inline(always)] - fn floor_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) - } - #[inline(always)] - fn ceil_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) - } - #[inline(always)] - fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2( - self.round_ties_even_f64x2(a0), - self.round_ties_even_f64x2(a1), - ) - } - #[inline(always)] - fn fract_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) - } - #[inline(always)] - fn trunc_f64x4(self, a: f64x4) -> f64x4 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) + fn cvt_f32_u32x8(self, a: u32x8) -> f32x8 { + let (a0, a1) = self.split_u32x8(a); + self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1)) } #[inline(always)] - fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_f64x4(b); - let (c0, c1) = self.split_f64x4(c); - self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) + fn splat_mask32x8(self, val: bool) -> mask32x8 { + let half = self.splat_mask32x4(val); + self.combine_mask32x4(half, half) } #[inline(always)] - fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { - f64x8 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8 { + mask32x8 { + val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { - ( - f64x2 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - f64x2 { - val: crate::support::Aligned128(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { - let (a0, a1) = self.split_f64x4(a); - self.combine_f32x4( - self.reinterpret_f32_f64x2(a0), - self.reinterpret_f32_f64x2(a1), - ) - } - #[inline(always)] - fn splat_mask64x4(self, val: bool) -> mask64x4 { - let half = self.splat_mask64x2(val); - self.combine_mask64x2(half, half) - } - #[inline(always)] - fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { - mask64x4 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn as_array_mask32x8(self, a: mask32x8) -> [i32; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i32; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { - crate::transmute::checked_transmute_copy::<[v128; 2usize], [i64; 4usize]>(&a.val.0) + fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8 { + let lo = self.from_bitmask_mask32x4(bits); + let hi = self.from_bitmask_mask32x4(bits >> 4usize); + self.combine_mask32x4(lo, hi) } #[inline(always)] - fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { - let lo = self.from_bitmask_mask64x2(bits); - let hi = self.from_bitmask_mask64x2(bits >> 2usize); - self.combine_mask64x2(lo, hi) + fn to_bitmask_mask32x8(self, a: mask32x8) -> u64 { + let (lo, hi) = self.split_mask32x8(a); + let lo = self.to_bitmask_mask32x4(lo); + let hi = self.to_bitmask_mask32x4(hi); + lo | (hi << 4usize) } #[inline(always)] - fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { - let (lo, hi) = self.split_mask64x4(a); - let lo = self.to_bitmask_mask64x2(lo); - let hi = self.to_bitmask_mask64x2(hi); - lo | (hi << 2usize) + fn set_mask32x8(self, a: &mut mask32x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask32x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x8(lanes); } #[inline(always)] - fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) + fn and_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1)) } #[inline(always)] - fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) + fn or_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1)) } #[inline(always)] - fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) + fn xor_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1)) } #[inline(always)] - fn not_mask64x4(self, a: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) + fn not_mask32x8(self, a: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1)) } #[inline(always)] - fn select_mask64x4( + fn select_mask32x8( self, - a: mask64x4, - b: mask64x4, - c: mask64x4, - ) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - let (c0, c1) = self.split_mask64x4(c); - self.combine_mask64x2( - self.select_mask64x2(a0, b0, c0), - self.select_mask64x2(a1, b1, c1), + a: mask32x8, + b: mask32x8, + c: mask32x8, + ) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + let (c0, c1) = self.split_mask32x8(c); + self.combine_mask32x4( + self.select_mask32x4(a0, b0, c0), + self.select_mask32x4(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { - let (a0, a1) = self.split_mask64x4(a); - let (b0, b1) = self.split_mask64x4(b); - self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) + fn simd_eq_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x8 { + let (a0, a1) = self.split_mask32x8(a); + let (b0, b1) = self.split_mask32x8(b); + self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1)) } #[inline(always)] - fn any_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) + fn any_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1) } #[inline(always)] - fn all_true_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) + fn all_true_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1) } #[inline(always)] - fn any_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) + fn any_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1) } #[inline(always)] - fn all_false_mask64x4(self, a: mask64x4) -> bool { - let (a0, a1) = self.split_mask64x4(a); - self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + fn all_false_mask32x8(self, a: mask32x8) -> bool { + let (a0, a1) = self.split_mask32x8(a); + self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1) } #[inline(always)] - fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { - mask64x8 { + fn combine_mask32x8(self, a: mask32x8, b: mask32x8) -> mask32x16 { + mask32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), simd: self, } } #[inline(always)] - fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + fn split_mask32x8(self, a: mask32x8) -> (mask32x4, mask32x4) { ( - mask64x2 { + mask32x4 { val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - mask64x2 { + mask32x4 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn splat_f32x16(self, val: f32) -> f32x16 { - let half = self.splat_f32x8(val); - self.combine_f32x8(half, half) + fn splat_f64x4(self, val: f64) -> f64x4 { + let half = self.splat_f64x2(val); + self.combine_f64x2(half, half) } #[inline(always)] - fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { - f32x16 { + fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [f32; 16usize]>(&a.val.0) + fn as_array_f64x4(self, a: f64x4) -> [f64; 4usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [f32; 16usize]>(&a.val.0) + fn as_array_ref_f64x4(self, a: &f64x4) -> &[f64; 4usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [f64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [f32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x4(self, a: &mut f64x4) -> &mut [f64; 4usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [f64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + fn store_array_f64x4(self, a: f64x4, dest: &mut [f64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { - f32x16 { + fn cvt_from_bytes_f64x4(self, a: u8x32) -> f64x4 { + f64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { - u8x64 { + fn cvt_to_bytes_f64x4(self, a: f64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - if SHIFT >= 16usize { + fn slide_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_slide_128x4( - self.cvt_to_bytes_f32x16(a).val.0, - self.cvt_to_bytes_f32x16(b).val.0, - SHIFT * 4usize, + let result = cross_block_slide_128x2( + self.cvt_to_bytes_f64x4(a).val.0, + self.cvt_to_bytes_f64x4(b).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_f32x16(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f32x16( + fn slide_within_blocks_f64x4( self, - a: f32x16, - b: f32x16, - ) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.slide_within_blocks_f32x8::(a0, b0), - self.slide_within_blocks_f32x8::(a1, b1), - ) - } - #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + a: f64x4, + b: f64x4, + ) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.slide_within_blocks_f64x2::(a0, b0), + self.slide_within_blocks_f64x2::(a1, b1), + ) } #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + fn abs_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) } #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + fn neg_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1)) } #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1)) + } + #[inline(always)] + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.approximate_recip_f64x2(a0), + self.approximate_recip_f64x2(a1), ) } #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1)) } #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + fn sub_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1)) } #[inline(always)] - fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + fn mul_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1)) } #[inline(always)] - fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + fn div_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1)) } #[inline(always)] - fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + fn copysign_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1)) } #[inline(always)] - fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + fn simd_eq_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1)) } #[inline(always)] - fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + fn simd_lt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1)) } #[inline(always)] - fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + fn simd_le_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1)) } #[inline(always)] - fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + fn simd_ge_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1)) } #[inline(always)] - fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + fn simd_gt_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1)) } #[inline(always)] - fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, _) = self.split_f32x16(a); - let (b0, _) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + fn zip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, _) = self.split_f64x4(a); + let (b0, _) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0)) } #[inline(always)] - fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (_, a1) = self.split_f32x16(a); - let (_, b1) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) + fn zip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (_, a1) = self.split_f64x4(a); + let (_, b1) = self.split_f64x4(b); + self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1)) } #[inline(always)] - fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) + fn unzip_low_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1)) } #[inline(always)] - fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) + fn unzip_high_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1)) } #[inline(always)] - fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_lo = self.zip_low_f32x8(a0, b0); - let lo_hi = self.zip_high_f32x8(a0, b0); - let hi_lo = self.zip_low_f32x8(a1, b1); - let hi_hi = self.zip_high_f32x8(a1, b1); + fn interleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_lo = self.zip_low_f64x2(a0, b0); + let lo_hi = self.zip_high_f64x2(a0, b0); + let hi_lo = self.zip_low_f64x2(a1, b1); + let hi_hi = self.zip_high_f64x2(a1, b1); ( - self.combine_f32x8(lo_lo, lo_hi), - self.combine_f32x8(hi_lo, hi_hi), + self.combine_f64x2(lo_lo, lo_hi), + self.combine_f64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let lo_even = self.unzip_low_f32x8(a0, a1); - let lo_odd = self.unzip_high_f32x8(a0, a1); - let hi_even = self.unzip_low_f32x8(b0, b1); - let hi_odd = self.unzip_high_f32x8(b0, b1); + fn deinterleave_f64x4(self, a: f64x4, b: f64x4) -> (f64x4, f64x4) { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let lo_even = self.unzip_low_f64x2(a0, a1); + let lo_odd = self.unzip_high_f64x2(a0, a1); + let hi_even = self.unzip_low_f64x2(b0, b1); + let hi_odd = self.unzip_high_f64x2(b0, b1); ( - self.combine_f32x8(lo_even, hi_even), - self.combine_f32x8(lo_odd, hi_odd), + self.combine_f64x2(lo_even, hi_even), + self.combine_f64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) + fn max_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1)) } #[inline(always)] - fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) + fn min_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1)) } #[inline(always)] - fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.max_precise_f32x8(a0, b0), - self.max_precise_f32x8(a1, b1), + fn max_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.max_precise_f64x2(a0, b0), + self.max_precise_f64x2(a1, b1), ) } #[inline(always)] - fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8( - self.min_precise_f32x8(a0, b0), - self.min_precise_f32x8(a1, b1), + fn min_precise_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + self.combine_f64x2( + self.min_precise_f64x2(a0, b0), + self.min_precise_f64x2(a1, b1), ) } #[inline(always)] - fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_add_f32x8(a0, b0, c0), - self.mul_add_f32x8(a1, b1, c1), + fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_add_f64x2(a0, b0, c0), + self.mul_add_f64x2(a1, b1, c1), ) } #[inline(always)] - fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8( - self.mul_sub_f32x8(a0, b0, c0), - self.mul_sub_f32x8(a1, b1, c1), + fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2( + self.mul_sub_f64x2(a0, b0, c0), + self.mul_sub_f64x2(a1, b1, c1), ) } #[inline(always)] - fn floor_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) + fn floor_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1)) } #[inline(always)] - fn ceil_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) + fn ceil_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1)) } #[inline(always)] - fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.round_ties_even_f32x8(a0), - self.round_ties_even_f32x8(a1), + fn round_ties_even_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2( + self.round_ties_even_f64x2(a0), + self.round_ties_even_f64x2(a1), ) } #[inline(always)] - fn fract_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) + fn fract_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1)) } #[inline(always)] - fn trunc_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) + fn trunc_f64x4(self, a: f64x4) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1)) } #[inline(always)] - fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) + fn select_f64x4(self, a: mask64x4, b: f64x4, c: f64x4) -> f64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_f64x4(b); + let (c0, c1) = self.split_f64x4(c); + self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1)) } #[inline(always)] - fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8 { + f64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2) { ( - f32x8 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + f64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - f32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + f64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f64x4( - self.reinterpret_f64_f32x8(a0), - self.reinterpret_f64_f32x8(a1), - ) - } - #[inline(always)] - fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.reinterpret_i32_f32x8(a0), - self.reinterpret_i32_f32x8(a1), - ) - } - #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_f32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_f32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_f32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (lower, upper) = self.split_f32x16(a); - let (v0_vec, v1_vec) = self.split_f32x8(lower); - let (v2_vec, v3_vec) = self.split_f32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] - fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) - } - #[inline(always)] - fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.reinterpret_u32_f32x8(a0), - self.reinterpret_u32_f32x8(a1), - ) - } - #[inline(always)] - fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) - } - #[inline(always)] - fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_u32x8( - self.cvt_u32_precise_f32x8(a0), - self.cvt_u32_precise_f32x8(a1), - ) - } - #[inline(always)] - fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) - } - #[inline(always)] - fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_i32x8( - self.cvt_i32_precise_f32x8(a0), - self.cvt_i32_precise_f32x8(a1), + fn reinterpret_f32_f64x4(self, a: f64x4) -> f32x8 { + let (a0, a1) = self.split_f64x4(a); + self.combine_f32x4( + self.reinterpret_f32_f64x2(a0), + self.reinterpret_f32_f64x2(a1), ) } #[inline(always)] - fn splat_i8x64(self, val: i8) -> i8x64 { - let half = self.splat_i8x32(val); - self.combine_i8x32(half, half) + fn splat_i64x4(self, val: i64) -> i64x4 { + let half = self.splat_i64x2(val); + self.combine_i64x2(half, half) } #[inline(always)] - fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_i64x4(self, val: [i64; 4usize]) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { - i8x64 { + fn load_array_ref_i64x4(self, val: &[i64; 4usize]) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_i64x4(self, a: i64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_ref_i64x4(self, a: &i64x4) -> &[i64; 4usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [i64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [i8; 64usize]>(&mut a.val.0) + fn as_array_mut_i64x4(self, a: &mut i64x4) -> &mut [i64; 4usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [i64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + fn store_array_i64x4(self, a: i64x4, dest: &mut [i64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { - i8x64 { + fn cvt_from_bytes_i64x4(self, a: u8x32) -> i64x4 { + i64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { - u8x64 { + fn cvt_to_bytes_i64x4(self, a: i64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - if SHIFT >= 64usize { + fn slide_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_slide_128x4( - self.cvt_to_bytes_i8x64(a).val.0, - self.cvt_to_bytes_i8x64(b).val.0, - SHIFT, + let result = cross_block_slide_128x2( + self.cvt_to_bytes_i64x4(a).val.0, + self.cvt_to_bytes_i64x4(b).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_i8x64(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i8x64( + fn slide_within_blocks_i64x4( self, - a: i8x64, - b: i8x64, - ) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32( - self.slide_within_blocks_i8x32::(a0, b0), - self.slide_within_blocks_i8x32::(a1, b1), + a: i64x4, + b: i64x4, + ) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2( + self.slide_within_blocks_i64x2::(a0, b0), + self.slide_within_blocks_i64x2::(a1, b1), ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) } #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) } #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) } #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) } #[inline(always)] - fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) } #[inline(always)] - fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) } #[inline(always)] - fn not_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) } #[inline(always)] - fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) } #[inline(always)] - fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) } #[inline(always)] - fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) } #[inline(always)] - fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) } #[inline(always)] - fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) } #[inline(always)] - fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) } #[inline(always)] - fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) } #[inline(always)] - fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) } #[inline(always)] - fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) } #[inline(always)] - fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, _) = self.split_i8x64(a); - let (b0, _) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) } #[inline(always)] - fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (_, a1) = self.split_i8x64(a); - let (_, b1) = self.split_i8x64(b); - self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) } #[inline(always)] - fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) } #[inline(always)] - fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) } #[inline(always)] - fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_lo = self.zip_low_i8x32(a0, b0); - let lo_hi = self.zip_high_i8x32(a0, b0); - let hi_lo = self.zip_low_i8x32(a1, b1); - let hi_hi = self.zip_high_i8x32(a1, b1); + fn interleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_lo = self.zip_low_i64x2(a0, b0); + let lo_hi = self.zip_high_i64x2(a0, b0); + let hi_lo = self.zip_low_i64x2(a1, b1); + let hi_hi = self.zip_high_i64x2(a1, b1); ( - self.combine_i8x32(lo_lo, lo_hi), - self.combine_i8x32(hi_lo, hi_hi), + self.combine_i64x2(lo_lo, lo_hi), + self.combine_i64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - let lo_even = self.unzip_low_i8x32(a0, a1); - let lo_odd = self.unzip_high_i8x32(a0, a1); - let hi_even = self.unzip_low_i8x32(b0, b1); - let hi_odd = self.unzip_high_i8x32(b0, b1); + fn deinterleave_i64x4(self, a: i64x4, b: i64x4) -> (i64x4, i64x4) { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + let lo_even = self.unzip_low_i64x2(a0, a1); + let lo_odd = self.unzip_high_i64x2(a0, a1); + let hi_even = self.unzip_low_i64x2(b0, b1); + let hi_odd = self.unzip_high_i64x2(b0, b1); ( - self.combine_i8x32(lo_even, hi_even), - self.combine_i8x32(lo_odd, hi_odd), + self.combine_i64x2(lo_even, hi_even), + self.combine_i64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_i8x64(b); - let (c0, c1) = self.split_i8x64(c); - self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) + fn select_i64x4(self, a: mask64x4, b: i64x4, c: i64x4) -> i64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_i64x4(b); + let (c0, c1) = self.split_i64x4(c); + self.combine_i64x2(self.select_i64x2(a0, b0, c0), self.select_i64x2(a1, b1, c1)) } #[inline(always)] - fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + fn min_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.min_i64x2(a0, b0), self.min_i64x2(a1, b1)) } #[inline(always)] - fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + fn max_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.max_i64x2(a0, b0), self.max_i64x2(a1, b1)) } #[inline(always)] - fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { + i64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2) { ( - i8x32 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + i64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - i8x32 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + i64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn neg_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) + fn neg_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] - fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) + fn reinterpret_u8_i64x4(self, a: i64x4) -> u8x32 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u8x16(self.reinterpret_u8_i64x2(a0), self.reinterpret_u8_i64x2(a1)) } #[inline(always)] - fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { - let (a0, a1) = self.split_i8x64(a); - self.combine_u32x8( - self.reinterpret_u32_i8x32(a0), - self.reinterpret_u32_i8x32(a1), + fn reinterpret_u32_i64x4(self, a: i64x4) -> u32x8 { + let (a0, a1) = self.split_i64x4(a); + self.combine_u32x4( + self.reinterpret_u32_i64x2(a0), + self.reinterpret_u32_i64x2(a1), ) } #[inline(always)] - fn splat_u8x64(self, val: u8) -> u8x64 { - let half = self.splat_u8x32(val); - self.combine_u8x32(half, half) + fn splat_u64x4(self, val: u64) -> u64x4 { + let half = self.splat_u64x2(val); + self.combine_u64x2(half, half) } #[inline(always)] - fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_u64x4(self, val: [u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { - u8x64 { + fn load_array_ref_u64x4(self, val: &[u64; 4usize]) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [u8; 64usize]>(&a.val.0) + fn as_array_u64x4(self, a: u64x4) -> [u64; 4usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [u64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [u8; 64usize]>(&a.val.0) + fn as_array_ref_u64x4(self, a: &u64x4) -> &[u64; 4usize] { + crate::transmute::checked_cast_ref::<[v128; 2usize], [u64; 4usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [u8; 64usize]>(&mut a.val.0) + fn as_array_mut_u64x4(self, a: &mut u64x4) -> &mut [u64; 4usize] { + crate::transmute::checked_cast_mut::<[v128; 2usize], [u64; 4usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + fn store_array_u64x4(self, a: u64x4, dest: &mut [u64; 4usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { - u8x64 { + fn cvt_from_bytes_u64x4(self, a: u8x32) -> u64x4 { + u64x4 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { - u8x64 { + fn cvt_to_bytes_u64x4(self, a: u64x4) -> u8x32 { + u8x32 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - if SHIFT >= 64usize { + fn slide_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + if SHIFT >= 4usize { return b; } - let result = cross_block_slide_128x4( - self.cvt_to_bytes_u8x64(a).val.0, - self.cvt_to_bytes_u8x64(b).val.0, - SHIFT, + let result = cross_block_slide_128x2( + self.cvt_to_bytes_u64x4(a).val.0, + self.cvt_to_bytes_u64x4(b).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_u8x64(u8x64 { - val: crate::support::Aligned512(result), + self.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u8x64( + fn slide_within_blocks_u64x4( self, - a: u8x64, - b: u8x64, - ) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32( - self.slide_within_blocks_u8x32::(a0, b0), - self.slide_within_blocks_u8x32::(a1, b1), + a: u64x4, + b: u64x4, + ) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2( + self.slide_within_blocks_u64x2::(a0, b0), + self.slide_within_blocks_u64x2::(a1, b1), ) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) } #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) } #[inline(always)] - fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + fn mul_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1)) } #[inline(always)] - fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + fn and_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.and_u64x2(a0, b0), self.and_u64x2(a1, b1)) } #[inline(always)] - fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + fn or_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.or_u64x2(a0, b0), self.or_u64x2(a1, b1)) } #[inline(always)] - fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + fn xor_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.xor_u64x2(a0, b0), self.xor_u64x2(a1, b1)) } #[inline(always)] - fn not_u8x64(self, a: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + fn not_u64x4(self, a: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.not_u64x2(a0), self.not_u64x2(a1)) } #[inline(always)] - fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + fn shl_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shl_u64x2(a0, shift), self.shl_u64x2(a1, shift)) } #[inline(always)] - fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + fn shlv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shlv_u64x2(a0, b0), self.shlv_u64x2(a1, b1)) } #[inline(always)] - fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + fn shr_u64x4(self, a: u64x4, shift: u32) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u64x2(self.shr_u64x2(a0, shift), self.shr_u64x2(a1, shift)) } #[inline(always)] - fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + fn shrv_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.shrv_u64x2(a0, b0), self.shrv_u64x2(a1, b1)) } #[inline(always)] - fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + fn simd_eq_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_eq_u64x2(a0, b0), self.simd_eq_u64x2(a1, b1)) } #[inline(always)] - fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + fn simd_lt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_lt_u64x2(a0, b0), self.simd_lt_u64x2(a1, b1)) } #[inline(always)] - fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + fn simd_le_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_le_u64x2(a0, b0), self.simd_le_u64x2(a1, b1)) } #[inline(always)] - fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + fn simd_ge_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_ge_u64x2(a0, b0), self.simd_ge_u64x2(a1, b1)) } #[inline(always)] - fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) + fn simd_gt_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_mask64x2(self.simd_gt_u64x2(a0, b0), self.simd_gt_u64x2(a1, b1)) } #[inline(always)] - fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, _) = self.split_u8x64(a); - let (b0, _) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) + fn zip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, _) = self.split_u64x4(a); + let (b0, _) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a0, b0), self.zip_high_u64x2(a0, b0)) } #[inline(always)] - fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (_, a1) = self.split_u8x64(a); - let (_, b1) = self.split_u8x64(b); - self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) + fn zip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (_, a1) = self.split_u64x4(a); + let (_, b1) = self.split_u64x4(b); + self.combine_u64x2(self.zip_low_u64x2(a1, b1), self.zip_high_u64x2(a1, b1)) } #[inline(always)] - fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + fn unzip_low_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_low_u64x2(a0, a1), self.unzip_low_u64x2(b0, b1)) } #[inline(always)] - fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + fn unzip_high_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.unzip_high_u64x2(a0, a1), self.unzip_high_u64x2(b0, b1)) } #[inline(always)] - fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_lo = self.zip_low_u8x32(a0, b0); - let lo_hi = self.zip_high_u8x32(a0, b0); - let hi_lo = self.zip_low_u8x32(a1, b1); - let hi_hi = self.zip_high_u8x32(a1, b1); + fn interleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_lo = self.zip_low_u64x2(a0, b0); + let lo_hi = self.zip_high_u64x2(a0, b0); + let hi_lo = self.zip_low_u64x2(a1, b1); + let hi_hi = self.zip_high_u64x2(a1, b1); ( - self.combine_u8x32(lo_lo, lo_hi), - self.combine_u8x32(hi_lo, hi_hi), + self.combine_u64x2(lo_lo, lo_hi), + self.combine_u64x2(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - let lo_even = self.unzip_low_u8x32(a0, a1); - let lo_odd = self.unzip_high_u8x32(a0, a1); - let hi_even = self.unzip_low_u8x32(b0, b1); - let hi_odd = self.unzip_high_u8x32(b0, b1); + fn deinterleave_u64x4(self, a: u64x4, b: u64x4) -> (u64x4, u64x4) { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + let lo_even = self.unzip_low_u64x2(a0, a1); + let lo_odd = self.unzip_high_u64x2(a0, a1); + let hi_even = self.unzip_low_u64x2(b0, b1); + let hi_odd = self.unzip_high_u64x2(b0, b1); ( - self.combine_u8x32(lo_even, hi_even), - self.combine_u8x32(lo_odd, hi_odd), + self.combine_u64x2(lo_even, hi_even), + self.combine_u64x2(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_u8x64(b); - let (c0, c1) = self.split_u8x64(c); - self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) + fn select_u64x4(self, a: mask64x4, b: u64x4, c: u64x4) -> u64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_u64x4(b); + let (c0, c1) = self.split_u64x4(c); + self.combine_u64x2(self.select_u64x2(a0, b0, c0), self.select_u64x2(a1, b1, c1)) } #[inline(always)] - fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) + fn min_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.min_u64x2(a0, b0), self.min_u64x2(a1, b1)) } #[inline(always)] - fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) + fn max_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.max_u64x2(a0, b0), self.max_u64x2(a1, b1)) } #[inline(always)] - fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { + u64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2) { ( - u8x32 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + u64x2 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, - u8x32 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + u64x2 { + val: crate::support::Aligned128(a.val.0[1]), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); - let v01_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); - let v23_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); - let v01_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); - let v23_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); - let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_lower, v23_lower, - ); - let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_lower, v23_lower, - ); - let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_upper, v23_upper, + fn reinterpret_u8_u64x4(self, a: u64x4) -> u8x32 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u8x16(self.reinterpret_u8_u64x2(a0), self.reinterpret_u8_u64x2(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x4(self, a: u64x4) -> u32x8 { + let (a0, a1) = self.split_u64x4(a); + self.combine_u32x4( + self.reinterpret_u32_u64x2(a0), + self.reinterpret_u32_u64x2(a1), + ) + } + #[inline(always)] + fn splat_mask64x4(self, val: bool) -> mask64x4 { + let half = self.splat_mask64x2(val); + self.combine_mask64x2(half, half) + } + #[inline(always)] + fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4 { + mask64x4 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask64x4(self, a: mask64x4) -> [i64; 4usize] { + crate::transmute::checked_transmute_copy::<[v128; 2usize], [i64; 4usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4 { + let lo = self.from_bitmask_mask64x2(bits); + let hi = self.from_bitmask_mask64x2(bits >> 2usize); + self.combine_mask64x2(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask64x4(self, a: mask64x4) -> u64 { + let (lo, hi) = self.split_mask64x4(a); + let lo = self.to_bitmask_mask64x2(lo); + let hi = self.to_bitmask_mask64x2(hi); + lo | (hi << 2usize) + } + #[inline(always)] + fn set_mask64x4(self, a: &mut mask64x4, index: usize, value: bool) -> () { + assert!( + index < 4usize, + "mask lane index {index} is out of bounds for {} lanes", + 4usize ); - let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_upper, v23_upper, + let mut lanes = self.as_array_mask64x4(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x4(lanes); + } + #[inline(always)] + fn and_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1)) + } + #[inline(always)] + fn or_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1)) + } + #[inline(always)] + fn xor_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1)) + } + #[inline(always)] + fn not_mask64x4(self, a: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1)) + } + #[inline(always)] + fn select_mask64x4( + self, + a: mask64x4, + b: mask64x4, + c: mask64x4, + ) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + let (c0, c1) = self.split_mask64x4(c); + self.combine_mask64x2( + self.select_mask64x2(a0, b0, c0), + self.select_mask64x2(a1, b1, c1), + ) + } + #[inline(always)] + fn simd_eq_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x4 { + let (a0, a1) = self.split_mask64x4(a); + let (b0, b1) = self.split_mask64x4(b); + self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1)) + } + #[inline(always)] + fn any_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1) + } + #[inline(always)] + fn all_true_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1) + } + #[inline(always)] + fn any_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1) + } + #[inline(always)] + fn all_false_mask64x4(self, a: mask64x4) -> bool { + let (a0, a1) = self.split_mask64x4(a); + self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1) + } + #[inline(always)] + fn combine_mask64x4(self, a: mask64x4, b: mask64x4) -> mask64x8 { + mask64x8 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_mask64x4(self, a: mask64x4) -> (mask64x2, mask64x2) { + ( + mask64x2 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + mask64x2 { + val: crate::support::Aligned128(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_f32x16(self, val: f32) -> f32x16 { + let half = self.splat_f32x8(val); + self.combine_f32x8(half, half) + } + #[inline(always)] + fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_f32x16(self, a: f32x16) -> [f32; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_f32x16(self, a: &f32x16) -> &[f32; 16usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [f32; 16usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_f32x16(self, a: &mut f32x16) -> &mut [f32; 16usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [f32; 16usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_f32x16(self, a: u8x64) -> f32x16 { + f32x16 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_f32x16(self, a: f32x16) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + if SHIFT >= 16usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_f32x16(a).val.0, + self.cvt_to_bytes_f32x16(b).val.0, + SHIFT * 4usize, ); - let combined_lower = self.combine_u8x16(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u8x16(out2.simd_into(self), out3.simd_into(self)); - self.combine_u8x32(combined_lower, combined_upper) + self.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_f32x16( + self, + a: f32x16, + b: f32x16, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.slide_within_blocks_f32x8::(a0, b0), + self.slide_within_blocks_f32x8::(a1, b1), + ) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + } + #[inline(always)] + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) + } + #[inline(always)] + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + } + #[inline(always)] + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + } + #[inline(always)] + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + } + #[inline(always)] + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + } + #[inline(always)] + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, _) = self.split_f32x16(a); + let (b0, _) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + } + #[inline(always)] + fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (_, a1) = self.split_f32x16(a); + let (_, b1) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1)) + } + #[inline(always)] + fn unzip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1)) + } + #[inline(always)] + fn unzip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1)) + } + #[inline(always)] + fn interleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_lo = self.zip_low_f32x8(a0, b0); + let lo_hi = self.zip_high_f32x8(a0, b0); + let hi_lo = self.zip_low_f32x8(a1, b1); + let hi_hi = self.zip_high_f32x8(a1, b1); + ( + self.combine_f32x8(lo_lo, lo_hi), + self.combine_f32x8(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_f32x16(self, a: f32x16, b: f32x16) -> (f32x16, f32x16) { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let lo_even = self.unzip_low_f32x8(a0, a1); + let lo_odd = self.unzip_high_f32x8(a0, a1); + let hi_even = self.unzip_low_f32x8(b0, b1); + let hi_odd = self.unzip_high_f32x8(b0, b1); + ( + self.combine_f32x8(lo_even, hi_even), + self.combine_f32x8(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn max_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1)) + } + #[inline(always)] + fn min_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1)) + } + #[inline(always)] + fn max_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.max_precise_f32x8(a0, b0), + self.max_precise_f32x8(a1, b1), + ) + } + #[inline(always)] + fn min_precise_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8( + self.min_precise_f32x8(a0, b0), + self.min_precise_f32x8(a1, b1), + ) + } + #[inline(always)] + fn mul_add_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_add_f32x8(a0, b0, c0), + self.mul_add_f32x8(a1, b1, c1), + ) + } + #[inline(always)] + fn mul_sub_f32x16(self, a: f32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8( + self.mul_sub_f32x8(a0, b0, c0), + self.mul_sub_f32x8(a1, b1, c1), + ) + } + #[inline(always)] + fn floor_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1)) + } + #[inline(always)] + fn ceil_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1)) + } + #[inline(always)] + fn round_ties_even_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.round_ties_even_f32x8(a0), + self.round_ties_even_f32x8(a1), + ) + } + #[inline(always)] + fn fract_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1)) + } + #[inline(always)] + fn trunc_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1)) + } + #[inline(always)] + fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) + } + #[inline(always)] + fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + ( + f32x8 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + f32x8 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn reinterpret_f64_f32x16(self, a: f32x16) -> f64x8 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f64x4( + self.reinterpret_f64_f32x8(a0), + self.reinterpret_f64_f32x8(a1), + ) + } + #[inline(always)] + fn reinterpret_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.reinterpret_i32_f32x8(a0), + self.reinterpret_i32_f32x8(a1), + ) + } + #[inline(always)] + fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + let combined_lower = self.combine_f32x4(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_f32x4(out2.simd_into(self), out3.simd_into(self)); + self.combine_f32x8(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { + let (lower, upper) = self.split_f32x16(a); + let (v0_vec, v1_vec) = self.split_f32x8(lower); + let (v2_vec, v3_vec) = self.split_f32x8(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] + fn reinterpret_u8_f32x16(self, a: f32x16) -> u8x64 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1)) + } + #[inline(always)] + fn reinterpret_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.reinterpret_u32_f32x8(a0), + self.reinterpret_u32_f32x8(a1), + ) + } + #[inline(always)] + fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) + } + #[inline(always)] + fn cvt_u32_precise_f32x16(self, a: f32x16) -> u32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_u32x8( + self.cvt_u32_precise_f32x8(a0), + self.cvt_u32_precise_f32x8(a1), + ) + } + #[inline(always)] + fn cvt_i32_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1)) + } + #[inline(always)] + fn cvt_i32_precise_f32x16(self, a: f32x16) -> i32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_i32x8( + self.cvt_i32_precise_f32x8(a0), + self.cvt_i32_precise_f32x8(a1), + ) + } + #[inline(always)] + fn splat_i8x64(self, val: i8) -> i8x64 { + let half = self.splat_i8x32(val); + self.combine_i8x32(half, half) + } + #[inline(always)] + fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i8x64(self, a: i8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i8x64(self, a: &i8x64) -> &[i8; 64usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i8x64(self, a: &mut i8x64) -> &mut [i8; 64usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [i8; 64usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i8x64(self, a: u8x64) -> i8x64 { + i8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i8x64(self, a: i8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + if SHIFT >= 64usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_i8x64(a).val.0, + self.cvt_to_bytes_i8x64(b).val.0, + SHIFT, + ); + self.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i8x64( + self, + a: i8x64, + b: i8x64, + ) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32( + self.slide_within_blocks_i8x32::(a0, b0), + self.slide_within_blocks_i8x32::(a1, b1), + ) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + } + #[inline(always)] + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + } + #[inline(always)] + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + } + #[inline(always)] + fn not_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + } + #[inline(always)] + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + } + #[inline(always)] + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + } + #[inline(always)] + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + } + #[inline(always)] + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + } + #[inline(always)] + fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, _) = self.split_i8x64(a); + let (b0, _) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0)) + } + #[inline(always)] + fn zip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (_, a1) = self.split_i8x64(a); + let (_, b1) = self.split_i8x64(b); + self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1)) + } + #[inline(always)] + fn unzip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1)) + } + #[inline(always)] + fn unzip_high_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1)) + } + #[inline(always)] + fn interleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_lo = self.zip_low_i8x32(a0, b0); + let lo_hi = self.zip_high_i8x32(a0, b0); + let hi_lo = self.zip_low_i8x32(a1, b1); + let hi_hi = self.zip_high_i8x32(a1, b1); + ( + self.combine_i8x32(lo_lo, lo_hi), + self.combine_i8x32(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i8x64(self, a: i8x64, b: i8x64) -> (i8x64, i8x64) { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + let lo_even = self.unzip_low_i8x32(a0, a1); + let lo_odd = self.unzip_high_i8x32(a0, a1); + let hi_even = self.unzip_low_i8x32(b0, b1); + let hi_odd = self.unzip_high_i8x32(b0, b1); + ( + self.combine_i8x32(lo_even, hi_even), + self.combine_i8x32(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i8x64(self, a: mask8x64, b: i8x64, c: i8x64) -> i8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_i8x64(b); + let (c0, c1) = self.split_i8x64(c); + self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1)) + } + #[inline(always)] + fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + } + #[inline(always)] + fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + } + #[inline(always)] + fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + ( + i8x32 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i8x32 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) + } + #[inline(always)] + fn reinterpret_u8_i8x64(self, a: i8x64) -> u8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1)) + } + #[inline(always)] + fn reinterpret_u32_i8x64(self, a: i8x64) -> u32x16 { + let (a0, a1) = self.split_i8x64(a); + self.combine_u32x8( + self.reinterpret_u32_i8x32(a0), + self.reinterpret_u32_i8x32(a1), + ) + } + #[inline(always)] + fn splat_u8x64(self, val: u8) -> u8x64 { + let half = self.splat_u8x32(val); + self.combine_u8x32(half, half) + } + #[inline(always)] + fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u8x64(self, a: u8x64) -> [u8; 64usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u8x64(self, a: &u8x64) -> &[u8; 64usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [u8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u8x64(self, a: &mut u8x64) -> &mut [u8; 64usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [u8; 64usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u8x64(self, a: u8x64) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + if SHIFT >= 64usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_u8x64(a).val.0, + self.cvt_to_bytes_u8x64(b).val.0, + SHIFT, + ); + self.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u8x64( + self, + a: u8x64, + b: u8x64, + ) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32( + self.slide_within_blocks_u8x32::(a0, b0), + self.slide_within_blocks_u8x32::(a1, b1), + ) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + } + #[inline(always)] + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + } + #[inline(always)] + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + } + #[inline(always)] + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + } + #[inline(always)] + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + } + #[inline(always)] + fn not_u8x64(self, a: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + } + #[inline(always)] + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + } + #[inline(always)] + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + } + #[inline(always)] + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + } + #[inline(always)] + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) + } + #[inline(always)] + fn zip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, _) = self.split_u8x64(a); + let (b0, _) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0)) + } + #[inline(always)] + fn zip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (_, a1) = self.split_u8x64(a); + let (_, b1) = self.split_u8x64(b); + self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1)) + } + #[inline(always)] + fn unzip_low_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1)) + } + #[inline(always)] + fn unzip_high_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1)) + } + #[inline(always)] + fn interleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_lo = self.zip_low_u8x32(a0, b0); + let lo_hi = self.zip_high_u8x32(a0, b0); + let hi_lo = self.zip_low_u8x32(a1, b1); + let hi_hi = self.zip_high_u8x32(a1, b1); + ( + self.combine_u8x32(lo_lo, lo_hi), + self.combine_u8x32(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u8x64(self, a: u8x64, b: u8x64) -> (u8x64, u8x64) { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + let lo_even = self.unzip_low_u8x32(a0, a1); + let lo_odd = self.unzip_high_u8x32(a0, a1); + let hi_even = self.unzip_low_u8x32(b0, b1); + let hi_odd = self.unzip_high_u8x32(b0, b1); + ( + self.combine_u8x32(lo_even, hi_even), + self.combine_u8x32(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u8x64(self, a: mask8x64, b: u8x64, c: u8x64) -> u8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_u8x64(b); + let (c0, c1) = self.split_u8x64(c); + self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) + } + #[inline(always)] + fn min_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1)) + } + #[inline(always)] + fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1)) + } + #[inline(always)] + fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { + ( + u8x32 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + u8x32 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + let combined_lower = self.combine_u8x16(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_u8x16(out2.simd_into(self), out3.simd_into(self)); + self.combine_u8x32(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { + let (lower, upper) = self.split_u8x64(a); + let (v0_vec, v1_vec) = self.split_u8x32(lower); + let (v2_vec, v3_vec) = self.split_u8x32(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] + fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u32x8( + self.reinterpret_u32_u8x32(a0), + self.reinterpret_u32_u8x32(a1), + ) + } + #[inline(always)] + fn splat_mask8x64(self, val: bool) -> mask8x64 { + let half = self.splat_mask8x32(val); + self.combine_mask8x32(half, half) + } + #[inline(always)] + fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { + mask8x64 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + } + #[inline(always)] + fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { + let lo = self.from_bitmask_mask8x32(bits); + let hi = self.from_bitmask_mask8x32(bits >> 32usize); + self.combine_mask8x32(lo, hi) + } + #[inline(always)] + fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { + let (lo, hi) = self.split_mask8x64(a); + let lo = self.to_bitmask_mask8x32(lo); + let hi = self.to_bitmask_mask8x32(hi); + lo | (hi << 32usize) + } + #[inline(always)] + fn set_mask8x64(self, a: &mut mask8x64, index: usize, value: bool) -> () { + assert!( + index < 64usize, + "mask lane index {index} is out of bounds for {} lanes", + 64usize + ); + let mut lanes = self.as_array_mask8x64(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask8x64(lanes); + } + #[inline(always)] + fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) + } + #[inline(always)] + fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) + } + #[inline(always)] + fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) + } + #[inline(always)] + fn not_mask8x64(self, a: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) + } + #[inline(always)] + fn select_mask8x64( + self, + a: mask8x64, + b: mask8x64, + c: mask8x64, + ) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + let (c0, c1) = self.split_mask8x64(c); + self.combine_mask8x32( + self.select_mask8x32(a0, b0, c0), + self.select_mask8x32(a1, b1, c1), + ) + } + #[inline(always)] + fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { + let (a0, a1) = self.split_mask8x64(a); + let (b0, b1) = self.split_mask8x64(b); + self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) + } + #[inline(always)] + fn any_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) + } + #[inline(always)] + fn all_true_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) + } + #[inline(always)] + fn any_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) + } + #[inline(always)] + fn all_false_mask8x64(self, a: mask8x64) -> bool { + let (a0, a1) = self.split_mask8x64(a); + self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) + } + #[inline(always)] + fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + ( + mask8x32 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + mask8x32 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn splat_i16x32(self, val: i16) -> i16x32 { + let half = self.splat_i16x16(val); + self.combine_i16x16(half, half) + } + #[inline(always)] + fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [i16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [i16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { + i16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_i16x32(a).val.0, + self.cvt_to_bytes_i16x32(b).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_i16x32( + self, + a: i16x32, + b: i16x32, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.slide_within_blocks_i16x16::(a0, b0), + self.slide_within_blocks_i16x16::(a1, b1), + ) + } + #[inline(always)] + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + } + #[inline(always)] + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + } + #[inline(always)] + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + } + #[inline(always)] + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) + } + #[inline(always)] + fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) + } + #[inline(always)] + fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) + } + #[inline(always)] + fn not_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) + } + #[inline(always)] + fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) + } + #[inline(always)] + fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) + } + #[inline(always)] + fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) + } + #[inline(always)] + fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) + } + #[inline(always)] + fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, _) = self.split_i16x32(a); + let (b0, _) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) + } + #[inline(always)] + fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (_, a1) = self.split_i16x32(a); + let (_, b1) = self.split_i16x32(b); + self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16( + self.unzip_high_i16x16(a0, a1), + self.unzip_high_i16x16(b0, b1), + ) + } + #[inline(always)] + fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_lo = self.zip_low_i16x16(a0, b0); + let lo_hi = self.zip_high_i16x16(a0, b0); + let hi_lo = self.zip_low_i16x16(a1, b1); + let hi_hi = self.zip_high_i16x16(a1, b1); + ( + self.combine_i16x16(lo_lo, lo_hi), + self.combine_i16x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + let lo_even = self.unzip_low_i16x16(a0, a1); + let lo_odd = self.unzip_high_i16x16(a0, a1); + let hi_even = self.unzip_low_i16x16(b0, b1); + let hi_odd = self.unzip_high_i16x16(b0, b1); + ( + self.combine_i16x16(lo_even, hi_even), + self.combine_i16x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_i16x32(b); + let (c0, c1) = self.split_i16x32(c); + self.combine_i16x16( + self.select_i16x16(a0, b0, c0), + self.select_i16x16(a1, b1, c1), + ) + } + #[inline(always)] + fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) + } + #[inline(always)] + fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) + } + #[inline(always)] + fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { + ( + i16x16 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i16x16 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i16x32(self, a: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { + let (a0, a1) = self.split_i16x32(a); + self.combine_u8x32( + self.reinterpret_u8_i16x16(a0), + self.reinterpret_u8_i16x16(a1), + ) + } + #[inline(always)] + fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { + let (a0, a1) = self.split_i16x32(a); + self.combine_u32x8( + self.reinterpret_u32_i16x16(a0), + self.reinterpret_u32_i16x16(a1), + ) + } + #[inline(always)] + fn splat_u16x32(self, val: u16) -> u16x32 { + let half = self.splat_u16x16(val); + self.combine_u16x16(half, half) + } + #[inline(always)] + fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [u16; 32usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [u16; 32usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { + u16x32 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + if SHIFT >= 32usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_u16x32(a).val.0, + self.cvt_to_bytes_u16x32(b).val.0, + SHIFT * 2usize, + ); + self.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u16x32( + self, + a: u16x32, + b: u16x32, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.slide_within_blocks_u16x16::(a0, b0), + self.slide_within_blocks_u16x16::(a1, b1), + ) + } + #[inline(always)] + fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) + } + #[inline(always)] + fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) + } + #[inline(always)] + fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) + } + #[inline(always)] + fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) + } + #[inline(always)] + fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) + } + #[inline(always)] + fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) + } + #[inline(always)] + fn not_u16x32(self, a: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) + } + #[inline(always)] + fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) + } + #[inline(always)] + fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) + } + #[inline(always)] + fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) + } + #[inline(always)] + fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) + } + #[inline(always)] + fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) + } + #[inline(always)] + fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, _) = self.split_u16x32(a); + let (b0, _) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) + } + #[inline(always)] + fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (_, a1) = self.split_u16x32(a); + let (_, b1) = self.split_u16x32(b); + self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) + } + #[inline(always)] + fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) + } + #[inline(always)] + fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16( + self.unzip_high_u16x16(a0, a1), + self.unzip_high_u16x16(b0, b1), + ) + } + #[inline(always)] + fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_lo = self.zip_low_u16x16(a0, b0); + let lo_hi = self.zip_high_u16x16(a0, b0); + let hi_lo = self.zip_low_u16x16(a1, b1); + let hi_hi = self.zip_high_u16x16(a1, b1); + ( + self.combine_u16x16(lo_lo, lo_hi), + self.combine_u16x16(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + let lo_even = self.unzip_low_u16x16(a0, a1); + let lo_odd = self.unzip_high_u16x16(a0, a1); + let hi_even = self.unzip_low_u16x16(b0, b1); + let hi_odd = self.unzip_high_u16x16(b0, b1); + ( + self.combine_u16x16(lo_even, hi_even), + self.combine_u16x16(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_u16x32(b); + let (c0, c1) = self.split_u16x32(c); + self.combine_u16x16( + self.select_u16x16(a0, b0, c0), + self.select_u16x16(a1, b1, c1), + ) + } + #[inline(always)] + fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) + } + #[inline(always)] + fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + } + #[inline(always)] + fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + ( + u16x16 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + u16x16 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (lower, upper) = self.split_u8x64(a); - let (v0_vec, v1_vec) = self.split_u8x32(lower); - let (v2_vec, v3_vec) = self.split_u8x32(upper); + fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + let combined_lower = self.combine_u16x8(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_u16x8(out2.simd_into(self), out3.simd_into(self)); + self.combine_u16x16(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + let (lower, upper) = self.split_u16x32(a); + let (v0_vec, v1_vec) = self.split_u16x16(lower); + let (v2_vec, v3_vec) = self.split_u16x16(upper); let v0: v128 = v0_vec.into(); let v1: v128 = v1_vec.into(); let v2: v128 = v2_vec.into(); let v3: v128 = v3_vec.into(); - let v02_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); - let v13_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); - let v02_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); - let v13_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); - let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_lower, v13_lower, - ); - let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_lower, v13_lower, - ); - let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_upper, v13_upper, - ); - let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_upper, v13_upper, - ); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { unreachable!() }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); } #[inline(always)] - fn reinterpret_u32_u8x64(self, a: u8x64) -> u32x16 { - let (a0, a1) = self.split_u8x64(a); + fn narrow_u16x32(self, a: u16x32) -> u8x32 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + } + #[inline(always)] + fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { + let (a0, a1) = self.split_u16x32(a); + self.combine_u8x32( + self.reinterpret_u8_u16x16(a0), + self.reinterpret_u8_u16x16(a1), + ) + } + #[inline(always)] + fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { + let (a0, a1) = self.split_u16x32(a); self.combine_u32x8( - self.reinterpret_u32_u8x32(a0), - self.reinterpret_u32_u8x32(a1), + self.reinterpret_u32_u16x16(a0), + self.reinterpret_u32_u16x16(a1), ) } #[inline(always)] - fn splat_mask8x64(self, val: bool) -> mask8x64 { - let half = self.splat_mask8x32(val); - self.combine_mask8x32(half, half) + fn splat_mask16x32(self, val: bool) -> mask16x32 { + let half = self.splat_mask16x16(val); + self.combine_mask16x16(half, half) } #[inline(always)] - fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64 { - mask8x64 { + fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { + mask16x32 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask8x64(self, a: mask8x64) -> [i8; 64usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i8; 64usize]>(&a.val.0) + fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i16; 32usize]>(&a.val.0) } #[inline(always)] - fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64 { - let lo = self.from_bitmask_mask8x32(bits); - let hi = self.from_bitmask_mask8x32(bits >> 32usize); - self.combine_mask8x32(lo, hi) + fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { + let lo = self.from_bitmask_mask16x16(bits); + let hi = self.from_bitmask_mask16x16(bits >> 16usize); + self.combine_mask16x16(lo, hi) } #[inline(always)] - fn to_bitmask_mask8x64(self, a: mask8x64) -> u64 { - let (lo, hi) = self.split_mask8x64(a); - let lo = self.to_bitmask_mask8x32(lo); - let hi = self.to_bitmask_mask8x32(hi); - lo | (hi << 32usize) + fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { + let (lo, hi) = self.split_mask16x32(a); + let lo = self.to_bitmask_mask16x16(lo); + let hi = self.to_bitmask_mask16x16(hi); + lo | (hi << 16usize) } #[inline(always)] - fn and_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1)) + fn set_mask16x32(self, a: &mut mask16x32, index: usize, value: bool) -> () { + assert!( + index < 32usize, + "mask lane index {index} is out of bounds for {} lanes", + 32usize + ); + let mut lanes = self.as_array_mask16x32(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask16x32(lanes); } #[inline(always)] - fn or_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1)) + fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) } #[inline(always)] - fn xor_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1)) + fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) } #[inline(always)] - fn not_mask8x64(self, a: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1)) + fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) } #[inline(always)] - fn select_mask8x64( + fn not_mask16x32(self, a: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + } + #[inline(always)] + fn select_mask16x32( self, - a: mask8x64, - b: mask8x64, - c: mask8x64, - ) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - let (c0, c1) = self.split_mask8x64(c); - self.combine_mask8x32( - self.select_mask8x32(a0, b0, c0), - self.select_mask8x32(a1, b1, c1), + a: mask16x32, + b: mask16x32, + c: mask16x32, + ) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + let (c0, c1) = self.split_mask16x32(c); + self.combine_mask16x16( + self.select_mask16x16(a0, b0, c0), + self.select_mask16x16(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask8x64(self, a: mask8x64, b: mask8x64) -> mask8x64 { - let (a0, a1) = self.split_mask8x64(a); - let (b0, b1) = self.split_mask8x64(b); - self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1)) + fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { + let (a0, a1) = self.split_mask16x32(a); + let (b0, b1) = self.split_mask16x32(b); + self.combine_mask16x16( + self.simd_eq_mask16x16(a0, b0), + self.simd_eq_mask16x16(a1, b1), + ) } #[inline(always)] - fn any_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1) + fn any_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) } #[inline(always)] - fn all_true_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1) + fn all_true_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) } #[inline(always)] - fn any_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1) + fn any_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) } #[inline(always)] - fn all_false_mask8x64(self, a: mask8x64) -> bool { - let (a0, a1) = self.split_mask8x64(a); - self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1) + fn all_false_mask16x32(self, a: mask16x32) -> bool { + let (a0, a1) = self.split_mask16x32(a); + self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) } #[inline(always)] - fn split_mask8x64(self, a: mask8x64) -> (mask8x32, mask8x32) { + fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { ( - mask8x32 { + mask16x16 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - mask8x32 { + mask16x16 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn splat_i16x32(self, val: i16) -> i16x32 { - let half = self.splat_i16x16(val); - self.combine_i16x16(half, half) + fn splat_i32x16(self, val: i32) -> i32x16 { + let half = self.splat_i32x8(val); + self.combine_i32x8(half, half) } #[inline(always)] - fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32 { - i16x32 { + fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32 { - i16x32 { + fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i16x32(self, a: i16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i16; 32usize]>(&a.val.0) + fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i16x32(self, a: &i16x32) -> &[i16; 32usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [i16; 32usize]>(&a.val.0) + fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [i32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i16x32(self, a: &mut i16x32) -> &mut [i16; 32usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [i16; 32usize]>(&mut a.val.0) + fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [i32; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i16x32(self, a: u8x64) -> i16x32 { - i16x32 { + fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { + i32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i16x32(self, a: i16x32) -> u8x64 { + fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - if SHIFT >= 32usize { + fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + if SHIFT >= 16usize { return b; } let result = cross_block_slide_128x4( - self.cvt_to_bytes_i16x32(a).val.0, - self.cvt_to_bytes_i16x32(b).val.0, - SHIFT * 2usize, + self.cvt_to_bytes_i32x16(a).val.0, + self.cvt_to_bytes_i32x16(b).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_i16x32(u8x64 { + self.cvt_from_bytes_i32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i16x32( + fn slide_within_blocks_i32x16( self, - a: i16x32, - b: i16x32, - ) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.slide_within_blocks_i16x16::(a0, b0), - self.slide_within_blocks_i16x16::(a1, b1), + a: i32x16, + b: i32x16, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8( + self.slide_within_blocks_i32x8::(a0, b0), + self.slide_within_blocks_i32x8::(a1, b1), ) } #[inline(always)] - fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) } #[inline(always)] - fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) } #[inline(always)] - fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) } #[inline(always)] - fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) + fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) } #[inline(always)] - fn or_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1)) + fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) } #[inline(always)] - fn xor_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1)) + fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) } #[inline(always)] - fn not_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1)) + fn not_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) } #[inline(always)] - fn shl_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift)) + fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) } #[inline(always)] - fn shlv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1)) + fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) } #[inline(always)] - fn shr_i16x32(self, a: i16x32, shift: u32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift)) + fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) } #[inline(always)] - fn shrv_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1)) + fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) } #[inline(always)] - fn simd_eq_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1)) + fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) } #[inline(always)] - fn simd_lt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1)) + fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) } #[inline(always)] - fn simd_le_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1)) + fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) } #[inline(always)] - fn simd_ge_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1)) + fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) } #[inline(always)] - fn simd_gt_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1)) + fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) } #[inline(always)] - fn zip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, _) = self.split_i16x32(a); - let (b0, _) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0)) + fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, _) = self.split_i32x16(a); + let (b0, _) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) } #[inline(always)] - fn zip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (_, a1) = self.split_i16x32(a); - let (_, b1) = self.split_i16x32(b); - self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1)) + fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (_, a1) = self.split_i32x16(a); + let (_, b1) = self.split_i32x16(b); + self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) } #[inline(always)] - fn unzip_low_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1)) + fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) } #[inline(always)] - fn unzip_high_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16( - self.unzip_high_i16x16(a0, a1), - self.unzip_high_i16x16(b0, b1), - ) + fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) } #[inline(always)] - fn interleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_lo = self.zip_low_i16x16(a0, b0); - let lo_hi = self.zip_high_i16x16(a0, b0); - let hi_lo = self.zip_low_i16x16(a1, b1); - let hi_hi = self.zip_high_i16x16(a1, b1); + fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_lo = self.zip_low_i32x8(a0, b0); + let lo_hi = self.zip_high_i32x8(a0, b0); + let hi_lo = self.zip_low_i32x8(a1, b1); + let hi_hi = self.zip_high_i32x8(a1, b1); ( - self.combine_i16x16(lo_lo, lo_hi), - self.combine_i16x16(hi_lo, hi_hi), + self.combine_i32x8(lo_lo, lo_hi), + self.combine_i32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_i16x32(self, a: i16x32, b: i16x32) -> (i16x32, i16x32) { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - let lo_even = self.unzip_low_i16x16(a0, a1); - let lo_odd = self.unzip_high_i16x16(a0, a1); - let hi_even = self.unzip_low_i16x16(b0, b1); - let hi_odd = self.unzip_high_i16x16(b0, b1); + fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + let lo_even = self.unzip_low_i32x8(a0, a1); + let lo_odd = self.unzip_high_i32x8(a0, a1); + let hi_even = self.unzip_low_i32x8(b0, b1); + let hi_odd = self.unzip_high_i32x8(b0, b1); ( - self.combine_i16x16(lo_even, hi_even), - self.combine_i16x16(lo_odd, hi_odd), - ) - } - #[inline(always)] - fn select_i16x32(self, a: mask16x32, b: i16x32, c: i16x32) -> i16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_i16x32(b); - let (c0, c1) = self.split_i16x32(c); - self.combine_i16x16( - self.select_i16x16(a0, b0, c0), - self.select_i16x16(a1, b1, c1), + self.combine_i32x8(lo_even, hi_even), + self.combine_i32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn min_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1)) + fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_i32x16(b); + let (c0, c1) = self.split_i32x16(c); + self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) } #[inline(always)] - fn max_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1)) + fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) } #[inline(always)] - fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { + fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (b0, b1) = self.split_i32x16(b); + self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + } + #[inline(always)] + fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( - i16x16 { + i32x8 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - i16x16 { + i32x8 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn neg_i16x32(self, a: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) + fn neg_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn reinterpret_u8_i16x32(self, a: i16x32) -> u8x64 { - let (a0, a1) = self.split_i16x32(a); - self.combine_u8x32( - self.reinterpret_u8_i16x16(a0), - self.reinterpret_u8_i16x16(a1), - ) + fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { + let (a0, a1) = self.split_i32x16(a); + self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) } #[inline(always)] - fn reinterpret_u32_i16x32(self, a: i16x32) -> u32x16 { - let (a0, a1) = self.split_i16x32(a); + fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { + let (a0, a1) = self.split_i32x16(a); self.combine_u32x8( - self.reinterpret_u32_i16x16(a0), - self.reinterpret_u32_i16x16(a1), + self.reinterpret_u32_i32x8(a0), + self.reinterpret_u32_i32x8(a1), ) } #[inline(always)] - fn splat_u16x32(self, val: u16) -> u16x32 { - let half = self.splat_u16x16(val); - self.combine_u16x16(half, half) + fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) } #[inline(always)] - fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32 { - u16x32 { + fn splat_u32x16(self, val: u32) -> u32x16 { + let half = self.splat_u32x8(val); + self.combine_u32x8(half, half) + } + #[inline(always)] + fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32 { - u16x32 { + fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_u16x32(self, a: u16x32) -> [u16; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [u16; 32usize]>(&a.val.0) + fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_u16x32(self, a: &u16x32) -> &[u16; 32usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [u16; 32usize]>(&a.val.0) + fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [u32; 16usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_u16x32(self, a: &mut u16x32) -> &mut [u16; 32usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [u16; 32usize]>(&mut a.val.0) + fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [u32; 16usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { + fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_u16x32(self, a: u8x64) -> u16x32 { - u16x32 { + fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { + u32x16 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_u16x32(self, a: u16x32) -> u8x64 { + fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - if SHIFT >= 32usize { + fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + if SHIFT >= 16usize { return b; } let result = cross_block_slide_128x4( - self.cvt_to_bytes_u16x32(a).val.0, - self.cvt_to_bytes_u16x32(b).val.0, - SHIFT * 2usize, + self.cvt_to_bytes_u32x16(a).val.0, + self.cvt_to_bytes_u32x16(b).val.0, + SHIFT * 4usize, ); - self.cvt_from_bytes_u16x32(u8x64 { + self.cvt_from_bytes_u32x16(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_u16x32( + fn slide_within_blocks_u32x16( self, - a: u16x32, - b: u16x32, - ) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.slide_within_blocks_u16x16::(a0, b0), - self.slide_within_blocks_u16x16::(a1, b1), + a: u32x16, + b: u32x16, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8( + self.slide_within_blocks_u32x8::(a0, b0), + self.slide_within_blocks_u32x8::(a1, b1), ) } #[inline(always)] - fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1)) - } - #[inline(always)] - fn sub_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1)) + fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) } #[inline(always)] - fn mul_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1)) + fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) } #[inline(always)] - fn and_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1)) + fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) } #[inline(always)] - fn or_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1)) + fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) } #[inline(always)] - fn xor_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1)) + fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) } #[inline(always)] - fn not_u16x32(self, a: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1)) + fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) } #[inline(always)] - fn shl_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift)) + fn not_u32x16(self, a: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) } #[inline(always)] - fn shlv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1)) + fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) } #[inline(always)] - fn shr_u16x32(self, a: u16x32, shift: u32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift)) + fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) } #[inline(always)] - fn shrv_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1)) + fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) } #[inline(always)] - fn simd_eq_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1)) + fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) } #[inline(always)] - fn simd_lt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1)) + fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) } #[inline(always)] - fn simd_le_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1)) + fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) } #[inline(always)] - fn simd_ge_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1)) + fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) } #[inline(always)] - fn simd_gt_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1)) + fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) } #[inline(always)] - fn zip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, _) = self.split_u16x32(a); - let (b0, _) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0)) + fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) } #[inline(always)] - fn zip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (_, a1) = self.split_u16x32(a); - let (_, b1) = self.split_u16x32(b); - self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1)) + fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, _) = self.split_u32x16(a); + let (b0, _) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) } #[inline(always)] - fn unzip_low_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1)) + fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (_, a1) = self.split_u32x16(a); + let (_, b1) = self.split_u32x16(b); + self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) } #[inline(always)] - fn unzip_high_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16( - self.unzip_high_u16x16(a0, a1), - self.unzip_high_u16x16(b0, b1), - ) + fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) } #[inline(always)] - fn interleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_lo = self.zip_low_u16x16(a0, b0); - let lo_hi = self.zip_high_u16x16(a0, b0); - let hi_lo = self.zip_low_u16x16(a1, b1); - let hi_hi = self.zip_high_u16x16(a1, b1); + fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + } + #[inline(always)] + fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_lo = self.zip_low_u32x8(a0, b0); + let lo_hi = self.zip_high_u32x8(a0, b0); + let hi_lo = self.zip_low_u32x8(a1, b1); + let hi_hi = self.zip_high_u32x8(a1, b1); ( - self.combine_u16x16(lo_lo, lo_hi), - self.combine_u16x16(hi_lo, hi_hi), + self.combine_u32x8(lo_lo, lo_hi), + self.combine_u32x8(hi_lo, hi_hi), ) } #[inline(always)] - fn deinterleave_u16x32(self, a: u16x32, b: u16x32) -> (u16x32, u16x32) { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - let lo_even = self.unzip_low_u16x16(a0, a1); - let lo_odd = self.unzip_high_u16x16(a0, a1); - let hi_even = self.unzip_low_u16x16(b0, b1); - let hi_odd = self.unzip_high_u16x16(b0, b1); + fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + let lo_even = self.unzip_low_u32x8(a0, a1); + let lo_odd = self.unzip_high_u32x8(a0, a1); + let hi_even = self.unzip_low_u32x8(b0, b1); + let hi_odd = self.unzip_high_u32x8(b0, b1); ( - self.combine_u16x16(lo_even, hi_even), - self.combine_u16x16(lo_odd, hi_odd), + self.combine_u32x8(lo_even, hi_even), + self.combine_u32x8(lo_odd, hi_odd), ) } #[inline(always)] - fn select_u16x32(self, a: mask16x32, b: u16x32, c: u16x32) -> u16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_u16x32(b); - let (c0, c1) = self.split_u16x32(c); - self.combine_u16x16( - self.select_u16x16(a0, b0, c0), - self.select_u16x16(a1, b1, c1), - ) + fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_u32x16(b); + let (c0, c1) = self.split_u32x16(c); + self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) } #[inline(always)] - fn min_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) + fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) } #[inline(always)] - fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (b0, b1) = self.split_u32x16(b); + self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) } #[inline(always)] - fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( - u16x16 { + u32x8 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - u16x16 { + u32x8 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { + fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { unreachable!() }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); - let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); - let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); - let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); - let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); - let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); - let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); - let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); - let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); - let combined_lower = self.combine_u16x8(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u16x8(out2.simd_into(self), out3.simd_into(self)); - self.combine_u16x16(combined_lower, combined_upper) + let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + let combined_lower = self.combine_u32x4(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_u32x4(out2.simd_into(self), out3.simd_into(self)); + self.combine_u32x8(combined_lower, combined_upper) } #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (lower, upper) = self.split_u16x32(a); - let (v0_vec, v1_vec) = self.split_u16x16(lower); - let (v2_vec, v3_vec) = self.split_u16x16(upper); + fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { + let (lower, upper) = self.split_u32x16(a); + let (v0_vec, v1_vec) = self.split_u32x8(lower); + let (v2_vec, v3_vec) = self.split_u32x8(upper); let v0: v128 = v0_vec.into(); let v1: v128 = v1_vec.into(); let v2: v128 = v2_vec.into(); let v3: v128 = v3_vec.into(); - let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); - let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); - let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); - let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); - let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); - let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); - let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); - let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { unreachable!() }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] - fn narrow_u16x32(self, a: u16x32) -> u8x32 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); } #[inline(always)] - fn reinterpret_u8_u16x32(self, a: u16x32) -> u8x64 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u8x32( - self.reinterpret_u8_u16x16(a0), - self.reinterpret_u8_u16x16(a1), - ) + fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { + let (a0, a1) = self.split_u32x16(a); + self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) } #[inline(always)] - fn reinterpret_u32_u16x32(self, a: u16x32) -> u32x16 { - let (a0, a1) = self.split_u16x32(a); - self.combine_u32x8( - self.reinterpret_u32_u16x16(a0), - self.reinterpret_u32_u16x16(a1), - ) + fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { + let (a0, a1) = self.split_u32x16(a); + self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) } #[inline(always)] - fn splat_mask16x32(self, val: bool) -> mask16x32 { - let half = self.splat_mask16x16(val); - self.combine_mask16x16(half, half) + fn splat_mask32x16(self, val: bool) -> mask32x16 { + let half = self.splat_mask32x8(val); + self.combine_mask32x8(half, half) } #[inline(always)] - fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32 { - mask16x32 { + fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { + mask32x16 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn as_array_mask16x32(self, a: mask16x32) -> [i16; 32usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i16; 32usize]>(&a.val.0) + fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i32; 16usize]>(&a.val.0) } #[inline(always)] - fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32 { - let lo = self.from_bitmask_mask16x16(bits); - let hi = self.from_bitmask_mask16x16(bits >> 16usize); - self.combine_mask16x16(lo, hi) + fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { + let lo = self.from_bitmask_mask32x8(bits); + let hi = self.from_bitmask_mask32x8(bits >> 8usize); + self.combine_mask32x8(lo, hi) } #[inline(always)] - fn to_bitmask_mask16x32(self, a: mask16x32) -> u64 { - let (lo, hi) = self.split_mask16x32(a); - let lo = self.to_bitmask_mask16x16(lo); - let hi = self.to_bitmask_mask16x16(hi); - lo | (hi << 16usize) + fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { + let (lo, hi) = self.split_mask32x16(a); + let lo = self.to_bitmask_mask32x8(lo); + let hi = self.to_bitmask_mask32x8(hi); + lo | (hi << 8usize) } #[inline(always)] - fn and_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1)) + fn set_mask32x16(self, a: &mut mask32x16, index: usize, value: bool) -> () { + assert!( + index < 16usize, + "mask lane index {index} is out of bounds for {} lanes", + 16usize + ); + let mut lanes = self.as_array_mask32x16(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask32x16(lanes); } #[inline(always)] - fn or_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1)) + fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) } #[inline(always)] - fn xor_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1)) + fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) } #[inline(always)] - fn not_mask16x32(self, a: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1)) + fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) } #[inline(always)] - fn select_mask16x32( + fn not_mask32x16(self, a: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + } + #[inline(always)] + fn select_mask32x16( self, - a: mask16x32, - b: mask16x32, - c: mask16x32, - ) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - let (c0, c1) = self.split_mask16x32(c); - self.combine_mask16x16( - self.select_mask16x16(a0, b0, c0), - self.select_mask16x16(a1, b1, c1), + a: mask32x16, + b: mask32x16, + c: mask32x16, + ) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + let (c0, c1) = self.split_mask32x16(c); + self.combine_mask32x8( + self.select_mask32x8(a0, b0, c0), + self.select_mask32x8(a1, b1, c1), ) } #[inline(always)] - fn simd_eq_mask16x32(self, a: mask16x32, b: mask16x32) -> mask16x32 { - let (a0, a1) = self.split_mask16x32(a); - let (b0, b1) = self.split_mask16x32(b); - self.combine_mask16x16( - self.simd_eq_mask16x16(a0, b0), - self.simd_eq_mask16x16(a1, b1), - ) + fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_mask32x16(b); + self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) } #[inline(always)] - fn any_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1) + fn any_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) } #[inline(always)] - fn all_true_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1) + fn all_true_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) } #[inline(always)] - fn any_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1) + fn any_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) } #[inline(always)] - fn all_false_mask16x32(self, a: mask16x32) -> bool { - let (a0, a1) = self.split_mask16x32(a); - self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1) + fn all_false_mask32x16(self, a: mask32x16) -> bool { + let (a0, a1) = self.split_mask32x16(a); + self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) } #[inline(always)] - fn split_mask16x32(self, a: mask16x32) -> (mask16x16, mask16x16) { + fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { ( - mask16x16 { + mask32x8 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - mask16x16 { + mask32x8 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn splat_i32x16(self, val: i32) -> i32x16 { - let half = self.splat_i32x8(val); - self.combine_i32x8(half, half) + fn splat_f64x8(self, val: f64) -> f64x8 { + let half = self.splat_f64x4(val); + self.combine_f64x4(half, half) } #[inline(always)] - fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16 { - i32x16 { + fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16 { - i32x16 { + fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_i32x16(self, a: i32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i32; 16usize]>(&a.val.0) + fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_i32x16(self, a: &i32x16) -> &[i32; 16usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [i32; 16usize]>(&a.val.0) + fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [f64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_i32x16(self, a: &mut i32x16) -> &mut [i32; 16usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [i32; 16usize]>(&mut a.val.0) + fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [f64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_i32x16(self, a: u8x64) -> i32x16 { - i32x16 { + fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { + f64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_i32x16(self, a: i32x16) -> u8x64 { + fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - if SHIFT >= 16usize { + fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + if SHIFT >= 8usize { return b; } let result = cross_block_slide_128x4( - self.cvt_to_bytes_i32x16(a).val.0, - self.cvt_to_bytes_i32x16(b).val.0, - SHIFT * 4usize, + self.cvt_to_bytes_f64x8(a).val.0, + self.cvt_to_bytes_f64x8(b).val.0, + SHIFT * 8usize, ); - self.cvt_from_bytes_i32x16(u8x64 { + self.cvt_from_bytes_f64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_i32x16( + fn slide_within_blocks_f64x8( self, - a: i32x16, - b: i32x16, - ) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8( - self.slide_within_blocks_i32x8::(a0, b0), - self.slide_within_blocks_i32x8::(a1, b1), + a: f64x8, + b: f64x8, + ) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.slide_within_blocks_f64x4::(a0, b0), + self.slide_within_blocks_f64x4::(a1, b1), ) } #[inline(always)] - fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1)) - } - #[inline(always)] - fn sub_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1)) - } - #[inline(always)] - fn mul_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1)) - } - #[inline(always)] - fn and_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1)) - } - #[inline(always)] - fn or_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1)) - } - #[inline(always)] - fn xor_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1)) - } - #[inline(always)] - fn not_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1)) - } - #[inline(always)] - fn shl_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift)) - } - #[inline(always)] - fn shlv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1)) - } - #[inline(always)] - fn shr_i32x16(self, a: i32x16, shift: u32) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift)) - } - #[inline(always)] - fn shrv_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1)) - } - #[inline(always)] - fn simd_le_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1)) - } - #[inline(always)] - fn simd_ge_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1)) - } - #[inline(always)] - fn simd_gt_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1)) - } - #[inline(always)] - fn zip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, _) = self.split_i32x16(a); - let (b0, _) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0)) - } - #[inline(always)] - fn zip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (_, a1) = self.split_i32x16(a); - let (_, b1) = self.split_i32x16(b); - self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1)) + fn abs_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) } #[inline(always)] - fn unzip_low_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1)) + fn neg_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) } #[inline(always)] - fn unzip_high_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1)) + fn sqrt_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) } #[inline(always)] - fn interleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_lo = self.zip_low_i32x8(a0, b0); - let lo_hi = self.zip_high_i32x8(a0, b0); - let hi_lo = self.zip_low_i32x8(a1, b1); - let hi_hi = self.zip_high_i32x8(a1, b1); - ( - self.combine_i32x8(lo_lo, lo_hi), - self.combine_i32x8(hi_lo, hi_hi), + fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.approximate_recip_f64x4(a0), + self.approximate_recip_f64x4(a1), ) } #[inline(always)] - fn deinterleave_i32x16(self, a: i32x16, b: i32x16) -> (i32x16, i32x16) { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - let lo_even = self.unzip_low_i32x8(a0, a1); - let lo_odd = self.unzip_high_i32x8(a0, a1); - let hi_even = self.unzip_low_i32x8(b0, b1); - let hi_odd = self.unzip_high_i32x8(b0, b1); - ( - self.combine_i32x8(lo_even, hi_even), - self.combine_i32x8(lo_odd, hi_odd), - ) + fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) } #[inline(always)] - fn select_i32x16(self, a: mask32x16, b: i32x16, c: i32x16) -> i32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_i32x16(b); - let (c0, c1) = self.split_i32x16(c); - self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1)) + fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) } #[inline(always)] - fn min_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1)) + fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) } #[inline(always)] - fn max_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - let (b0, b1) = self.split_i32x16(b); - self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1)) + fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) } #[inline(always)] - fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { - ( - i32x8 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - i32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) + fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) } #[inline(always)] - fn neg_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) + fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) } #[inline(always)] - fn reinterpret_u8_i32x16(self, a: i32x16) -> u8x64 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1)) + fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) } #[inline(always)] - fn reinterpret_u32_i32x16(self, a: i32x16) -> u32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_u32x8( - self.reinterpret_u32_i32x8(a0), - self.reinterpret_u32_i32x8(a1), - ) + fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) } #[inline(always)] - fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) + fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) } #[inline(always)] - fn splat_u32x16(self, val: u32) -> u32x16 { - let half = self.splat_u32x8(val); - self.combine_u32x8(half, half) + fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) } #[inline(always)] - fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16 { - u32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, _) = self.split_f64x8(a); + let (b0, _) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) } #[inline(always)] - fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16 { - u32x16 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (_, a1) = self.split_f64x8(a); + let (_, b1) = self.split_f64x8(b); + self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) } #[inline(always)] - fn as_array_u32x16(self, a: u32x16) -> [u32; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [u32; 16usize]>(&a.val.0) + fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) } #[inline(always)] - fn as_array_ref_u32x16(self, a: &u32x16) -> &[u32; 16usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [u32; 16usize]>(&a.val.0) + fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) } #[inline(always)] - fn as_array_mut_u32x16(self, a: &mut u32x16) -> &mut [u32; 16usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [u32; 16usize]>(&mut a.val.0) + fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_lo = self.zip_low_f64x4(a0, b0); + let lo_hi = self.zip_high_f64x4(a0, b0); + let hi_lo = self.zip_low_f64x4(a1, b1); + let hi_hi = self.zip_high_f64x4(a1, b1); + ( + self.combine_f64x4(lo_lo, lo_hi), + self.combine_f64x4(hi_lo, hi_hi), + ) } #[inline(always)] - fn store_array_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); + fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let lo_even = self.unzip_low_f64x4(a0, a1); + let lo_odd = self.unzip_high_f64x4(a0, a1); + let hi_even = self.unzip_low_f64x4(b0, b1); + let hi_odd = self.unzip_high_f64x4(b0, b1); + ( + self.combine_f64x4(lo_even, hi_even), + self.combine_f64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn cvt_from_bytes_u32x16(self, a: u8x64) -> u32x16 { - u32x16 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) } #[inline(always)] - fn cvt_to_bytes_u32x16(self, a: u32x16) -> u8x64 { - u8x64 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } + fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) } #[inline(always)] - fn slide_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - if SHIFT >= 16usize { - return b; - } - let result = cross_block_slide_128x4( - self.cvt_to_bytes_u32x16(a).val.0, - self.cvt_to_bytes_u32x16(b).val.0, - SHIFT * 4usize, - ); - self.cvt_from_bytes_u32x16(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) + fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.max_precise_f64x4(a0, b0), + self.max_precise_f64x4(a1, b1), + ) } #[inline(always)] - fn slide_within_blocks_u32x16( - self, - a: u32x16, - b: u32x16, - ) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8( - self.slide_within_blocks_u32x8::(a0, b0), - self.slide_within_blocks_u32x8::(a1, b1), + fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + self.combine_f64x4( + self.min_precise_f64x4(a0, b0), + self.min_precise_f64x4(a1, b1), ) } #[inline(always)] - fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1)) + fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_add_f64x4(a0, b0, c0), + self.mul_add_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn sub_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1)) + fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4( + self.mul_sub_f64x4(a0, b0, c0), + self.mul_sub_f64x4(a1, b1, c1), + ) } #[inline(always)] - fn mul_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1)) + fn floor_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) } #[inline(always)] - fn and_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1)) + fn ceil_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) } #[inline(always)] - fn or_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1)) + fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4( + self.round_ties_even_f64x4(a0), + self.round_ties_even_f64x4(a1), + ) } #[inline(always)] - fn xor_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1)) + fn fract_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) } #[inline(always)] - fn not_u32x16(self, a: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1)) + fn trunc_f64x8(self, a: f64x8) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) } #[inline(always)] - fn shl_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift)) + fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_f64x8(b); + let (c0, c1) = self.split_f64x8(c); + self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) } #[inline(always)] - fn shlv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1)) + fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + ( + f64x4 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + f64x4 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] - fn shr_u32x16(self, a: u32x16, shift: u32) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift)) + fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { + let (a0, a1) = self.split_f64x8(a); + self.combine_f32x8( + self.reinterpret_f32_f64x4(a0), + self.reinterpret_f32_f64x4(a1), + ) } #[inline(always)] - fn shrv_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1)) + fn splat_i64x8(self, val: i64) -> i64x8 { + let half = self.splat_i64x4(val); + self.combine_i64x4(half, half) } #[inline(always)] - fn simd_eq_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1)) + fn load_array_i64x8(self, val: [i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } } #[inline(always)] - fn simd_lt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1)) + fn load_array_ref_i64x8(self, val: &[i64; 8usize]) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] - fn simd_le_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1)) + fn as_array_i64x8(self, a: i64x8) -> [i64; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_ge_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1)) + fn as_array_ref_i64x8(self, a: &i64x8) -> &[i64; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [i64; 8usize]>(&a.val.0) } #[inline(always)] - fn simd_gt_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1)) + fn as_array_mut_i64x8(self, a: &mut i64x8) -> &mut [i64; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [i64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn zip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, _) = self.split_u32x16(a); - let (b0, _) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0)) + fn store_array_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn zip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (_, a1) = self.split_u32x16(a); - let (_, b1) = self.split_u32x16(b); - self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1)) + fn cvt_from_bytes_i64x8(self, a: u8x64) -> i64x8 { + i64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn unzip_low_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1)) + fn cvt_to_bytes_i64x8(self, a: i64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } } #[inline(always)] - fn unzip_high_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1)) + fn slide_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_slide_128x4( + self.cvt_to_bytes_i64x8(a).val.0, + self.cvt_to_bytes_i64x8(b).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_i64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) } #[inline(always)] - fn interleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_lo = self.zip_low_u32x8(a0, b0); - let lo_hi = self.zip_high_u32x8(a0, b0); - let hi_lo = self.zip_low_u32x8(a1, b1); - let hi_hi = self.zip_high_u32x8(a1, b1); - ( - self.combine_u32x8(lo_lo, lo_hi), - self.combine_u32x8(hi_lo, hi_hi), + fn slide_within_blocks_i64x8( + self, + a: i64x8, + b: i64x8, + ) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4( + self.slide_within_blocks_i64x4::(a0, b0), + self.slide_within_blocks_i64x4::(a1, b1), ) } #[inline(always)] - fn deinterleave_u32x16(self, a: u32x16, b: u32x16) -> (u32x16, u32x16) { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - let lo_even = self.unzip_low_u32x8(a0, a1); - let lo_odd = self.unzip_high_u32x8(a0, a1); - let hi_even = self.unzip_low_u32x8(b0, b1); - let hi_odd = self.unzip_high_u32x8(b0, b1); - ( - self.combine_u32x8(lo_even, hi_even), - self.combine_u32x8(lo_odd, hi_odd), - ) + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) } #[inline(always)] - fn select_u32x16(self, a: mask32x16, b: u32x16, c: u32x16) -> u32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_u32x16(b); - let (c0, c1) = self.split_u32x16(c); - self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1)) + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) } #[inline(always)] - fn min_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1)) + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) } #[inline(always)] - fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { - let (a0, a1) = self.split_u32x16(a); - let (b0, b1) = self.split_u32x16(b); - self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1)) + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) } #[inline(always)] - fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { - ( - u32x8 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, - }, - u32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_u32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_u32x8(combined_lower, combined_upper) + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) } #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (lower, upper) = self.split_u32x16(a); - let (v0_vec, v1_vec) = self.split_u32x8(lower); - let (v2_vec, v3_vec) = self.split_u32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) } #[inline(always)] - fn reinterpret_u8_u32x16(self, a: u32x16) -> u8x64 { - let (a0, a1) = self.split_u32x16(a); - self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1)) + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) } #[inline(always)] - fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { - let (a0, a1) = self.split_u32x16(a); - self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) } #[inline(always)] - fn splat_mask32x16(self, val: bool) -> mask32x16 { - let half = self.splat_mask32x8(val); - self.combine_mask32x8(half, half) + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) } #[inline(always)] - fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16 { - mask32x16 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) } #[inline(always)] - fn as_array_mask32x16(self, a: mask32x16) -> [i32; 16usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [i32; 16usize]>(&a.val.0) + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) } #[inline(always)] - fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16 { - let lo = self.from_bitmask_mask32x8(bits); - let hi = self.from_bitmask_mask32x8(bits >> 8usize); - self.combine_mask32x8(lo, hi) + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) } #[inline(always)] - fn to_bitmask_mask32x16(self, a: mask32x16) -> u64 { - let (lo, hi) = self.split_mask32x16(a); - let lo = self.to_bitmask_mask32x8(lo); - let hi = self.to_bitmask_mask32x8(hi); - lo | (hi << 8usize) + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) } #[inline(always)] - fn and_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1)) + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) } #[inline(always)] - fn or_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1)) + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) } #[inline(always)] - fn xor_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1)) + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) } #[inline(always)] - fn not_mask32x16(self, a: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1)) + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) } #[inline(always)] - fn select_mask32x16( - self, - a: mask32x16, - b: mask32x16, - c: mask32x16, - ) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - let (c0, c1) = self.split_mask32x16(c); - self.combine_mask32x8( - self.select_mask32x8(a0, b0, c0), - self.select_mask32x8(a1, b1, c1), - ) + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) } #[inline(always)] - fn simd_eq_mask32x16(self, a: mask32x16, b: mask32x16) -> mask32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_mask32x16(b); - self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1)) + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) } #[inline(always)] - fn any_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1) + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) } #[inline(always)] - fn all_true_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1) + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) } #[inline(always)] - fn any_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1) + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) } #[inline(always)] - fn all_false_mask32x16(self, a: mask32x16) -> bool { - let (a0, a1) = self.split_mask32x16(a); - self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1) + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) } #[inline(always)] - fn split_mask32x16(self, a: mask32x16) -> (mask32x8, mask32x8) { + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( - mask32x8 { + i64x4 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - mask32x8 { + i64x4 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn splat_f64x8(self, val: f64) -> f64x8 { - let half = self.splat_f64x4(val); - self.combine_f64x4(half, half) + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8 { - f64x8 { + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), + ) + } + #[inline(always)] + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(&val), simd: self, } } #[inline(always)] - fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8 { - f64x8 { + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(val), simd: self, } } #[inline(always)] - fn as_array_f64x8(self, a: f64x8) -> [f64; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [f64; 8usize]>(&a.val.0) + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [u64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_ref_f64x8(self, a: &f64x8) -> &[f64; 8usize] { - crate::transmute::checked_cast_ref::<[v128; 4usize], [f64; 8usize]>(&a.val.0) + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::<[v128; 4usize], [u64; 8usize]>(&a.val.0) } #[inline(always)] - fn as_array_mut_f64x8(self, a: &mut f64x8) -> &mut [f64; 8usize] { - crate::transmute::checked_cast_mut::<[v128; 4usize], [f64; 8usize]>(&mut a.val.0) + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::<[v128; 4usize], [u64; 8usize]>(&mut a.val.0) } #[inline(always)] - fn store_array_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); } #[inline(always)] - fn cvt_from_bytes_f64x8(self, a: u8x64) -> f64x8 { - f64x8 { + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn cvt_to_bytes_f64x8(self, a: f64x8) -> u8x64 { + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { u8x64 { val: crate::transmute::checked_transmute_copy(&a.val), simd: self, } } #[inline(always)] - fn slide_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { if SHIFT >= 8usize { return b; } let result = cross_block_slide_128x4( - self.cvt_to_bytes_f64x8(a).val.0, - self.cvt_to_bytes_f64x8(b).val.0, + self.cvt_to_bytes_u64x8(a).val.0, + self.cvt_to_bytes_u64x8(b).val.0, SHIFT * 8usize, ); - self.cvt_from_bytes_f64x8(u8x64 { + self.cvt_from_bytes_u64x8(u8x64 { val: crate::support::Aligned512(result), simd: self, }) } #[inline(always)] - fn slide_within_blocks_f64x8( + fn slide_within_blocks_u64x8( self, - a: f64x8, - b: f64x8, - ) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.slide_within_blocks_f64x4::(a0, b0), - self.slide_within_blocks_f64x4::(a1, b1), - ) - } - #[inline(always)] - fn abs_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) - } - #[inline(always)] - fn neg_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1)) - } - #[inline(always)] - fn sqrt_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1)) - } - #[inline(always)] - fn approximate_recip_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.approximate_recip_f64x4(a0), - self.approximate_recip_f64x4(a1), + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), ) } #[inline(always)] - fn add_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1)) - } - #[inline(always)] - fn sub_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1)) + fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.add_u64x4(a0, b0), self.add_u64x4(a1, b1)) } #[inline(always)] - fn mul_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1)) + fn sub_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.sub_u64x4(a0, b0), self.sub_u64x4(a1, b1)) } #[inline(always)] - fn div_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1)) + fn mul_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1)) } #[inline(always)] - fn copysign_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1)) + fn and_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.and_u64x4(a0, b0), self.and_u64x4(a1, b1)) } #[inline(always)] - fn simd_eq_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1)) + fn or_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.or_u64x4(a0, b0), self.or_u64x4(a1, b1)) } #[inline(always)] - fn simd_lt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1)) + fn xor_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.xor_u64x4(a0, b0), self.xor_u64x4(a1, b1)) } #[inline(always)] - fn simd_le_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1)) + fn not_u64x8(self, a: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.not_u64x4(a0), self.not_u64x4(a1)) } #[inline(always)] - fn simd_ge_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1)) + fn shl_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shl_u64x4(a0, shift), self.shl_u64x4(a1, shift)) } #[inline(always)] - fn simd_gt_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1)) + fn shlv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shlv_u64x4(a0, b0), self.shlv_u64x4(a1, b1)) } #[inline(always)] - fn zip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, _) = self.split_f64x8(a); - let (b0, _) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0)) + fn shr_u64x8(self, a: u64x8, shift: u32) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u64x4(self.shr_u64x4(a0, shift), self.shr_u64x4(a1, shift)) } #[inline(always)] - fn zip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (_, a1) = self.split_f64x8(a); - let (_, b1) = self.split_f64x8(b); - self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1)) + fn shrv_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.shrv_u64x4(a0, b0), self.shrv_u64x4(a1, b1)) } #[inline(always)] - fn unzip_low_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1)) + fn simd_eq_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_eq_u64x4(a0, b0), self.simd_eq_u64x4(a1, b1)) } #[inline(always)] - fn unzip_high_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1)) + fn simd_lt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_lt_u64x4(a0, b0), self.simd_lt_u64x4(a1, b1)) } #[inline(always)] - fn interleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_lo = self.zip_low_f64x4(a0, b0); - let lo_hi = self.zip_high_f64x4(a0, b0); - let hi_lo = self.zip_low_f64x4(a1, b1); - let hi_hi = self.zip_high_f64x4(a1, b1); - ( - self.combine_f64x4(lo_lo, lo_hi), - self.combine_f64x4(hi_lo, hi_hi), - ) + fn simd_le_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_le_u64x4(a0, b0), self.simd_le_u64x4(a1, b1)) } #[inline(always)] - fn deinterleave_f64x8(self, a: f64x8, b: f64x8) -> (f64x8, f64x8) { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let lo_even = self.unzip_low_f64x4(a0, a1); - let lo_odd = self.unzip_high_f64x4(a0, a1); - let hi_even = self.unzip_low_f64x4(b0, b1); - let hi_odd = self.unzip_high_f64x4(b0, b1); - ( - self.combine_f64x4(lo_even, hi_even), - self.combine_f64x4(lo_odd, hi_odd), - ) + fn simd_ge_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_ge_u64x4(a0, b0), self.simd_ge_u64x4(a1, b1)) } #[inline(always)] - fn max_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1)) + fn simd_gt_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_mask64x4(self.simd_gt_u64x4(a0, b0), self.simd_gt_u64x4(a1, b1)) } #[inline(always)] - fn min_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1)) + fn zip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, _) = self.split_u64x8(a); + let (b0, _) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a0, b0), self.zip_high_u64x4(a0, b0)) } #[inline(always)] - fn max_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.max_precise_f64x4(a0, b0), - self.max_precise_f64x4(a1, b1), - ) + fn zip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (_, a1) = self.split_u64x8(a); + let (_, b1) = self.split_u64x8(b); + self.combine_u64x4(self.zip_low_u64x4(a1, b1), self.zip_high_u64x4(a1, b1)) } #[inline(always)] - fn min_precise_f64x8(self, a: f64x8, b: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - self.combine_f64x4( - self.min_precise_f64x4(a0, b0), - self.min_precise_f64x4(a1, b1), - ) + fn unzip_low_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_low_u64x4(a0, a1), self.unzip_low_u64x4(b0, b1)) } #[inline(always)] - fn mul_add_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_add_f64x4(a0, b0, c0), - self.mul_add_f64x4(a1, b1, c1), - ) + fn unzip_high_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.unzip_high_u64x4(a0, a1), self.unzip_high_u64x4(b0, b1)) } #[inline(always)] - fn mul_sub_f64x8(self, a: f64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4( - self.mul_sub_f64x4(a0, b0, c0), - self.mul_sub_f64x4(a1, b1, c1), + fn interleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_lo = self.zip_low_u64x4(a0, b0); + let lo_hi = self.zip_high_u64x4(a0, b0); + let hi_lo = self.zip_low_u64x4(a1, b1); + let hi_hi = self.zip_high_u64x4(a1, b1); + ( + self.combine_u64x4(lo_lo, lo_hi), + self.combine_u64x4(hi_lo, hi_hi), ) } #[inline(always)] - fn floor_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1)) - } - #[inline(always)] - fn ceil_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1)) - } - #[inline(always)] - fn round_ties_even_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4( - self.round_ties_even_f64x4(a0), - self.round_ties_even_f64x4(a1), + fn deinterleave_u64x8(self, a: u64x8, b: u64x8) -> (u64x8, u64x8) { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + let lo_even = self.unzip_low_u64x4(a0, a1); + let lo_odd = self.unzip_high_u64x4(a0, a1); + let hi_even = self.unzip_low_u64x4(b0, b1); + let hi_odd = self.unzip_high_u64x4(b0, b1); + ( + self.combine_u64x4(lo_even, hi_even), + self.combine_u64x4(lo_odd, hi_odd), ) } #[inline(always)] - fn fract_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1)) + fn select_u64x8(self, a: mask64x8, b: u64x8, c: u64x8) -> u64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_u64x8(b); + let (c0, c1) = self.split_u64x8(c); + self.combine_u64x4(self.select_u64x4(a0, b0, c0), self.select_u64x4(a1, b1, c1)) } #[inline(always)] - fn trunc_f64x8(self, a: f64x8) -> f64x8 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1)) + fn min_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.min_u64x4(a0, b0), self.min_u64x4(a1, b1)) } #[inline(always)] - fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_f64x8(b); - let (c0, c1) = self.split_f64x8(c); - self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1)) + fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4(self.max_u64x4(a0, b0), self.max_u64x4(a1, b1)) } #[inline(always)] - fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4) { + fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( - f64x4 { + u64x4 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, }, - f64x4 { + u64x4 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), simd: self, }, ) } #[inline(always)] - fn reinterpret_f32_f64x8(self, a: f64x8) -> f32x16 { - let (a0, a1) = self.split_f64x8(a); - self.combine_f32x8( - self.reinterpret_f32_f64x4(a0), - self.reinterpret_f32_f64x4(a1), + fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + let combined_lower = self.combine_u64x2(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_u64x2(out2.simd_into(self), out3.simd_into(self)); + self.combine_u64x4(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + let (lower, upper) = self.split_u64x8(a); + let (v0_vec, v1_vec) = self.split_u64x4(lower); + let (v2_vec, v3_vec) = self.split_u64x4(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] + fn reinterpret_u8_u64x8(self, a: u64x8) -> u8x64 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u8x32(self.reinterpret_u8_u64x4(a0), self.reinterpret_u8_u64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_u64x8(self, a: u64x8) -> u32x16 { + let (a0, a1) = self.split_u64x8(a); + self.combine_u32x8( + self.reinterpret_u32_u64x4(a0), + self.reinterpret_u32_u64x4(a1), ) } #[inline(always)] @@ -7851,6 +9768,17 @@ impl Simd for WasmSimd128 { lo | (hi << 4usize) } #[inline(always)] + fn set_mask64x8(self, a: &mut mask64x8, index: usize, value: bool) -> () { + assert!( + index < 8usize, + "mask lane index {index} is out of bounds for {} lanes", + 8usize + ); + let mut lanes = self.as_array_mask64x8(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.load_array_mask64x8(lanes); + } + #[inline(always)] fn and_mask64x8(self, a: mask64x8, b: mask64x8) -> mask64x8 { let (a0, a1) = self.split_mask64x8(a); let (b0, b1) = self.split_mask64x8(b); @@ -8093,6 +10021,36 @@ impl From> for v128 { crate::transmute::checked_transmute_copy(&value.val) } } +impl SimdFrom for i64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: v128) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for v128 { + #[inline(always)] + fn from(value: i64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} +impl SimdFrom for u64x2 { + #[inline(always)] + fn simd_from(simd: S, arch: v128) -> Self { + Self { + val: crate::transmute::checked_transmute_copy(&arch), + simd, + } + } +} +impl From> for v128 { + #[inline(always)] + fn from(value: u64x2) -> Self { + crate::transmute::checked_transmute_copy(&value.val) + } +} impl SimdFrom for mask64x2 { #[inline(always)] fn simd_from(simd: S, arch: v128) -> Self { diff --git a/fearless_simd/src/kernel_macros.rs b/fearless_simd/src/kernel_macros.rs index a2ff2c218..169645885 100644 --- a/fearless_simd/src/kernel_macros.rs +++ b/fearless_simd/src/kernel_macros.rs @@ -8,7 +8,7 @@ /// use platform-specific intrinsics for parts of the computation. /// /// The first argument must be a SIMD token written as `token: Neon`, -/// `token: WasmSimd128`, `token: Sse4_2`, or `token: Avx2`. +/// `token: WasmSimd128`, `token: Sse4_2`, `token: Avx2`, or `token: Avx512`. /// /// For levels with runtime-detected target features, the macro runs your body /// inside an inner function annotated with the appropriate `#[target_feature]` @@ -54,7 +54,7 @@ /// However, the body of the function can be as complex as you like. /// /// The SIMD token type must be written as a bare supported name: -/// literally `Neon`, `WasmSimd128`, `Sse4_2`, or `Avx2`. No paths or aliases. +/// literally `Neon`, `WasmSimd128`, `Sse4_2`, `Avx2`, or `Avx512`. No paths or aliases. /// /// For soundness, this macro only accepts safe functions. /// @@ -93,7 +93,7 @@ macro_rules! kernel { ) => { compile_error!(concat!( "fearless_simd::kernel! expects its SIMD token argument type to be written as ", - "one of `Neon`, `WasmSimd128`, `Sse4_2`, or `Avx2`; got `", + "one of `Neon`, `WasmSimd128`, `Sse4_2`, `Avx2`, or `Avx512`; got `", stringify!($token_ty), "`", )); @@ -153,13 +153,27 @@ macro_rules! __fearless_simd_kernel_dispatch { } }; + ( + Avx512, + $($body:tt)* + ) => { + $crate::__fearless_simd_kernel_impl! { + @cfg any(target_arch = "x86", target_arch = "x86_64"); + @token_ty $crate::Avx512; + @kernel_attrs #[target_feature( + enable = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves" + )]; + $($body)* + } + }; + ( $token_ty:ident, $($body:tt)* ) => { compile_error!(concat!( "fearless_simd::kernel! expects its SIMD token argument type to be written as ", - "one of `Neon`, `WasmSimd128`, `Sse4_2`, or `Avx2`; got `", + "one of `Neon`, `WasmSimd128`, `Sse4_2`, `Avx2`, or `Avx512`; got `", stringify!($token_ty), "`", )); @@ -216,9 +230,9 @@ mod tests { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] use core::arch::wasm32::{f32x4_add, v128}; #[cfg(target_arch = "x86")] - use core::arch::x86::{__m256i, _mm256_add_epi32}; + use core::arch::x86::{__m256i, __m512i, _mm256_add_epi32, _mm512_add_epi32}; #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m256i, _mm256_add_epi32}; + use core::arch::x86_64::{__m256i, __m512i, _mm256_add_epi32, _mm512_add_epi32}; crate::kernel!( fn add_f32x4_neon(neon: Neon, a: float32x4_t, b: float32x4_t) -> float32x4_t { @@ -238,6 +252,12 @@ mod tests { } ); + crate::kernel! { + fn add_i32x16_avx512(avx512: Avx512, a: __m512i, b: __m512i) -> __m512i { + _mm512_add_epi32(a, b) + } + } + #[cfg(target_arch = "aarch64")] #[test] fn kernel_instantiates_for_neon() { @@ -291,4 +311,28 @@ mod tests { "`kernel!` should instantiate a working AVX2 kernel" ); } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn kernel_instantiates_for_avx512() { + let Some(avx512) = crate::Level::new().as_avx512() else { + return; + }; + + let a: crate::i32x16<_> = + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].simd_into(avx512); + let b: crate::i32x16<_> = [ + 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, + ] + .simd_into(avx512); + let sum: crate::i32x16<_> = add_i32x16_avx512(avx512, a.into(), b.into()).simd_into(avx512); + + assert_eq!( + <[i32; 16]>::from(sum), + [ + 11, 22, 33, 44, 55, 66, 77, 88, 99, 110, 121, 132, 143, 154, 165, 176 + ], + "`kernel!` should instantiate a working AVX-512 kernel" + ); + } } diff --git a/fearless_simd/src/lib.rs b/fearless_simd/src/lib.rs index 76d34098d..68ca3d0e4 100644 --- a/fearless_simd/src/lib.rs +++ b/fearless_simd/src/lib.rs @@ -114,7 +114,7 @@ //! //! # Instruction set support //! -//! - x86/x86-64: [v2](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (SSE4.2), [v3](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (AVX2) +//! - x86/x86-64: [v2](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (SSE4.2), [v3](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) (AVX2), [Ice Lake](https://en.wikipedia.org/wiki/AVX-512#CPUs_with_AVX-512) (AVX-512, avoiding early slow implementations) //! - Aarch64: Baseline [NEON](https://en.wikipedia.org/wiki/Arm_architecture_family#Advanced_SIMD_(Neon)) //! - WebAssembly: [128-bit packed SIMD](https://github.com/WebAssembly/spec/blob/main/proposals/simd/SIMD.md), [relaxed SIMD](https://github.com/WebAssembly/relaxed-simd/blob/main/proposals/relaxed-simd/Overview.md) //! @@ -226,9 +226,46 @@ pub mod wasm32 { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub mod x86 { pub use crate::generated::Avx2; + pub use crate::generated::Avx512; pub use crate::generated::Sse4_2; } +#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))] +#[inline] +fn x86_detects_icelake_avx512() -> bool { + std::arch::is_x86_feature_detected!("adx") + && std::arch::is_x86_feature_detected!("aes") + && std::arch::is_x86_feature_detected!("avx512bitalg") + && std::arch::is_x86_feature_detected!("avx512bw") + && std::arch::is_x86_feature_detected!("avx512cd") + && std::arch::is_x86_feature_detected!("avx512dq") + && std::arch::is_x86_feature_detected!("avx512f") + && std::arch::is_x86_feature_detected!("avx512ifma") + && std::arch::is_x86_feature_detected!("avx512vbmi") + && std::arch::is_x86_feature_detected!("avx512vbmi2") + && std::arch::is_x86_feature_detected!("avx512vl") + && std::arch::is_x86_feature_detected!("avx512vnni") + && std::arch::is_x86_feature_detected!("avx512vpopcntdq") + && std::arch::is_x86_feature_detected!("bmi1") + && std::arch::is_x86_feature_detected!("bmi2") + && std::arch::is_x86_feature_detected!("cmpxchg16b") + && std::arch::is_x86_feature_detected!("fma") + && std::arch::is_x86_feature_detected!("gfni") + && std::arch::is_x86_feature_detected!("lzcnt") + && std::arch::is_x86_feature_detected!("movbe") + && std::arch::is_x86_feature_detected!("pclmulqdq") + && std::arch::is_x86_feature_detected!("popcnt") + && std::arch::is_x86_feature_detected!("rdrand") + && std::arch::is_x86_feature_detected!("rdseed") + && std::arch::is_x86_feature_detected!("sha") + && std::arch::is_x86_feature_detected!("vaes") + && std::arch::is_x86_feature_detected!("vpclmulqdq") + && std::arch::is_x86_feature_detected!("xsave") + && std::arch::is_x86_feature_detected!("xsavec") + && std::arch::is_x86_feature_detected!("xsaveopt") + && std::arch::is_x86_feature_detected!("xsaves") +} + /// The level enum with the specific SIMD capabilities available. /// /// The contained values serve as a proof that the associated target @@ -290,6 +327,9 @@ pub enum Level { )) ))] Sse4_2(Sse4_2), + /// Ice Lake-class AVX-512 on (32 and 64 bit) x86. + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + Avx512(Avx512), /// The x86-64-v3 instruction set on (32 and 64 bit) x86, including AVX2 and FMA. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Avx2(Avx2), @@ -341,6 +381,10 @@ impl Level { } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + if x86_detects_icelake_avx512() { + return unsafe { Self::Avx512(Avx512::new_unchecked()) }; + } + // Feature list sourced from `rustc --print=cfg --target x86_64-unknown-linux-gnu -C target-cpu=x86-64-v3` // However, the following features are implied by avx2 and do not need to be spelled out: // avx,fxsr,sse,sse2,sse3,sse4.1,sse4.2,ssse3 @@ -514,6 +558,9 @@ impl Level { #[inline] pub fn as_sse4_2(self) -> Option { match self { + // Safety: The Avx512 struct represents an Ice Lake feature set, which includes the + // `sse4.2`, `cmpxchg16b`, and `popcnt` features required by Sse4_2. + Self::Avx512(_avx512) => unsafe { Some(Sse4_2::new_unchecked()) }, // Safety: The Avx2 struct represents the x86-64-v3 feature set being enabled, which // includes the `sse4.2`, `cmpxchg16b`, and `popcnt` features required by Sse4_2. Self::Avx2(_avx) => unsafe { Some(Sse4_2::new_unchecked()) }, @@ -557,11 +604,29 @@ impl Level { reason = "On machines which statically support `avx2`, there is only one variant." )] match self { + // Safety: The Ice Lake AVX-512 feature set includes the x86-64-v3 features required by Avx2. + Self::Avx512(_avx512) => unsafe { Some(Avx2::new_unchecked()) }, Self::Avx2(avx2) => Some(avx2), _ => None, } } + /// If this is a proof that the Ice Lake AVX-512 feature set is available, access that + /// instruction set. + /// + /// See [`Avx512::new_unchecked`] for the exact list of CPU features this token enables. + /// + /// This can be used in combination with the [kernel] macro to safely access level-specific + /// SIMD intrinsics. + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[inline] + pub fn as_avx512(self) -> Option { + match self { + Self::Avx512(avx512) => Some(avx512), + _ => None, + } + } + /// Get the strongest statically supported SIMD level. /// /// That is, if your compilation run ambiently declares that a target feature is enabled, @@ -604,6 +669,40 @@ impl Level { } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + #[cfg(all( + target_feature = "adx", + target_feature = "aes", + target_feature = "avx512bitalg", + target_feature = "avx512bw", + target_feature = "avx512cd", + target_feature = "avx512dq", + target_feature = "avx512f", + target_feature = "avx512ifma", + target_feature = "avx512vbmi", + target_feature = "avx512vbmi2", + target_feature = "avx512vl", + target_feature = "avx512vnni", + target_feature = "avx512vpopcntdq", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "fma", + target_feature = "gfni", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "pclmulqdq", + target_feature = "popcnt", + target_feature = "rdrand", + target_feature = "rdseed", + target_feature = "sha", + target_feature = "vaes", + target_feature = "vpclmulqdq", + target_feature = "xsave", + target_feature = "xsavec", + target_feature = "xsaveopt", + target_feature = "xsaves" + ))] + return unsafe { Self::Avx512(Avx512::new_unchecked()) }; #[cfg(all( target_feature = "avx2", target_feature = "bmi1", @@ -614,7 +713,40 @@ impl Level { target_feature = "lzcnt", target_feature = "movbe", target_feature = "popcnt", - target_feature = "xsave" + target_feature = "xsave", + not(all( + target_feature = "adx", + target_feature = "aes", + target_feature = "avx512bitalg", + target_feature = "avx512bw", + target_feature = "avx512cd", + target_feature = "avx512dq", + target_feature = "avx512f", + target_feature = "avx512ifma", + target_feature = "avx512vbmi", + target_feature = "avx512vbmi2", + target_feature = "avx512vl", + target_feature = "avx512vnni", + target_feature = "avx512vpopcntdq", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "fma", + target_feature = "gfni", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "pclmulqdq", + target_feature = "popcnt", + target_feature = "rdrand", + target_feature = "rdseed", + target_feature = "sha", + target_feature = "vaes", + target_feature = "vpclmulqdq", + target_feature = "xsave", + target_feature = "xsavec", + target_feature = "xsaveopt", + target_feature = "xsaves" + )) ))] return unsafe { Self::Avx2(Avx2::new_unchecked()) }; #[cfg(all( diff --git a/fearless_simd/src/macros.rs b/fearless_simd/src/macros.rs index 346913862..be73bd6d1 100644 --- a/fearless_simd/src/macros.rs +++ b/fearless_simd/src/macros.rs @@ -103,6 +103,15 @@ macro_rules! dispatch { ) } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + $crate::Level::Avx512(avx512) => { + let $simd = launder(avx512); + $crate::Simd::vectorize( + avx512, + #[inline(always)] + || $op, + ) + } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] $crate::Level::Avx2(avx2) => { let $simd = launder(avx2); $crate::Simd::vectorize( diff --git a/fearless_simd/src/traits.rs b/fearless_simd/src/traits.rs index e51b0a5fb..77b63f5f2 100644 --- a/fearless_simd/src/traits.rs +++ b/fearless_simd/src/traits.rs @@ -66,6 +66,7 @@ impl Seal for u16 {} impl Seal for i16 {} impl Seal for u32 {} impl Seal for i32 {} +impl Seal for u64 {} impl Seal for i64 {} /// Value conversion, adding a SIMD blessing. @@ -141,6 +142,10 @@ impl SimdElement for i32 { type Mask = Self; } +impl SimdElement for u64 { + type Mask = i64; +} + impl SimdElement for i64 { type Mask = Self; } diff --git a/fearless_simd/src/transmute.rs b/fearless_simd/src/transmute.rs index 0c3b2a33e..7baad51a6 100644 --- a/fearless_simd/src/transmute.rs +++ b/fearless_simd/src/transmute.rs @@ -26,14 +26,18 @@ use core::arch::aarch64::{ int8x16_t, int8x16x2_t, int8x16x4_t, int16x8_t, int16x8x2_t, int16x8x4_t, int32x4_t, int32x4x2_t, int32x4x4_t, int64x2_t, int64x2x2_t, int64x2x4_t, uint8x16_t, uint8x16x2_t, uint8x16x4_t, uint16x8_t, uint16x8x2_t, uint16x8x4_t, uint32x4_t, uint32x4x2_t, uint32x4x4_t, - uint64x2_t, + uint64x2_t, uint64x2x2_t, uint64x2x4_t, }; #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] use core::arch::wasm32::v128; #[cfg(target_arch = "x86")] -use core::arch::x86::{__m128, __m128d, __m128i, __m256, __m256d, __m256i}; +use core::arch::x86::{ + __m128, __m128d, __m128i, __m256, __m256d, __m256i, __m512, __m512d, __m512i, +}; #[cfg(target_arch = "x86_64")] -use core::arch::x86_64::{__m128, __m128d, __m128i, __m256, __m256d, __m256i}; +use core::arch::x86_64::{ + __m128, __m128d, __m128i, __m256, __m256d, __m256i, __m512, __m512d, __m512i, +}; /// Types that can be safely copied through an arbitrary same-sized bit representation. /// @@ -98,6 +102,7 @@ impl_aligned_simd_pod!( Aligned128<[u8; 16]>, Aligned128<[u16; 8]>, Aligned128<[u32; 4]>, + Aligned128<[u64; 2]>, Aligned256<[f32; 8]>, Aligned256<[f64; 4]>, Aligned256<[i8; 32]>, @@ -107,6 +112,7 @@ impl_aligned_simd_pod!( Aligned256<[u8; 32]>, Aligned256<[u16; 16]>, Aligned256<[u32; 8]>, + Aligned256<[u64; 4]>, Aligned512<[f32; 16]>, Aligned512<[f64; 8]>, Aligned512<[i8; 64]>, @@ -116,6 +122,7 @@ impl_aligned_simd_pod!( Aligned512<[u8; 64]>, Aligned512<[u16; 32]>, Aligned512<[u32; 16]>, + Aligned512<[u64; 8]>, ); // the `const` is just to only use a single cfg annotation, nothing to do with const evaluation @@ -134,6 +141,9 @@ const _: () = { unsafe impl SimdPod for __m256 {} unsafe impl SimdPod for __m256d {} unsafe impl SimdPod for __m256i {} + unsafe impl SimdPod for __m512 {} + unsafe impl SimdPod for __m512d {} + unsafe impl SimdPod for __m512i {} }; #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] @@ -172,6 +182,8 @@ const _: () = { unsafe impl SimdPod for uint32x4x2_t {} unsafe impl SimdPod for uint32x4x4_t {} unsafe impl SimdPod for uint64x2_t {} + unsafe impl SimdPod for uint64x2x2_t {} + unsafe impl SimdPod for uint64x2x4_t {} }; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] @@ -191,6 +203,9 @@ impl_aligned_simd_pod!( Aligned512<[__m256; 2]>, Aligned512<[__m256d; 2]>, Aligned512<[__m256i; 2]>, + Aligned512<__m512>, + Aligned512<__m512d>, + Aligned512<__m512i>, ); #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] @@ -211,6 +226,7 @@ impl_aligned_simd_pod!( Aligned128, Aligned128, Aligned128, + Aligned128, Aligned256, Aligned256, Aligned256, @@ -220,6 +236,7 @@ impl_aligned_simd_pod!( Aligned256, Aligned256, Aligned256, + Aligned256, Aligned512, Aligned512, Aligned512, @@ -229,6 +246,7 @@ impl_aligned_simd_pod!( Aligned512, Aligned512, Aligned512, + Aligned512, ); /// Like [`core::mem::transmute_copy`], but statically rejects differently-sized diff --git a/fearless_simd_dev_macros/src/lib.rs b/fearless_simd_dev_macros/src/lib.rs index 438632cb9..78b301110 100644 --- a/fearless_simd_dev_macros/src/lib.rs +++ b/fearless_simd_dev_macros/src/lib.rs @@ -21,6 +21,7 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream { let neon_name = get_ident("neon"); let sse4_name = get_ident("sse4"); let avx2_name = get_ident("avx2"); + let avx512_name = get_ident("avx512"); let wasm_name = get_ident("wasm"); let ignore_attr = |f: fn(&str) -> bool| { @@ -40,6 +41,7 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream { let ignore_neon = ignore_attr(exclude_neon); let ignore_sse4 = ignore_attr(exclude_sse4); let ignore_avx2 = ignore_attr(exclude_avx2); + let ignore_avx512 = ignore_attr(exclude_avx512); let ignore_wasm = ignore_attr(exclude_wasm); let fallback_snippet = quote! { @@ -116,6 +118,52 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream { } }; + let avx512_snippet = quote! { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + #ignore_avx512 + fn #avx512_name() { + if std::arch::is_x86_feature_detected!("adx") + && std::arch::is_x86_feature_detected!("aes") + && std::arch::is_x86_feature_detected!("avx512bitalg") + && std::arch::is_x86_feature_detected!("avx512bw") + && std::arch::is_x86_feature_detected!("avx512cd") + && std::arch::is_x86_feature_detected!("avx512dq") + && std::arch::is_x86_feature_detected!("avx512f") + && std::arch::is_x86_feature_detected!("avx512ifma") + && std::arch::is_x86_feature_detected!("avx512vbmi") + && std::arch::is_x86_feature_detected!("avx512vbmi2") + && std::arch::is_x86_feature_detected!("avx512vl") + && std::arch::is_x86_feature_detected!("avx512vnni") + && std::arch::is_x86_feature_detected!("avx512vpopcntdq") + && std::arch::is_x86_feature_detected!("bmi1") + && std::arch::is_x86_feature_detected!("bmi2") + && std::arch::is_x86_feature_detected!("cmpxchg16b") + && std::arch::is_x86_feature_detected!("fma") + && std::arch::is_x86_feature_detected!("gfni") + && std::arch::is_x86_feature_detected!("lzcnt") + && std::arch::is_x86_feature_detected!("movbe") + && std::arch::is_x86_feature_detected!("pclmulqdq") + && std::arch::is_x86_feature_detected!("popcnt") + && std::arch::is_x86_feature_detected!("rdrand") + && std::arch::is_x86_feature_detected!("rdseed") + && std::arch::is_x86_feature_detected!("sha") + && std::arch::is_x86_feature_detected!("vaes") + && std::arch::is_x86_feature_detected!("vpclmulqdq") + && std::arch::is_x86_feature_detected!("xsave") + && std::arch::is_x86_feature_detected!("xsavec") + && std::arch::is_x86_feature_detected!("xsaveopt") + && std::arch::is_x86_feature_detected!("xsaves") + { + let avx512 = unsafe { fearless_simd::x86::Avx512::new_unchecked() }; + avx512.vectorize( + #[inline(always)] + || #input_fn_name(avx512) + ); + } + } + }; + let wasm_snippet = quote! { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] #[test] @@ -135,6 +183,7 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream { #wasm_snippet #sse4_snippet #avx2_snippet + #avx512_snippet } .into() } @@ -158,6 +207,10 @@ fn exclude_avx2(_test_name: &str) -> bool { false } +fn exclude_avx512(_test_name: &str) -> bool { + false +} + fn exclude_wasm(_test_name: &str) -> bool { false } diff --git a/fearless_simd_gen/src/arch/neon.rs b/fearless_simd_gen/src/arch/neon.rs index f32a4f3e2..b0a11cf96 100644 --- a/fearless_simd_gen/src/arch/neon.rs +++ b/fearless_simd_gen/src/arch/neon.rs @@ -44,8 +44,19 @@ fn translate_op(op: &str) -> Option<&'static str> { // expects args and return value in arch dialect pub(crate) fn expr(op: &str, ty: &VecType, args: &[TokenStream]) -> TokenStream { // There is no logical NOT for 64-bit, so we need this workaround. - if op == "not" && ty.scalar_bits == 64 && ty.scalar == ScalarType::Mask { - return quote! { vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a.into()))) }; + if op == "not" && ty.scalar_bits == 64 { + let a = &args[0]; + return match ty.scalar { + ScalarType::Int | ScalarType::Mask => { + quote! { vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(#a))) } + } + ScalarType::Unsigned => { + quote! { vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(#a))) } + } + ScalarType::Float => { + unreachable!("64-bit floating point vectors do not support logical NOT") + } + }; } if let Some(xlat) = translate_op(op) { diff --git a/fearless_simd_gen/src/arch/x86.rs b/fearless_simd_gen/src/arch/x86.rs index 43dd8b5a5..b09715657 100644 --- a/fearless_simd_gen/src/arch/x86.rs +++ b/fearless_simd_gen/src/arch/x86.rs @@ -171,9 +171,10 @@ pub(crate) fn coarse_type(vec_ty: &VecType) -> &'static str { pub(crate) fn set1_intrinsic(vec_ty: &VecType) -> Ident { use ScalarType::*; - let suffix = match (vec_ty.scalar, vec_ty.scalar_bits) { - (Int | Unsigned | Mask, 64) => "epi64x", - (scalar, bits) => op_suffix(scalar, bits, false), + let suffix = match (vec_ty.scalar, vec_ty.scalar_bits, vec_ty.n_bits()) { + (Int | Unsigned | Mask, 64, 512) => "epi64", + (Int | Unsigned | Mask, 64, _) => "epi64x", + (scalar, bits, _) => op_suffix(scalar, bits, false), }; intrinsic_ident("set1", suffix, vec_ty.n_bits()) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index ac27af318..a7c6a37cb 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -208,6 +208,9 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } } + OpSig::MaskSet => { + panic!("Mask set must operate on the full mask vector") + } OpSig::LoadInterleaved { block_size, block_count, @@ -309,8 +312,78 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } -pub(crate) fn scalar_binary(f: TokenStream) -> TokenStream { - quote! { core::array::from_fn(|i| #f(a[i], b[i])).simd_into(self) } +pub(crate) fn unrolled_array(len: usize, item: impl FnMut(usize) -> TokenStream) -> TokenStream { + let items = (0..len).map(item).collect::>(); + quote! { [#(#items),*] } +} + +pub(crate) fn scalar_binary(f: TokenStream, vec_ty: &VecType, simd: impl ToTokens) -> TokenStream { + let scalar = vec_ty.scalar.rust(vec_ty.scalar_bits); + let len = vec_ty.len; + let items = unrolled_array(len, |idx| quote! { #f(a[#idx], b[#idx]) }); + + quote! { + let a: [#scalar; #len] = a.into(); + let b: [#scalar; #len] = b.into(); + let result: [#scalar; #len] = #items; + result.simd_into(#simd) + } +} + +pub(crate) fn scalar_binary_method( + method: &str, + vec_ty: &VecType, + simd: impl ToTokens, +) -> TokenStream { + let method = Ident::new(method, Span::call_site()); + let scalar = vec_ty.scalar.rust(vec_ty.scalar_bits); + let len = vec_ty.len; + let items = unrolled_array(len, |idx| quote! { a[#idx].#method(b[#idx]) }); + + quote! { + let a: [#scalar; #len] = a.into(); + let b: [#scalar; #len] = b.into(); + let result: [#scalar; #len] = #items; + result.simd_into(#simd) + } +} + +pub(crate) fn scalar_shift(f: TokenStream, vec_ty: &VecType, simd: impl ToTokens) -> TokenStream { + let scalar = vec_ty.scalar.rust(vec_ty.scalar_bits); + let len = vec_ty.len; + let items = unrolled_array(len, |idx| quote! { #f(a[#idx], shift) }); + + quote! { + let a: [#scalar; #len] = a.into(); + let result: [#scalar; #len] = #items; + result.simd_into(#simd) + } +} + +pub(crate) fn scalar_compare(method: &str, vec_ty: &VecType, simd: impl ToTokens) -> TokenStream { + let scalar = vec_ty.scalar.rust(vec_ty.scalar_bits); + let mask_scalar = ScalarType::Mask.rust(vec_ty.scalar_bits); + let len = vec_ty.len; + let op = match method { + "simd_eq" => quote! { == }, + "simd_lt" => quote! { < }, + "simd_le" => quote! { <= }, + "simd_ge" => quote! { >= }, + "simd_gt" => quote! { > }, + _ => unreachable!("unsupported scalar comparison: {method}"), + }; + let items = unrolled_array(len, |idx| { + quote! { if a[#idx] #op b[#idx] { true_lane } else { false_lane } } + }); + + quote! { + let a: [#scalar; #len] = a.into(); + let b: [#scalar; #len] = b.into(); + let true_lane: #mask_scalar = !0; + let false_lane: #mask_scalar = 0; + let result: [#mask_scalar; #len] = #items; + result.simd_into(#simd) + } } pub(crate) fn generic_block_split( @@ -375,10 +448,23 @@ pub(crate) fn generic_from_array( } else { quote! { val } }; - // There are architecture-specific "load" intrinsics, but they can actually be *worse* for performance. If they // lower to LLVM intrinsics, they will likely not be optimized until much later in the pipeline (if at all), // resulting in substantially worse codegen. See https://github.com/linebender/fearless_simd/pull/185. + // + // Safety: The native vector type backing any implementation will be: + // - A `#[repr(simd)]` type, which has the same layout as an array of scalars + // - An array of `#[repr(simd)]` types + // - For AArch64 specifically, a `#[repr(C)]` tuple of `#[repr(simd)]` types + // + // These all have the same layout as a flat array of the corresponding scalars. `checked_transmute_copy` + // statically verifies that the source and destination sizes match. The native vector types probably have + // greater alignment requirements than the source array type we're copying from, but that's explicitly allowed by + // transmute_copy: + // + // > This function will unsafely assume the pointer src is valid for size_of:: bytes by transmuting &Src to + // > &Dst and then reading the &Dst **(except that this is done in a way that is correct even when &Dst has + // > stricter alignment requirements than &Src).** let expr = quote! { crate::transmute::checked_transmute_copy(#inner_ref) }; @@ -452,11 +538,18 @@ pub(crate) fn generic_from_bytes(method_sig: TokenStream, vec_ty: &VecType) -> T pub(crate) fn generic_mask_from_bitmask(method_sig: TokenStream, vec_ty: &VecType) -> TokenStream { let scalar = vec_ty.scalar.rust(vec_ty.scalar_bits); let len = vec_ty.len; + let lanes = unrolled_array(len, |idx| { + let bit = if idx == 0 { + quote! { bits & 1 } + } else { + quote! { (bits >> #idx) & 1 } + }; + quote! { if #bit != 0 { !0 } else { 0 } } + }); quote! { #method_sig { - let lanes: [#scalar; #len] = - core::array::from_fn(|i| if ((bits >> i) & 1) != 0 { !0 } else { 0 }); + let lanes: [#scalar; #len] = #lanes; lanes.simd_into(self) } } @@ -481,3 +574,22 @@ pub(crate) fn generic_mask_to_bitmask(method_sig: TokenStream, vec_ty: &VecType) } } } + +pub(crate) fn generic_mask_set(method_sig: TokenStream, vec_ty: &VecType) -> TokenStream { + let from_array = generic_op_name("load_array", vec_ty); + let as_array = generic_op_name("as_array", vec_ty); + let len = vec_ty.len; + + quote! { + #method_sig { + assert!( + index < #len, + "mask lane index {index} is out of bounds for {} lanes", + #len + ); + let mut lanes = self.#as_array(*a); + lanes[index] = if value { !0 } else { 0 }; + *a = self.#from_array(lanes); + } + } +} diff --git a/fearless_simd_gen/src/level.rs b/fearless_simd_gen/src/level.rs index ed12b6f84..3680a2cdd 100644 --- a/fearless_simd_gen/src/level.rs +++ b/fearless_simd_gen/src/level.rs @@ -34,11 +34,22 @@ pub(crate) trait Level { /// type *larger* than [`Level::max_block_size`], since [`VecType::aligned_wrapper_ty`] will split those up into /// smaller blocks. fn arch_ty(&self, vec_ty: &VecType) -> TokenStream; + /// The associated storage type used by a public SIMD vector for this level. + /// + /// Most levels wrap their native storage in an `Aligned*` newtype, but some compact scalar-like + /// representations, such as AVX-512 masks, can store the native type directly. + fn arch_storage_ty(&self, vec_ty: &VecType) -> TokenStream { + vec_ty.aligned_wrapper_ty(|vec_ty| self.arch_ty(vec_ty), self.max_block_size()) + } /// The docstring for this SIMD level token. fn token_doc(&self) -> &'static str; /// Any additional imports or supporting code necessary for the module (for instance, importing /// implementation-specific functions from `core::arch`). fn make_module_prelude(&self) -> TokenStream; + /// Inner attributes to place at the top of the generated module. + fn make_module_attrs(&self) -> TokenStream { + TokenStream::new() + } /// The body of the SIMD token's inherent `impl` block. By convention, this contains an unsafe `new_unchecked` /// method for constructing a SIMD token that may not be supported on current hardware, or a safe `new` method for /// constructing a SIMD token that is statically known to be supported. @@ -68,8 +79,7 @@ pub(crate) trait Level { let mut assoc_types = vec![]; for vec_ty in SIMD_TYPES { let ty_ident = vec_ty.rust(); - let wrapper_ty = - vec_ty.aligned_wrapper_ty(|vec_ty| self.arch_ty(vec_ty), self.max_block_size()); + let wrapper_ty = self.arch_storage_ty(vec_ty); assoc_types.push(quote! { type #ty_ident = #wrapper_ty; }); @@ -99,6 +109,19 @@ pub(crate) trait Level { } } + fn should_impl_arch_type_conversion(&self, ty: &VecType) -> bool { + let n_bits = ty.n_bits(); + n_bits <= self.max_block_size() && n_bits >= self.native_width() + } + + fn should_use_bitmask_arch_type_conversion(&self, _ty: &VecType) -> bool { + false + } + + fn custom_arch_type_conversion(&self, _ty: &VecType) -> Option { + None + } + fn make_simd_impl(&self) -> TokenStream { let level_tok = self.token(); let native_width = self.native_width(); @@ -153,6 +176,8 @@ pub(crate) trait Level { (ScalarType::Int, 16), (ScalarType::Unsigned, 32), (ScalarType::Int, 32), + (ScalarType::Unsigned, 64), + (ScalarType::Int, 64), (ScalarType::Mask, 8), (ScalarType::Mask, 16), (ScalarType::Mask, 32), @@ -189,19 +214,40 @@ pub(crate) trait Level { } fn make_type_impl(&self) -> TokenStream { - let native_width = self.native_width(); - let max_block_size = self.max_block_size(); let mut result = vec![]; for ty in SIMD_TYPES { - let n_bits = ty.n_bits(); // If n_bits is below our native width (e.g. 128 bits for AVX2), another module will have already // implemented the conversion. - if n_bits > max_block_size || n_bits < native_width { + if !self.should_impl_arch_type_conversion(ty) { continue; } let simd = ty.rust(); let arch = self.arch_ty(ty); - result.push(quote! { + let type_impl = if let Some(type_impl) = self.custom_arch_type_conversion(ty) { + type_impl + } else if self.should_use_bitmask_arch_type_conversion(ty) { + assert_eq!( + ty.scalar, + ScalarType::Mask, + "bitmask arch type conversions are only valid for mask types" + ); + quote! { + impl SimdFrom<#arch, S> for #simd { + #[inline(always)] + fn simd_from(simd: S, arch: #arch) -> Self { + Self::from_bitmask(simd, u64::from(arch)) + } + } + impl From<#simd> for #arch { + #[inline(always)] + #[allow(trivial_numeric_casts, reason = "generated uniformly for all __mmask widths")] + fn from(value: #simd) -> Self { + value.to_bitmask() as #arch + } + } + } + } else { + quote! { impl SimdFrom<#arch, S> for #simd { #[inline(always)] fn simd_from(simd: S, arch: #arch) -> Self { @@ -217,7 +263,9 @@ pub(crate) trait Level { crate::transmute::checked_transmute_copy(&value.val) } } - }); + } + }; + result.push(type_impl); } quote! { #( #result )* @@ -228,6 +276,7 @@ pub(crate) trait Level { let level_tok = self.token(); let token_doc = self.token_doc(); let imports = type_imports(); + let module_attrs = self.make_module_attrs(); let module_prelude = self.make_module_prelude(); let impl_body = self.make_impl_body(); let arch_types_impl = self.impl_arch_types(); @@ -236,6 +285,8 @@ pub(crate) trait Level { let footer = self.make_module_footer(); quote! { + #module_attrs + use crate::{prelude::*, seal::Seal, arch_types::ArchTypes, Level}; #imports diff --git a/fearless_simd_gen/src/main.rs b/fearless_simd_gen/src/main.rs index 10efdfd99..57df1ba3a 100644 --- a/fearless_simd_gen/src/main.rs +++ b/fearless_simd_gen/src/main.rs @@ -36,6 +36,7 @@ enum Module { Fallback, Sse4_2, Avx2, + Avx512, } #[derive(Parser)] @@ -66,6 +67,7 @@ impl Module { Self::Fallback => mk_fallback::Fallback.make_module(), Self::Sse4_2 => mk_x86::X86::Sse4_2.make_module(), Self::Avx2 => mk_x86::X86::Avx2.make_module(), + Self::Avx512 => mk_x86::X86::Avx512.make_module(), } } @@ -105,6 +107,7 @@ impl Module { Self::Wasm => "wasm", Self::Sse4_2 => "sse4_2", Self::Avx2 => "avx2", + Self::Avx512 => "avx512", } } } @@ -118,6 +121,7 @@ const MODULES: &[Module] = &[ Module::Wasm, Module::Sse4_2, Module::Avx2, + Module::Avx512, ]; const FILE_BASE: &str = "./fearless_simd/src/generated"; diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 70122a9e7..92099258a 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -3,8 +3,8 @@ use crate::arch::fallback; use crate::generic::{ - generic_from_bytes, generic_mask_from_bitmask, generic_mask_to_bitmask, generic_op_name, - generic_to_bytes, integer_lane_mask_splat_arg, + generic_from_bytes, generic_mask_from_bitmask, generic_mask_set, generic_mask_to_bitmask, + generic_op_name, generic_to_bytes, integer_lane_mask_splat_arg, }; use crate::level::Level; use crate::ops::{Op, OpSig, RefKind, valid_reinterpret}; @@ -466,6 +466,7 @@ impl Level for Fallback { } OpSig::MaskFromBitmask => generic_mask_from_bitmask(method_sig, vec_ty), OpSig::MaskToBitmask => generic_mask_to_bitmask(method_sig, vec_ty), + OpSig::MaskSet => generic_mask_set(method_sig, vec_ty), OpSig::LoadInterleaved { block_size, block_count, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 401cf1644..a70f44d08 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -5,8 +5,8 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{ToTokens as _, format_ident, quote}; use crate::generic::{ - generic_as_array, generic_from_array, generic_from_bytes, generic_op_name, generic_store_array, - generic_to_bytes, integer_lane_mask_splat_arg, + generic_as_array, generic_from_array, generic_from_bytes, generic_mask_set, generic_op_name, + generic_store_array, generic_to_bytes, integer_lane_mask_splat_arg, scalar_binary_method, }; use crate::level::Level; use crate::ops::{Op, SlideGranularity, valid_reinterpret}; @@ -204,6 +204,18 @@ impl Level for Neon { } } OpSig::Binary => self.kernel_method(op, vec_ty, |token| match method { + "mul" + if vec_ty.scalar_bits == 64 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) => + { + scalar_binary_method("wrapping_mul", vec_ty, token) + } + "min" | "max" + if vec_ty.scalar_bits == 64 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) => + { + scalar_binary_method(method, vec_ty, token) + } "shlv" | "shrv" => { let mut args = if vec_ty.scalar == ScalarType::Int { // Signed case @@ -486,6 +498,7 @@ impl Level for Neon { } OpSig::MaskFromBitmask => self.handle_mask_from_bitmask(op, vec_ty), OpSig::MaskToBitmask => self.handle_mask_to_bitmask(op, vec_ty), + OpSig::MaskSet => generic_mask_set(method_sig, vec_ty), OpSig::FromArray { kind } => generic_from_array(method_sig, vec_ty, kind), OpSig::AsArray { kind } => { generic_as_array(method_sig, vec_ty, kind, self.max_block_size(), |vec_ty| { diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index 6b205022c..cbfe9e8ab 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -43,8 +43,8 @@ pub(crate) fn mk_simd_trait() -> TokenStream { /// # Associated Types /// /// The trait defines associated types for the highest "native" vector width of each scalar type (e.g. `f32s`, - /// `u32s`). These are always at least 128 bits, but may be larger. Currently, they are 128 bits everywhere but - /// AVX2, where they are 256 bits. + /// `u32s`). These are always at least 128 bits, but may be larger. Currently, they are 128 bits on the + /// fallback, NEON, WASM, and SSE4.2 backends, 256 bits on AVX2, and 512 bits on AVX-512. /// /// # Example /// @@ -68,7 +68,7 @@ pub(crate) fn mk_simd_trait() -> TokenStream { /// A native-width SIMD vector of [`f32`]s. type f32s: SimdFloat, Mask = Self::mask32s, Bytes = ::Bytes> + SimdCvtFloat + SimdCvtFloat; /// A native-width SIMD vector of [`f64`]s. - type f64s: SimdFloat, Mask = Self::mask64s>; + type f64s: SimdFloat, Mask = Self::mask64s, Bytes = ::Bytes>; /// A native-width SIMD vector of [`u8`]s. type u8s: SimdInt, Mask = Self::mask8s>; /// A native-width SIMD vector of [`i8`]s. @@ -82,6 +82,11 @@ pub(crate) fn mk_simd_trait() -> TokenStream { /// A native-width SIMD vector of [`i32`]s. type i32s: SimdInt, Mask = Self::mask32s, Bytes = ::Bytes> + SimdCvtTruncate + core::ops::Neg; + /// A native-width SIMD vector of [`u64`]s. + type u64s: SimdInt, Mask = Self::mask64s>; + /// A native-width SIMD vector of [`i64`]s. + type i64s: SimdInt, Mask = Self::mask64s, Bytes = ::Bytes> + + core::ops::Neg; /// A native-width SIMD mask with 8-bit lanes. type mask8s: SimdMask + Select + Select + Select; /// A native-width SIMD mask with 16-bit lanes. @@ -89,7 +94,7 @@ pub(crate) fn mk_simd_trait() -> TokenStream { /// A native-width SIMD mask with 32-bit lanes. type mask32s: SimdMask + Select + Select + Select + Select; /// A native-width SIMD mask with 64-bit lanes. - type mask64s: SimdMask + Select + Select; + type mask64s: SimdMask + Select + Select + Select + Select; /// This SIMD token's feature level. fn level(self) -> Level; diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index 3960e3281..7c66796d4 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -5,7 +5,7 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{format_ident, quote}; use crate::{ - generic::generic_op_name, + generic::{generic_op_name, unrolled_array}, ops::{ F32_TO_I32, F32_TO_I32_PRECISE, F32_TO_U32, F32_TO_U32_PRECISE, I32_TO_F32, Op, OpSig, TyFlavor, U32_TO_F32, vec_trait_ops_for, @@ -298,6 +298,7 @@ fn simd_mask_impl(ty: &VecType) -> TokenStream { let splat = generic_op_name("splat", ty); let from_bitmask_op = generic_op_name("from_bitmask", ty); let to_bitmask_op = generic_op_name("to_bitmask", ty); + let set_op = generic_op_name("set", ty); let from_array_op = generic_op_name("load_array", ty); let as_array_op = generic_op_name("as_array", ty); let mut methods = vec![]; @@ -322,9 +323,6 @@ fn simd_mask_impl(ty: &VecType) -> TokenStream { } } - // Current backends store masks as signed integer lanes, so `set` uses a generic - // spill/update/reload path. Future compact predicate backends such as AVX-512 can - // switch this implementation to `to_bitmask`/`from_bitmask`. quote! { impl SimdMask for #name { type Element = #scalar; @@ -352,14 +350,7 @@ fn simd_mask_impl(ty: &VecType) -> TokenStream { #[inline(always)] fn set(&mut self, index: usize, value: bool) { - assert!( - index < #len, - "mask lane index {index} is out of bounds for {} lanes", - #len - ); - let mut lanes = self.simd.#as_array_op(*self); - lanes[index] = if value { !0 } else { 0 }; - *self = self.simd.#from_array_op(lanes); + self.simd.#set_op(self, index, value); } #[inline(always)] @@ -383,6 +374,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { let name = ty.rust(); let scalar = ty.scalar.rust(ty.scalar_bits); let len = Literal::usize_unsuffixed(ty.len); + let from_fn_items = unrolled_array(ty.len, |idx| quote! { f(#idx) }); let vec_trait = match ty.scalar { ScalarType::Float => "SimdFloat", ScalarType::Unsigned | ScalarType::Int => "SimdInt", @@ -482,8 +474,8 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { } #[inline(always)] - fn from_fn(simd: S, f: impl FnMut(usize) -> #scalar) -> Self { - simd.#from_array_op(core::array::from_fn(f)) + fn from_fn(simd: S, mut f: impl FnMut(usize) -> #scalar) -> Self { + simd.#from_array_op(#from_fn_items) } #[inline(always)] diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 6eda81e99..9a53d5b61 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -7,8 +7,8 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ generic_as_array, generic_block_combine, generic_block_split, generic_from_array, - generic_from_bytes, generic_op_name, generic_store_array, generic_to_bytes, - integer_lane_mask_splat_arg, scalar_binary, + generic_from_bytes, generic_mask_set, generic_op_name, generic_store_array, generic_to_bytes, + integer_lane_mask_splat_arg, scalar_binary, scalar_binary_method, scalar_compare, }; use crate::level::Level; use crate::ops::{Op, Quantifier, SlideGranularity, valid_reinterpret}; @@ -246,8 +246,14 @@ impl Level for WasmSimd128 { { #expr.simd_into(self) } } } - "shlv" => scalar_binary(quote!(core::ops::Shl::shl)), - "shrv" => scalar_binary(quote!(core::ops::Shr::shr)), + "min" | "max" + if vec_ty.scalar_bits == 64 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) => + { + scalar_binary_method(method, vec_ty, quote! { self }) + } + "shlv" => scalar_binary(quote!(core::ops::Shl::shl), vec_ty, quote! { self }), + "shrv" => scalar_binary(quote!(core::ops::Shr::shr), vec_ty, quote! { self }), "copysign" => { let splat = simple_intrinsic("splat", vec_ty); let sign_mask_literal = match vec_ty.scalar_bits { @@ -306,11 +312,16 @@ impl Level for WasmSimd128 { } } OpSig::Compare => { - let args = [quote! { a.into() }, quote! { b.into() }]; - let expr = wasm::expr(method, vec_ty, &args); + let expr = if vec_ty.scalar == ScalarType::Unsigned && vec_ty.scalar_bits == 64 { + scalar_compare(method, vec_ty, quote! { self }) + } else { + let args = [quote! { a.into() }, quote! { b.into() }]; + let expr = wasm::expr(method, vec_ty, &args); + quote! { #expr.simd_into(self) } + }; quote! { #method_sig { - #expr.simd_into(self) + #expr } } } @@ -592,6 +603,7 @@ impl Level for WasmSimd128 { } OpSig::MaskFromBitmask => mask_from_bitmask(method_sig, vec_ty), OpSig::MaskToBitmask => mask_to_bitmask(method_sig, vec_ty), + OpSig::MaskSet => generic_mask_set(method_sig, vec_ty), OpSig::LoadInterleaved { block_size, block_count, @@ -625,6 +637,13 @@ impl Level for WasmSimd128 { quote! { 2, 3, 6, 7 }, quote! { u32x4_shuffle }, ), + 64 => ( + quote! { 0, 2 }, + quote! { 1, 3 }, + quote! { 0, 2 }, + quote! { 1, 3 }, + quote! { u64x2_shuffle }, + ), _ => panic!("unsupported scalar_bits"), }; @@ -641,6 +660,31 @@ impl Level for WasmSimd128 { self.#combine_method_2x(combined_lower, combined_upper) }; + let shuffle_code = if vec_ty.scalar_bits == 64 { + quote! { + let out0 = #shuffle_fn::<#i1>(v0, v2); + let out1 = #shuffle_fn::<#i2>(v0, v2); + let out2 = #shuffle_fn::<#i1>(v1, v3); + let out3 = #shuffle_fn::<#i2>(v1, v3); + } + } else { + quote! { + // InterleaveLowerLanes(v0, v1) and InterleaveLowerLanes(v2, v3) + let v01_lower = #shuffle_fn::<#i1>(v0, v1); + let v23_lower = #shuffle_fn::<#i1>(v2, v3); + + // InterleaveUpperLanes(v0, v1) and InterleaveUpperLanes(v2, v3) + let v01_upper = #shuffle_fn::<#i2>(v0, v1); + let v23_upper = #shuffle_fn::<#i2>(v2, v3); + + // Interleave lower and upper to get final result + let out0 = #shuffle_fn::<#i3>(v01_lower, v23_lower); + let out1 = #shuffle_fn::<#i4>(v01_lower, v23_lower); + let out2 = #shuffle_fn::<#i3>(v01_upper, v23_upper); + let out3 = #shuffle_fn::<#i4>(v01_upper, v23_upper); + } + }; + quote! { #method_sig { let (chunks, []) = src.as_chunks::<#elems_per_vec>() else { @@ -659,20 +703,7 @@ impl Level for WasmSimd128 { &chunks[3], ); - // InterleaveLowerLanes(v0, v2) and InterleaveLowerLanes(v1, v3) - let v01_lower = #shuffle_fn::<#i1>(v0, v1); - let v23_lower = #shuffle_fn::<#i1>(v2, v3); - - // InterleaveUpperLanes(v0, v2) and InterleaveUpperLanes(v1, v3) - let v01_upper = #shuffle_fn::<#i2>(v0, v1); - let v23_upper = #shuffle_fn::<#i2>(v2, v3); - - // Interleave lower and upper to get final result - let out0 = #shuffle_fn::<#i3>(v01_lower, v23_lower); - let out1 = #shuffle_fn::<#i4>(v01_lower, v23_lower); - let out2 = #shuffle_fn::<#i3>(v01_upper, v23_upper); - let out3 = #shuffle_fn::<#i4>(v01_upper, v23_upper); - + #shuffle_code #combine_code } } @@ -701,6 +732,7 @@ impl Level for WasmSimd128 { quote! { 2, 6, 3, 7 }, quote! { u32x4_shuffle }, ), + 64 => (quote! { 0, 2 }, quote! { 1, 3 }, quote! { u64x2_shuffle }), _ => panic!("unsupported scalar_bits"), }; @@ -724,10 +756,15 @@ impl Level for WasmSimd128 { let v3: v128 = v3_vec.into(); }; - quote! { - #method_sig { - #split_code - + let shuffle_code = if vec_ty.scalar_bits == 64 { + quote! { + let out0 = #shuffle_fn::<#lower_indices>(v0, v1); + let out1 = #shuffle_fn::<#lower_indices>(v2, v3); + let out2 = #shuffle_fn::<#upper_indices>(v0, v1); + let out3 = #shuffle_fn::<#upper_indices>(v2, v3); + } + } else { + quote! { // InterleaveLowerLanes(v0, v2) and InterleaveLowerLanes(v1, v3) let v02_lower = #shuffle_fn::<#lower_indices>(v0, v2); let v13_lower = #shuffle_fn::<#lower_indices>(v1, v3); @@ -741,6 +778,14 @@ impl Level for WasmSimd128 { let out1 = #shuffle_fn::<#upper_indices>(v02_lower, v13_lower); let out2 = #shuffle_fn::<#lower_indices>(v02_upper, v13_upper); let out3 = #shuffle_fn::<#upper_indices>(v02_upper, v13_upper); + } + }; + + quote! { + #method_sig { + #split_code + + #shuffle_code let (chunks, []) = dest.as_chunks_mut::<#elems_per_vec>() else { unreachable!() diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 64b7ff3f1..9896ba5bb 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -8,8 +8,8 @@ use crate::arch::x86::{ }; use crate::generic::{ generic_as_array, generic_block_combine, generic_block_split, generic_from_array, - generic_from_bytes, generic_op_name, generic_store_array, generic_to_bytes, - integer_lane_mask_splat_arg, scalar_binary, + generic_from_bytes, generic_mask_set, generic_op_name, generic_store_array, generic_to_bytes, + integer_lane_mask_splat_arg, scalar_binary, scalar_binary_method, scalar_compare, scalar_shift, }; use crate::level::Level; use crate::ops::{Op, OpSig, Quantifier, SlideGranularity, valid_reinterpret}; @@ -21,13 +21,17 @@ use quote::{ToTokens as _, format_ident, quote}; pub(crate) enum X86 { Sse4_2, Avx2, + Avx512, } +pub(crate) const AVX512_FEATURES: &str = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves"; + impl Level for X86 { fn name(&self) -> &'static str { match self { Self::Sse4_2 => "Sse4_2", Self::Avx2 => "Avx2", + Self::Avx512 => "Avx512", } } @@ -35,6 +39,7 @@ impl Level for X86 { match self { Self::Sse4_2 => 128, Self::Avx2 => 256, + Self::Avx512 => 512, } } @@ -46,16 +51,18 @@ impl Level for X86 { Some(match self { Self::Sse4_2 => "sse4.2,cmpxchg16b,popcnt", Self::Avx2 => "avx2,bmi1,bmi2,cmpxchg16b,f16c,fma,lzcnt,movbe,popcnt,xsave", + Self::Avx512 => AVX512_FEATURES, }) } fn arch_ty(&self, vec_ty: &VecType) -> TokenStream { - // Future AVX-512 backends should be able to keep mask types opaque by storing them as - // `__mmask*` predicate registers instead of `__m*i` vectors: for example, `mask8x64` - // maps naturally to `__mmask64`, `mask16x32` to `__mmask32`, and `mask32x16`/`mask64x8` - // to `__mmask16`/`__mmask8`. Comparisons would return `_mm512_cmp*_mask`, selects would - // use `_mm512_mask_blend_*`, and legacy integer-lane interop could materialize vectors - // with `_mm512_movm_epi*` only at the API boundary. + // AVX-512 masks are compact predicate registers, not vector registers. + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let bits = avx512_mask_register_bits(vec_ty); + let name = format!("__mmask{bits}"); + return Ident::new(&name, Span::call_site()).into_token_stream(); + } + let suffix = match (vec_ty.scalar, vec_ty.scalar_bits) { (ScalarType::Float, 32) => "", (ScalarType::Float, 64) => "d", @@ -66,6 +73,14 @@ impl Level for X86 { Ident::new(&name, Span::call_site()).into_token_stream() } + fn arch_storage_ty(&self, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + self.arch_ty(vec_ty) + } else { + vec_ty.aligned_wrapper_ty(|vec_ty| self.arch_ty(vec_ty), self.max_block_size()) + } + } + fn token_doc(&self) -> &'static str { match self { Self::Sse4_2 => { @@ -74,6 +89,9 @@ impl Level for X86 { Self::Avx2 => { "A token for AVX2 intrinsics on `x86` and `x86_64`, representing the x86-64-v3 level." } + Self::Avx512 => { + "A token for AVX-512 intrinsics on `x86` and `x86_64`, representing an Ice Lake feature level." + } } } @@ -86,11 +104,29 @@ impl Level for X86 { } } + fn make_module_attrs(&self) -> TokenStream { + if *self != Self::Avx512 { + return TokenStream::new(); + } + + quote! { + #![allow( + clippy::identity_op, + reason = "AVX-512 mask code is generated uniformly for all __mmask widths" + )] + #![allow( + clippy::useless_conversion, + reason = "AVX-512 mask code is generated uniformly for all __mmask widths" + )] + } + } + fn make_module_footer(&self) -> TokenStream { let alignr_helpers = self.dyn_alignr_helpers(); let slide_helpers = match self { Self::Sse4_2 => Self::sse42_slide_helpers(), Self::Avx2 => Self::avx2_slide_helpers(), + Self::Avx512 => TokenStream::new(), }; quote! { @@ -135,9 +171,53 @@ impl Level for X86 { Self::Avx2 => quote! { Level::#level_tok(self) }, + Self::Avx512 => quote! { + Level::#level_tok(self) + }, } } + fn should_impl_arch_type_conversion(&self, ty: &VecType) -> bool { + let n_bits = ty.n_bits(); + if *self == Self::Avx512 && ty.scalar == ScalarType::Mask { + return n_bits <= self.max_block_size(); + } + n_bits <= self.max_block_size() && n_bits >= self.native_width() + } + + fn should_use_bitmask_arch_type_conversion(&self, ty: &VecType) -> bool { + *self == Self::Avx512 && ty.scalar == ScalarType::Mask + } + + fn custom_arch_type_conversion(&self, ty: &VecType) -> Option { + if *self == Self::Avx512 || ty.scalar != ScalarType::Mask { + return None; + } + + let simd = ty.rust(); + let arch = self.arch_ty(ty); + let lane_ty = ScalarType::Int.rust(ty.scalar_bits); + let len = ty.len; + + Some(quote! { + impl SimdFrom<#arch, S> for #simd { + #[inline(always)] + fn simd_from(simd: S, arch: #arch) -> Self { + let lanes: [#lane_ty; #len] = + crate::transmute::checked_transmute_copy(&arch); + lanes.simd_into(simd) + } + } + impl From<#simd> for #arch { + #[inline(always)] + fn from(value: #simd) -> Self { + let lanes: [#lane_ty; #len] = value.into(); + crate::transmute::checked_transmute_copy(&lanes) + } + } + }) + } + fn make_impl_body(&self) -> TokenStream { match self { Self::Sse4_2 => quote! { @@ -165,10 +245,35 @@ impl Level for X86 { Self { _private: () } } }, + Self::Avx512 => quote! { + /// Create a SIMD token. + /// + /// # Safety + /// + /// The Ice Lake AVX-512 CPU feature set must be available. + #[inline] + pub const unsafe fn new_unchecked() -> Self { + Self { _private: () } + } + }, } } fn should_use_generic_op(&self, op: &Op, vec_ty: &VecType) -> bool { + if *self == Self::Avx512 + && matches!( + op.sig, + OpSig::Slide { + granularity: SlideGranularity::WithinBlocks, + .. + } + ) + && vec_ty.scalar == ScalarType::Mask + && vec_ty.n_bits() > 128 + { + return true; + } + let should_use_generic = op.sig.should_use_generic_op(vec_ty, self.native_width()); if !should_use_generic { return false; @@ -216,6 +321,10 @@ impl Level for X86 { } => self.handle_mask_reduce(op, vec_ty, quantifier, condition), OpSig::MaskFromBitmask => self.handle_mask_from_bitmask(op, vec_ty), OpSig::MaskToBitmask => self.handle_mask_to_bitmask(op, vec_ty), + OpSig::MaskSet if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask => { + self.handle_avx512_mask_set(method_sig, vec_ty) + } + OpSig::MaskSet => generic_mask_set(method_sig, vec_ty), OpSig::LoadInterleaved { block_size, block_count, @@ -224,7 +333,17 @@ impl Level for X86 { block_size, block_count, } => self.handle_store_interleaved(op, vec_ty, block_size, block_count), + OpSig::FromArray { kind } + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask => + { + self.handle_avx512_mask_from_array(op, vec_ty, kind) + } OpSig::FromArray { kind } => generic_from_array(method_sig, vec_ty, kind), + OpSig::AsArray { kind } + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask => + { + self.handle_avx512_mask_as_array(op, vec_ty, kind) + } OpSig::AsArray { kind } => { generic_as_array(method_sig, vec_ty, kind, self.max_block_size(), |vec_ty| { self.arch_ty(vec_ty) @@ -597,8 +716,218 @@ fn signed_literal(value: u64, bits: u32) -> TokenStream { } } +fn avx512_mask_register_bits(vec_ty: &VecType) -> usize { + match vec_ty.len { + 0..=8 => 8, + 9..=16 => 16, + 17..=32 => 32, + 33..=64 => 64, + _ => unreachable!("SIMD masks never have more than 64 lanes"), + } +} + +fn avx512_mask_lane_bits(vec_ty: &VecType) -> TokenStream { + if vec_ty.len == 64 { + quote! { u64::MAX } + } else { + let bits = (1_u64 << vec_ty.len) - 1; + quote! { #bits } + } +} + +fn avx512_mask_value_with_simd( + vec_ty: &VecType, + bits: TokenStream, + simd: TokenStream, +) -> TokenStream { + let ty = vec_ty.rust(); + let bits = if avx512_mask_register_bits(vec_ty) == 64 { + bits + } else { + quote! { (#bits) as _ } + }; + quote! { + #ty { + val: #bits, + simd: #simd, + } + } +} + +fn avx512_mask_value(vec_ty: &VecType, bits: TokenStream) -> TokenStream { + avx512_mask_value_with_simd(vec_ty, bits, quote! { self }) +} + +fn avx512_mask_register_value_with_simd( + vec_ty: &VecType, + bits: TokenStream, + simd: TokenStream, +) -> TokenStream { + let ty = vec_ty.rust(); + quote! { + #ty { + val: #bits, + simd: #simd, + } + } +} + +fn avx512_mask_bits_expr(expr: TokenStream) -> TokenStream { + quote! { u64::from((#expr).val) } +} + +fn avx512_compare_op(method: &str) -> &'static str { + match method { + "simd_eq" => "cmpeq", + "simd_lt" => "cmplt", + "simd_le" => "cmple", + "simd_ge" => "cmpge", + "simd_gt" => "cmpgt", + _ => unreachable!(), + } +} + +fn avx512_float_compare_predicate(method: &str) -> i32 { + match method { + "simd_eq" => 0x00, + "simd_lt" => 0x11, + "simd_le" => 0x12, + "simd_ge" => 0x1D, + "simd_gt" => 0x1E, + "ord" => 0x07, + "unord" => 0x03, + _ => unreachable!(), + } +} + +fn avx512_mask_compare_expr(method: &str, vec_ty: &VecType) -> TokenStream { + let lane_mask = avx512_mask_lane_bits(vec_ty); + match method { + "simd_eq" => quote! { !u64::from(a.val ^ b.val) & #lane_mask }, + _ => unreachable!("masks only support equality comparison"), + } +} + +fn avx512_permutex2var_intrinsic(vec_ty: &VecType) -> Ident { + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, false); + intrinsic_ident("permutex2var", suffix, vec_ty.n_bits()) +} + +fn avx512_should_use_unzip_permutex2var(vec_ty: &VecType) -> bool { + vec_ty.scalar != ScalarType::Mask + && (vec_ty.n_bits() >= 256 + || (vec_ty.n_bits() == 128 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned))) +} + +fn avx512_permutexvar_intrinsic(vec_ty: &VecType) -> Ident { + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, false); + intrinsic_ident("permutexvar", suffix, vec_ty.n_bits()) +} + +fn avx512_mask_blend_intrinsic(vec_ty: &VecType) -> Ident { + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, false); + intrinsic_ident("mask_blend", suffix, vec_ty.n_bits()) +} + +fn avx512_index_vector(vec_ty: &VecType, indices: impl IntoIterator) -> TokenStream { + let indices: Vec = indices.into_iter().collect(); + let n_bits = vec_ty.n_bits(); + let scalar_bits = vec_ty.scalar_bits; + match (n_bits, scalar_bits) { + (128, 8) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 8)); + quote! { _mm_setr_epi8(#(#lanes),*) } + } + (256, 8) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 8)); + quote! { _mm256_setr_epi8(#(#lanes),*) } + } + (512, 8) => { + let lanes = indices + .into_iter() + .rev() + .map(|i| signed_literal(i as u64, 8)); + quote! { _mm512_set_epi8(#(#lanes),*) } + } + (128, 16) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 16)); + quote! { _mm_setr_epi16(#(#lanes),*) } + } + (256, 16) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 16)); + quote! { _mm256_setr_epi16(#(#lanes),*) } + } + (512, 16) => { + let lanes = indices + .into_iter() + .rev() + .map(|i| signed_literal(i as u64, 16)); + quote! { _mm512_set_epi16(#(#lanes),*) } + } + (128, 32) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 32)); + quote! { _mm_setr_epi32(#(#lanes),*) } + } + (256, 32) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 32)); + quote! { _mm256_setr_epi32(#(#lanes),*) } + } + (512, 32) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 32)); + quote! { _mm512_setr_epi32(#(#lanes),*) } + } + (128, 64) => { + let mut lanes = indices + .into_iter() + .map(|i| signed_literal(i as u64, 64)) + .collect::>(); + lanes.reverse(); + quote! { _mm_set_epi64x(#(#lanes),*) } + } + (256, 64) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 64)); + quote! { _mm256_setr_epi64x(#(#lanes),*) } + } + (512, 64) => { + let lanes = indices.into_iter().map(|i| signed_literal(i as u64, 64)); + quote! { _mm512_setr_epi64(#(#lanes),*) } + } + _ => unreachable!(), + } +} + +fn interleaved_load_indices(len: usize, block_count: usize) -> Vec { + let stream_len = len / block_count; + (0..block_count) + .flat_map(|stream| (0..stream_len).map(move |i| i * block_count + stream)) + .collect() +} + +fn interleaved_store_indices(len: usize, block_count: usize) -> Vec { + let stream_len = len / block_count; + (0..stream_len) + .flat_map(|i| (0..block_count).map(move |stream| stream * stream_len + i)) + .collect() +} + impl X86 { pub(crate) fn handle_splat(&self, op: Op, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let result = avx512_mask_value( + vec_ty, + quote! { + if val { #lane_mask } else { 0 } + }, + ); + let method_sig = op.simd_trait_method_sig(vec_ty); + return quote! { + #method_sig { + #result + } + }; + } let intrinsic = set1_intrinsic(vec_ty); let cast = match vec_ty.scalar { ScalarType::Unsigned => quote!(.cast_signed()), @@ -614,6 +943,9 @@ impl X86 { } fn has_specialized_mask_from_bitmask(&self, vec_ty: &VecType) -> bool { + if *self == Self::Avx512 { + return true; + } self.has_wide_byte_mask_from_bitmask(vec_ty) || self.has_wide_avx2_mask_from_bitmask(vec_ty) } @@ -633,9 +965,105 @@ impl X86 { } fn has_specialized_mask_to_bitmask(&self, vec_ty: &VecType) -> bool { + if *self == Self::Avx512 { + return true; + } vec_ty.scalar == ScalarType::Mask && vec_ty.scalar_bits == 16 } + pub(crate) fn handle_avx512_mask_from_array( + &self, + op: Op, + vec_ty: &VecType, + kind: crate::ops::RefKind, + ) -> TokenStream { + assert_eq!( + vec_ty.scalar, + ScalarType::Mask, + "AVX-512 mask array loads only operate on mask types" + ); + let movepi_mask = intrinsic_ident( + &format!("movepi{}", vec_ty.scalar_bits), + "mask", + vec_ty.n_bits(), + ); + let transmute_src = if kind == crate::ops::RefKind::Value { + quote! { &val } + } else { + quote! { val } + }; + // Mask arrays are specified as either 0 or -1 per lane, so the sign bit is the + // truth value. Other lane values have unspecified results. + self.kernel_method(op, vec_ty, |token| { + let result = avx512_mask_register_value_with_simd( + vec_ty, + quote! { #movepi_mask(lanes) }, + quote! { #token }, + ); + quote! { + let lanes = crate::transmute::checked_transmute_copy(#transmute_src); + #result + } + }) + } + + pub(crate) fn handle_avx512_mask_as_array( + &self, + op: Op, + vec_ty: &VecType, + kind: crate::ops::RefKind, + ) -> TokenStream { + assert_eq!( + vec_ty.scalar, + ScalarType::Mask, + "AVX-512 mask array stores only operate on mask types" + ); + assert!( + kind == crate::ops::RefKind::Value, + "mask array references are not exposed" + ); + let movm = intrinsic_ident( + "movm", + op_suffix(vec_ty.scalar, vec_ty.scalar_bits, true), + vec_ty.n_bits(), + ); + self.kernel_method(op, vec_ty, |_| { + quote! { + let lanes = #movm(a.val); + crate::transmute::checked_transmute_copy(&lanes) + } + }) + } + + pub(crate) fn handle_avx512_mask_set( + &self, + method_sig: TokenStream, + vec_ty: &VecType, + ) -> TokenStream { + assert_eq!( + vec_ty.scalar, + ScalarType::Mask, + "AVX-512 mask set only operates on mask types" + ); + let len = vec_ty.len; + let bits = avx512_mask_bits_expr(quote! { a }); + let result = avx512_mask_value(vec_ty, quote! { bits }); + + quote! { + #method_sig { + assert!( + index < #len, + "mask lane index {index} is out of bounds for {} lanes", + #len + ); + let bit = 1u64 << index; + let bits = #bits; + let bits = if value { bits | bit } else { bits & !bit }; + *a = #result; + } + } + } + pub(crate) fn handle_mask_from_bitmask(&self, op: Op, vec_ty: &VecType) -> TokenStream { assert_eq!( vec_ty.scalar, @@ -643,6 +1071,17 @@ impl X86 { "mask bitmask conversion only operates on masks" ); + if *self == Self::Avx512 { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let result = avx512_mask_value(vec_ty, quote! { bits & #lane_mask }); + let method_sig = op.simd_trait_method_sig(vec_ty); + return quote! { + #method_sig { + #result + } + }; + } + if self.has_wide_byte_mask_from_bitmask(vec_ty) { return self.kernel_method(op, vec_ty, |token| { mask_from_bitmask_wide_bytes(self.native_width(), vec_ty, token) @@ -679,6 +1118,17 @@ impl X86 { "mask bitmask conversion only operates on masks" ); + if *self == Self::Avx512 { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let bits = avx512_mask_bits_expr(quote! { a }); + let method_sig = op.simd_trait_method_sig(vec_ty); + return quote! { + #method_sig { + #bits & #lane_mask + } + }; + } + match vec_ty.scalar_bits { 8 => { let bits_ty = vec_ty.reinterpret(ScalarType::Int, 8); @@ -710,6 +1160,51 @@ impl X86 { } pub(crate) fn handle_compare(&self, op: Op, method: &str, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 { + if vec_ty.scalar == ScalarType::Mask { + let method_sig = op.simd_trait_method_sig(vec_ty); + let expr = avx512_mask_compare_expr(method, vec_ty); + let result = avx512_mask_value(vec_ty, expr); + return quote! { + #method_sig { + #result + } + }; + } + + return self.kernel_method(op, vec_ty, |token| { + let mask_ty = vec_ty.mask_ty(); + if vec_ty.scalar == ScalarType::Float { + let predicate = avx512_float_compare_predicate(method); + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, false); + let intrinsic = + intrinsic_ident("cmp", &format!("{suffix}_mask"), vec_ty.n_bits()); + avx512_mask_register_value_with_simd( + &mask_ty, + quote! { #intrinsic::<#predicate>(a.into(), b.into()) }, + quote! { #token }, + ) + } else { + let cmp = avx512_compare_op(method); + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, true); + let intrinsic = + intrinsic_ident(cmp, &format!("{suffix}_mask"), vec_ty.n_bits()); + avx512_mask_register_value_with_simd( + &mask_ty, + quote! { #intrinsic(a.into(), b.into()) }, + quote! { #token }, + ) + } + }); + } + + if vec_ty.scalar_bits == 64 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) + && method != "simd_eq" + { + return self.kernel_method(op, vec_ty, |token| scalar_compare(method, vec_ty, token)); + } + let args = [quote! { a.into() }, quote! { b.into() }]; let expr = if vec_ty.scalar != ScalarType::Float { @@ -790,6 +1285,60 @@ impl X86 { method: &str, vec_ty: &VecType, ) -> TokenStream { + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let body = match method { + "not" => { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let bits = avx512_mask_bits_expr(quote! { a }); + let result = avx512_mask_value(vec_ty, quote! { (!#bits) & #lane_mask }); + quote! { #result } + } + _ => unreachable!(), + }; + return quote! { + #method_sig { + #body + } + }; + } + + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Float { + match method { + "floor" | "ceil" | "round_ties_even" | "trunc" if vec_ty.n_bits() == 512 => { + let intrinsic = intrinsic_ident( + "roundscale", + op_suffix(vec_ty.scalar, vec_ty.scalar_bits, true), + vec_ty.n_bits(), + ); + let rounding_mode = match method { + "floor" => quote! { _MM_FROUND_TO_NEG_INF }, + "ceil" => quote! { _MM_FROUND_TO_POS_INF }, + "round_ties_even" => quote! { _MM_FROUND_TO_NEAREST_INT }, + "trunc" => quote! { _MM_FROUND_TO_ZERO }, + _ => unreachable!(), + }; + return self.kernel_method(op, vec_ty, |token| { + quote! { + #intrinsic::<{ #rounding_mode | _MM_FROUND_NO_EXC }>(a.into()).simd_into(#token) + } + }); + } + "approximate_recip" => { + let intrinsic = intrinsic_ident( + "rcp14", + op_suffix(vec_ty.scalar, vec_ty.scalar_bits, true), + vec_ty.n_bits(), + ); + return self.kernel_method(op, vec_ty, |token| { + quote! { + #intrinsic(a.into()).simd_into(#token) + } + }); + } + _ => {} + } + } + match method { "fract" => { let trunc_op = generic_op_name("trunc", vec_ty); @@ -843,7 +1392,18 @@ impl X86 { self.kernel_method(op, vec_ty, |token| match method { "widen" => { match (self, dst_width, vec_ty.n_bits()) { - (Self::Avx2, 256, 128) => { + (Self::Avx2 | Self::Avx512, 256, 128) => { + let extend = extend_intrinsic( + vec_ty.scalar, + vec_ty.scalar_bits, + target_ty.scalar_bits, + dst_width, + ); + quote! { + #extend(a.into()).simd_into(#token) + } + } + (Self::Avx512, 512, 256) => { let extend = extend_intrinsic( vec_ty.scalar, vec_ty.scalar_bits, @@ -898,6 +1458,12 @@ impl X86 { } "narrow" => { match (self, dst_width, vec_ty.n_bits()) { + (Self::Avx512, 128, 256) | (Self::Avx512, 256, 512) => { + let narrow = intrinsic_ident("cvtepi16", "epi8", vec_ty.n_bits()); + quote! { + #narrow(a.into()).simd_into(#token) + } + } (Self::Avx2, 128, 256) => { let mask = match target_ty.scalar_bits { 8 => { @@ -971,12 +1537,88 @@ impl X86 { pub(crate) fn handle_binary(&self, op: Op, method: &str, vec_ty: &VecType) -> TokenStream { let method_sig = op.simd_trait_method_sig(vec_ty); - match method { - "shlv" | "shrv" if !(*self == Self::Avx2 && vec_ty.scalar_bits >= 32) => { - // SSE2 has shift operations, but they shift every lane by the same amount, so we can't use them here. - let body = match method { - "shlv" => scalar_binary(quote!(core::ops::Shl::shl)), - "shrv" => scalar_binary(quote!(core::ops::Shr::shr)), + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let a_bits = avx512_mask_bits_expr(quote! { a }); + let b_bits = avx512_mask_bits_expr(quote! { b }); + let expr = match method { + "and" => quote! { (#a_bits & #b_bits) & #lane_mask }, + "or" => quote! { (#a_bits | #b_bits) & #lane_mask }, + "xor" => quote! { (#a_bits ^ #b_bits) & #lane_mask }, + _ => unreachable!(), + }; + let result = avx512_mask_value(vec_ty, expr); + return quote! { + #method_sig { + #result + } + }; + } + + if *self == Self::Avx512 + && vec_ty.scalar == ScalarType::Float + && matches!(method, "min_precise" | "max_precise") + { + let suffix = op_suffix(vec_ty.scalar, vec_ty.scalar_bits, true); + let range = intrinsic_ident("range", suffix, vec_ty.n_bits()); + let imm = if method == "max_precise" { + 0b0101 + } else { + 0b0100 + }; + return self.kernel_method(op, vec_ty, |token| { + quote! { + #range::<#imm>(a.into(), b.into()).simd_into(#token) + } + }); + } + + if *self != Self::Avx512 + && vec_ty.scalar_bits == 64 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) + && matches!(method, "mul" | "min" | "max") + { + let body = if method == "mul" { + scalar_binary_method("wrapping_mul", vec_ty, quote! { self }) + } else { + scalar_binary_method(method, vec_ty, quote! { self }) + }; + return quote! { + #method_sig { + #body + } + }; + } + + match method { + "shrv" + if *self != Self::Avx512 + && vec_ty.scalar == ScalarType::Int + && vec_ty.scalar_bits == 64 => + { + let body = scalar_binary(quote!(core::ops::Shr::shr), vec_ty, quote! { self }); + quote! { + #method_sig { + #body + } + } + } + "shlv" | "shrv" + if *self == Self::Avx512 + && matches!(vec_ty.scalar, ScalarType::Int | ScalarType::Unsigned) + && matches!(vec_ty.scalar_bits, 8 | 16) => + { + self.kernel_method(op, vec_ty, |token| { + self.handle_avx512_narrow_variable_shift(method, vec_ty, token) + }) + } + "shlv" | "shrv" + if !(matches!(self, Self::Avx2 | Self::Avx512) && vec_ty.scalar_bits >= 32) => + { + // SSE2 has shift operations, but they shift every lane by the same amount, so we can't use them here. + let body = match method { + "shlv" => scalar_binary(quote!(core::ops::Shl::shl), vec_ty, quote! { self }), + "shrv" => scalar_binary(quote!(core::ops::Shr::shr), vec_ty, quote! { self }), _ => unreachable!(), }; quote! { @@ -1025,7 +1667,84 @@ impl X86 { } } + fn handle_avx512_narrow_variable_shift( + &self, + method: &str, + vec_ty: &VecType, + token: &Ident, + ) -> TokenStream { + assert!( + *self == Self::Avx512, + "narrow variable shifts are specialized for AVX-512" + ); + assert!( + matches!(vec_ty.scalar_bits, 8 | 16), + "narrow variable shifts only handle 8-bit and 16-bit lanes" + ); + let name = match (method, vec_ty.scalar) { + ("shrv", ScalarType::Int) => "srav", + ("shrv", _) => "srlv", + ("shlv", _) => "sllv", + _ => unreachable!(), + }; + let shift_intrinsic = intrinsic_ident(name, "epi16", vec_ty.n_bits()); + + if vec_ty.scalar_bits == 16 { + return quote! { + #shift_intrinsic(a.into(), b.into()).simd_into(#token) + }; + } + + let ty_bits = vec_ty.n_bits(); + let unpack_hi = unpack_intrinsic(ScalarType::Int, 8, false, ty_bits); + let unpack_lo = unpack_intrinsic(ScalarType::Int, 8, true, ty_bits); + let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), ty_bits); + let and = intrinsic_ident("and", coarse_type(vec_ty), ty_bits); + let set1_epi16 = intrinsic_ident("set1", "epi16", ty_bits); + let pack = pack_intrinsic(16, false, ty_bits); + let value_extend = match (method, vec_ty.scalar) { + ("shlv", _) | (_, ScalarType::Unsigned) => quote! { zero }, + ("shrv", ScalarType::Int) if ty_bits == 512 => { + quote! { _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(zero, val)) } + } + ("shrv", ScalarType::Int) => { + let cmpgt = intrinsic_ident("cmpgt", "epi8", ty_bits); + quote! { #cmpgt(zero, val) } + } + _ => unreachable!(), + }; + + quote! { + let val = a.into(); + let counts = b.into(); + let zero = #set0(); + let value_extend = #value_extend; + let lo_values = #unpack_lo(val, value_extend); + let hi_values = #unpack_hi(val, value_extend); + let lo_counts = #unpack_lo(counts, zero); + let hi_counts = #unpack_hi(counts, zero); + let byte_mask = #set1_epi16(0x00ff); + let lo_shifted = #and(#shift_intrinsic(lo_values, lo_counts), byte_mask); + let hi_shifted = #and(#shift_intrinsic(hi_values, hi_counts), byte_mask); + #pack(lo_shifted, hi_shifted).simd_into(#token) + } + } + pub(crate) fn handle_shift(&self, op: Op, method: &str, vec_ty: &VecType) -> TokenStream { + let method_sig = op.simd_trait_method_sig(vec_ty); + if *self != Self::Avx512 + && method == "shr" + && vec_ty.scalar == ScalarType::Int + && vec_ty.scalar_bits == 64 + { + let body = scalar_shift(quote!(core::ops::Shr::shr), vec_ty, quote! { self }); + return quote! { + #method_sig { + #body + } + }; + } + let shift_op = match (method, vec_ty.scalar) { ("shr", ScalarType::Unsigned) => "srl", ("shr", ScalarType::Int) => "sra", @@ -1048,9 +1767,16 @@ impl X86 { #expr(val, #set0()) }, ScalarType::Int => { - let cmp_intrinsic = intrinsic_ident("cmpgt", "epi8", ty_bits); + let sign_bits = if *self == Self::Avx512 && ty_bits == 512 { + quote! { + _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(#set0(), val)) + } + } else { + let cmp_intrinsic = intrinsic_ident("cmpgt", "epi8", ty_bits); + quote! { #cmp_intrinsic(#set0(), val) } + }; quote! { - #expr(val, #cmp_intrinsic(#set0(), val)) + #expr(val, #sign_bits) } } _ => unimplemented!(), @@ -1093,7 +1819,7 @@ impl X86 { vec_ty: &VecType, ) -> TokenStream { match method { - "mul_add" if *self == Self::Avx2 => { + "mul_add" if matches!(self, Self::Avx2 | Self::Avx512) => { let intrinsic = simple_intrinsic("fmadd", vec_ty); self.kernel_method( op, @@ -1101,7 +1827,7 @@ impl X86 { |token| quote! { #intrinsic(a.into(), b.into(), c.into()).simd_into(#token) }, ) } - "mul_sub" if *self == Self::Avx2 => { + "mul_sub" if matches!(self, Self::Avx2 | Self::Avx512) => { let intrinsic = simple_intrinsic("fmsub", vec_ty); self.kernel_method( op, @@ -1139,6 +1865,32 @@ impl X86 { } pub(crate) fn handle_select(&self, op: Op, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 { + let method_sig = op.simd_trait_method_sig(vec_ty); + if vec_ty.scalar == ScalarType::Mask { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let a_bits = avx512_mask_bits_expr(quote! { a }); + let b_bits = avx512_mask_bits_expr(quote! { b }); + let c_bits = avx512_mask_bits_expr(quote! { c }); + let result = avx512_mask_value( + vec_ty, + quote! { ((#a_bits & #b_bits) | ((!#a_bits) & #c_bits)) & #lane_mask }, + ); + return quote! { + #method_sig { + #result + } + }; + } + + let blend = avx512_mask_blend_intrinsic(vec_ty); + return self.kernel_method(op, vec_ty, |token| { + quote! { + #blend(a.val, c.into(), b.into()).simd_into(#token) + } + }); + } + // Our select ops' argument order is mask, a, b; Intel's intrinsics are b, a, mask let args = [ quote! { c.into() }, @@ -1165,7 +1917,48 @@ impl X86 { } pub(crate) fn handle_split(&self, op: Op, vec_ty: &VecType, half_ty: &VecType) -> TokenStream { - if *self == Self::Avx2 && half_ty.n_bits() == 128 { + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let method_sig = op.simd_trait_method_sig(vec_ty); + let half_rust = half_ty.rust(); + let half_len = half_ty.len; + let half_mask = avx512_mask_lane_bits(half_ty); + return quote! { + #method_sig { + let bits = u64::from(a.val); + ( + #half_rust { val: (bits & #half_mask) as _, simd: self }, + #half_rust { val: ((bits >> #half_len) & #half_mask) as _, simd: self }, + ) + } + }; + } + + if *self == Self::Avx512 && half_ty.n_bits() == 256 { + let (lo, hi) = match vec_ty.scalar { + ScalarType::Float if vec_ty.scalar_bits == 32 => ( + quote! { _mm512_castps512_ps256(a.into()) }, + quote! { _mm512_extractf32x8_ps::<1>(a.into()) }, + ), + ScalarType::Float if vec_ty.scalar_bits == 64 => ( + quote! { _mm512_castpd512_pd256(a.into()) }, + quote! { _mm512_extractf64x4_pd::<1>(a.into()) }, + ), + _ => ( + quote! { _mm512_castsi512_si256(a.into()) }, + quote! { _mm512_extracti64x4_epi64::<1>(a.into()) }, + ), + }; + return self.kernel_method(op, vec_ty, |token| { + quote! { + ( + #lo.simd_into(#token), + #hi.simd_into(#token), + ) + } + }); + } + + if matches!(self, Self::Avx2 | Self::Avx512) && half_ty.n_bits() == 128 { let extract_op = match vec_ty.scalar { ScalarType::Float => "extractf128", _ => "extracti128", @@ -1191,7 +1984,44 @@ impl X86 { vec_ty: &VecType, combined_ty: &VecType, ) -> TokenStream { - if *self == Self::Avx2 && combined_ty.n_bits() == 256 { + let method_sig = op.simd_trait_method_sig(vec_ty); + if *self == Self::Avx512 && vec_ty.scalar == ScalarType::Mask { + let combined_rust = combined_ty.rust(); + let shift = vec_ty.len; + let lane_mask = avx512_mask_lane_bits(combined_ty); + let bits = if avx512_mask_register_bits(combined_ty) == 64 { + quote! { bits } + } else { + quote! { bits as _ } + }; + return quote! { + #method_sig { + let bits = (u64::from(a.val) | (u64::from(b.val) << #shift)) & #lane_mask; + #combined_rust { val: #bits, simd: self } + } + }; + } + + if *self == Self::Avx512 && combined_ty.n_bits() == 512 { + let expr = match vec_ty.scalar { + ScalarType::Float if vec_ty.scalar_bits == 32 => quote! { + _mm512_insertf32x8::<1>(_mm512_castps256_ps512(a.into()), b.into()) + }, + ScalarType::Float if vec_ty.scalar_bits == 64 => quote! { + _mm512_insertf64x4::<1>(_mm512_castpd256_pd512(a.into()), b.into()) + }, + _ => quote! { + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()) + }, + }; + return self.kernel_method(op, vec_ty, |token| { + quote! { + #expr.simd_into(#token) + } + }); + } + + if matches!(self, Self::Avx2 | Self::Avx512) && combined_ty.n_bits() == 256 { let suffix = match (vec_ty.scalar, vec_ty.scalar_bits) { (ScalarType::Float, 32) => "m128", (ScalarType::Float, 64) => "m128d", @@ -1204,12 +2034,30 @@ impl X86 { |token| quote! { #set_intrinsic(a.into(), b.into()).simd_into(#token) }, ) } else { - let method_sig = op.simd_trait_method_sig(vec_ty); generic_block_combine(method_sig, combined_ty, self.max_block_size()) } } pub(crate) fn handle_zip(&self, op: Op, vec_ty: &VecType, select_low: bool) -> TokenStream { + if *self == Self::Avx512 && vec_ty.scalar != ScalarType::Mask && vec_ty.n_bits() >= 256 { + let offset = if select_low { 0 } else { vec_ty.len / 2 }; + let indices = (0..vec_ty.len).map(|i| { + let source_lane = offset + (i / 2); + if i % 2 == 0 { + source_lane + } else { + vec_ty.len + source_lane + } + }); + let idx = avx512_index_vector(vec_ty, indices); + let permute = avx512_permutex2var_intrinsic(vec_ty); + return self.kernel_method(op, vec_ty, |token| { + quote! { + #permute(a.into(), #idx, b.into()).simd_into(#token) + } + }); + } + self.kernel_method(op, vec_ty, |token| match vec_ty.n_bits() { 128 => { let op = if select_low { "unpacklo" } else { "unpackhi" }; @@ -1251,6 +2099,38 @@ impl X86 { } pub(crate) fn handle_interleave(&self, op: Op, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 && vec_ty.scalar != ScalarType::Mask && vec_ty.n_bits() >= 256 { + let lo_indices = (0..vec_ty.len).map(|i| { + let source_lane = i / 2; + if i % 2 == 0 { + source_lane + } else { + vec_ty.len + source_lane + } + }); + let hi_indices = (0..vec_ty.len).map(|i| { + let source_lane = (vec_ty.len / 2) + (i / 2); + if i % 2 == 0 { + source_lane + } else { + vec_ty.len + source_lane + } + }); + let lo_idx = avx512_index_vector(vec_ty, lo_indices); + let hi_idx = avx512_index_vector(vec_ty, hi_indices); + let permute = avx512_permutex2var_intrinsic(vec_ty); + return self.kernel_method(op, vec_ty, |token| { + quote! { + let a = a.into(); + let b = b.into(); + ( + #permute(a, #lo_idx, b).simd_into(#token), + #permute(a, #hi_idx, b).simd_into(#token), + ) + } + }); + } + match vec_ty.n_bits() { 256 => { // Optimized path: compute unpacklo and unpackhi once, then use permute2f128 to @@ -1294,6 +2174,36 @@ impl X86 { } pub(crate) fn handle_deinterleave(&self, op: Op, vec_ty: &VecType) -> TokenStream { + if *self == Self::Avx512 && avx512_should_use_unzip_permutex2var(vec_ty) { + let even_indices = (0..vec_ty.len).map(|i| { + if i < vec_ty.len / 2 { + i * 2 + } else { + vec_ty.len + ((i - vec_ty.len / 2) * 2) + } + }); + let odd_indices = (0..vec_ty.len).map(|i| { + if i < vec_ty.len / 2 { + i * 2 + 1 + } else { + vec_ty.len + ((i - vec_ty.len / 2) * 2 + 1) + } + }); + let even_idx = avx512_index_vector(vec_ty, even_indices); + let odd_idx = avx512_index_vector(vec_ty, odd_indices); + let permute = avx512_permutex2var_intrinsic(vec_ty); + return self.kernel_method(op, vec_ty, |token| { + quote! { + let a = a.into(); + let b = b.into(); + ( + #permute(a, #even_idx, b).simd_into(#token), + #permute(a, #odd_idx, b).simd_into(#token), + ) + } + }); + } + match vec_ty.n_bits() { 256 => { // Optimized path: compute the per-input shuffles once, then use permute2f128 / @@ -1380,6 +2290,24 @@ impl X86 { } pub(crate) fn handle_unzip(&self, op: Op, vec_ty: &VecType, select_even: bool) -> TokenStream { + if *self == Self::Avx512 && avx512_should_use_unzip_permutex2var(vec_ty) { + let lane_offset = if select_even { 0 } else { 1 }; + let indices = (0..vec_ty.len).map(|i| { + if i < vec_ty.len / 2 { + i * 2 + lane_offset + } else { + vec_ty.len + ((i - vec_ty.len / 2) * 2 + lane_offset) + } + }); + let idx = avx512_index_vector(vec_ty, indices); + let permute = avx512_permutex2var_intrinsic(vec_ty); + return self.kernel_method(op, vec_ty, |token| { + quote! { + #permute(a.into(), #idx, b.into()).simd_into(#token) + } + }); + } + self.kernel_method(op, vec_ty, |token| { match (vec_ty.scalar, vec_ty.n_bits(), vec_ty.scalar_bits) { (ScalarType::Float, 128, _) => { @@ -1397,6 +2325,14 @@ impl X86 { quote! { #intrinsic::<#mask>(a.into(), b.into()).simd_into(#token) } } + (ScalarType::Int | ScalarType::Mask | ScalarType::Unsigned, 128, 64) => { + let op = if select_even { "unpacklo" } else { "unpackhi" }; + let intrinsic = intrinsic_ident(op, "epi64", vec_ty.n_bits()); + + quote! { + #intrinsic(a.into(), b.into()).simd_into(#token) + } + } (ScalarType::Int | ScalarType::Mask | ScalarType::Unsigned, 128, 32) => { // 128-bit shuffle of 32-bit integers; unlike with floats, there is no single shuffle instruction that // combines two vectors @@ -1476,6 +2412,74 @@ impl X86 { let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + if *self == Self::Avx512 + && granularity == WithinBlocks + && vec_ty.scalar != ScalarType::Mask + && vec_ty.n_bits() >= 256 + { + let alignr = format_ident!("dyn_alignr_{}", vec_ty.n_bits()); + let byte_shift = if scalar_bytes == 1 { + quote! { SHIFT } + } else { + quote! { SHIFT * #scalar_bytes } + }; + + return quote! { + #method_sig { + if SHIFT == 0 { + return a; + } + if SHIFT >= #max_shift { + return b; + } + + let a = self.#to_bytes(a).val.0; + let b = self.#to_bytes(b).val.0; + let result = #alignr(self, b, a, #byte_shift); + self.#from_bytes(#combined_bytes { val: #block_wrapper(result), simd: self }) + } + }; + } + + if *self == Self::Avx512 && granularity == AcrossBlocks && vec_ty.n_bits() >= 256 { + let level = self.token(); + let ty = vec_ty.rust(); + let vec = quote! { #ty<#level> }; + let byte_ty = vec_ty.reinterpret(ScalarType::Unsigned, 8); + let base_idx = avx512_index_vector(&byte_ty, 0..byte_ty.len); + let set_shift = set1_intrinsic(&byte_ty); + let add = simple_sign_unaware_intrinsic("add", &byte_ty); + let permute = avx512_permutex2var_intrinsic(&byte_ty); + let byte_shift = if scalar_bytes == 1 { + quote! { shift } + } else { + quote! { shift * #scalar_bytes } + }; + + return quote! { + #method_sig { + crate::kernel!( + #[inline(always)] + fn kernel(token: #level, a: #vec, b: #vec, shift: usize) -> #vec { + if shift >= #max_shift { + return b; + } + + let idx = #add(#base_idx, #set_shift((#byte_shift) as i8)); + let result = #permute( + token.#to_bytes(a).val.0, + idx, + token.#to_bytes(b).val.0, + ); + token.#from_bytes(#combined_bytes { val: #block_wrapper(result), simd: token }) + } + ); + + kernel(self, a, b, SHIFT) + } + }; + } + let alignr_op = match (granularity, vec_ty.n_bits(), self) { (WithinBlocks, 128, _) => { panic!("This should have been handled by generic_op"); @@ -1527,6 +2531,116 @@ impl X86 { vec_ty.scalar_bits, target_scalar_bits, "we currently only support converting between types of the same width" ); + + if *self == Self::Avx512 + && vec_ty.scalar == ScalarType::Float + && target_scalar == ScalarType::Unsigned + { + let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); + let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); + return self.kernel_method(op, vec_ty, |token| { + if precise { + let max = simple_intrinsic("max", vec_ty); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_float = + intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + quote! { + let a = #max(a.into(), #set0_float()); + let mut converted = #convert(a); + let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); + converted = #blend( + exceeds_unsigned_range, + converted, + #set1_int(u32::MAX.cast_signed()), + ); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } + } + }); + } + + if *self == Self::Avx512 + && vec_ty.scalar == ScalarType::Float + && target_scalar == ScalarType::Int + && vec_ty.scalar_bits == 32 + { + let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); + let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); + return self.kernel_method(op, vec_ty, |token| { + if precise { + let masked_convert = intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_int = + intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + let ord = avx512_float_compare_predicate("ord"); + quote! { + let a = a.into(); + let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); + let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); + let is_not_nan = #cmp::<#ord>(a, a); + converted = #blend(is_not_nan, #set0_int(), converted); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } + } + }); + } + + if *self == Self::Avx512 + && matches!(vec_ty.n_bits(), 128 | 256) + && vec_ty.scalar == ScalarType::Unsigned + && target_scalar == ScalarType::Float + && vec_ty.scalar_bits == 32 + { + // We cannot emit the intrinsics for the conversion instructions + // because the required intrinsics are mysteriously absent from stdarch: + // https://github.com/rust-lang/rust/issues/158196 + // Fortunately LLVM optimizes this sequence into the single instruction we're after. + let bits = vec_ty.n_bits(); + let zext = format_ident!("_mm512_zextsi{bits}_si512"); + let convert = intrinsic_ident("cvtepu32", "ps", 512); + let cast = format_ident!("_mm512_castps512_ps{bits}"); + return self.kernel_method(op, vec_ty, |token| { + quote! { + #cast(#convert(#zext(a.into()))).simd_into(#token) + } + }); + } + + if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); + return self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { + (ScalarType::Int, ScalarType::Float) => { + let intrinsic = simple_intrinsic("cvtepi32", &target_ty); + quote! { + #intrinsic(a.into()).simd_into(#token) + } + } + (ScalarType::Unsigned, ScalarType::Float) => { + let intrinsic = simple_intrinsic("cvtepu32", &target_ty); + quote! { + #intrinsic(a.into()).simd_into(#token) + } + } + _ => unimplemented!(), + }); + } + self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { (ScalarType::Float, ScalarType::Int | ScalarType::Unsigned) => { let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); @@ -1731,6 +2845,24 @@ impl X86 { "mask reduce ops only operate on masks" ); + if *self == Self::Avx512 { + let lane_mask = avx512_mask_lane_bits(vec_ty); + let bits = avx512_mask_bits_expr(quote! { a }); + let expr = match (quantifier, condition) { + (Quantifier::Any, true) => quote! { bits != 0 }, + (Quantifier::Any, false) => quote! { bits != #lane_mask }, + (Quantifier::All, true) => quote! { bits == #lane_mask }, + (Quantifier::All, false) => quote! { bits == 0 }, + }; + let method_sig = method_op.simd_trait_method_sig(vec_ty); + return quote! { + #method_sig { + let bits = #bits & #lane_mask; + #expr + } + }; + } + let (movemask, all_ones) = match vec_ty.scalar_bits { 32 | 64 => { let float_ty = vec_ty.cast(ScalarType::Float); @@ -1789,10 +2921,18 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); + if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + return self.handle_avx512_load_interleaved(op, vec_ty, block_size, block_count); + } match vec_ty.scalar_bits { - 32 | 16 | 8 => { - let block_ty = - VecType::new(vec_ty.scalar, vec_ty.scalar_bits, 128 / vec_ty.scalar_bits); + 64 | 32 | 16 | 8 => { + let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_u64 { + 4 + } else { + block_size as usize / vec_ty.scalar_bits + }; + let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); @@ -1812,7 +2952,24 @@ impl X86 { &format!("combine_{}", vec_combined.rust_name()), Span::call_site(), ); - let block_len = block_size as usize / vec_ty.scalar_bits; + if avx2_u64 { + return self.kernel_method(op, vec_ty, |token| { + quote! { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: #native_ty = crate::transmute::checked_transmute_copy::<[#scalar_ty; 4], #native_ty>(&chunks[0]); + let v1: #native_ty = crate::transmute::checked_transmute_copy::<[#scalar_ty; 4], #native_ty>(&chunks[1]); + + let lo = #unpacklo_64(v0, v1); // [0,4,2,6] + let hi = #unpackhi_64(v0, v1); // [1,5,3,7] + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); // [0,4,1,5] + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); // [2,6,3,7] + + #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)) + } + }); + } let init_shuffle = match vec_ty.scalar_bits { 16 => Some(quote! { @@ -1842,36 +2999,53 @@ impl X86 { _ => None, }; - let final_unpack = if vec_ty.scalar == ScalarType::Float && vec_ty.scalar_bits == 32 - { - let cast_32 = cast_ident( - ScalarType::Float, - ScalarType::Float, - 64, - 32, - block_ty.n_bits(), - ); - let cast_64 = cast_ident( - ScalarType::Float, - ScalarType::Float, - 32, - 64, - block_ty.n_bits(), - ); + let initial_unpack = if vec_ty.scalar_bits == 64 { + None + } else { + Some(quote! { + let tmp0 = #unpacklo_32(v0, v1); // [0,4,1,5] + let tmp1 = #unpackhi_32(v0, v1); // [2,6,3,7] + let tmp2 = #unpacklo_32(v2, v3); // [8,12,9,13] + let tmp3 = #unpackhi_32(v2, v3); // [10,14,11,15] + }) + }; - quote! { - let out0 = #cast_32(#unpacklo_64(#cast_64(tmp0), #cast_64(tmp2))); // [0,4,8,12] - let out1 = #cast_32(#unpackhi_64(#cast_64(tmp0), #cast_64(tmp2))); // [1,5,9,13] - let out2 = #cast_32(#unpacklo_64(#cast_64(tmp1), #cast_64(tmp3))); // [2,6,10,14] - let out3 = #cast_32(#unpackhi_64(#cast_64(tmp1), #cast_64(tmp3))); // [3,7,11,15] + let final_unpack = match (vec_ty.scalar, vec_ty.scalar_bits) { + (_, 64) => quote! { + let out0 = #unpacklo_64(v0, v2); // [0,4] + let out1 = #unpackhi_64(v0, v2); // [1,5] + let out2 = #unpacklo_64(v1, v3); // [2,6] + let out3 = #unpackhi_64(v1, v3); // [3,7] + }, + (ScalarType::Float, 32) => { + let cast_32 = cast_ident( + ScalarType::Float, + ScalarType::Float, + 64, + 32, + block_ty.n_bits(), + ); + let cast_64 = cast_ident( + ScalarType::Float, + ScalarType::Float, + 32, + 64, + block_ty.n_bits(), + ); + + quote! { + let out0 = #cast_32(#unpacklo_64(#cast_64(tmp0), #cast_64(tmp2))); // [0,4,8,12] + let out1 = #cast_32(#unpackhi_64(#cast_64(tmp0), #cast_64(tmp2))); // [1,5,9,13] + let out2 = #cast_32(#unpacklo_64(#cast_64(tmp1), #cast_64(tmp3))); // [2,6,10,14] + let out3 = #cast_32(#unpackhi_64(#cast_64(tmp1), #cast_64(tmp3))); // [3,7,11,15] + } } - } else { - quote! { + _ => quote! { let out0 = #unpacklo_64(tmp0, tmp2); // [0,4,8,12] let out1 = #unpackhi_64(tmp0, tmp2); // [1,5,9,13] let out2 = #unpacklo_64(tmp1, tmp3); // [2,6,10,14] let out3 = #unpackhi_64(tmp1, tmp3); // [3,7,11,15] - } + }, }; self.kernel_method(op, vec_ty, |token| { @@ -1894,11 +3068,7 @@ impl X86 { #init_shuffle - let tmp0 = #unpacklo_32(v0, v1); // [0,4,1,5] - let tmp1 = #unpackhi_32(v0, v1); // [2,6,3,7] - let tmp2 = #unpacklo_32(v2, v3); // [8,12,9,13] - let tmp3 = #unpackhi_32(v2, v3); // [10,14,11,15] - + #initial_unpack #final_unpack #token.#combine_full( @@ -1912,6 +3082,43 @@ impl X86 { } } + pub(crate) fn handle_avx512_load_interleaved( + &self, + op: Op, + vec_ty: &VecType, + block_size: u16, + block_count: u16, + ) -> TokenStream { + assert_eq!( + block_size, 128, + "only 128-bit blocks are currently supported" + ); + assert_eq!(block_count, 4, "only count of 4 is currently supported"); + assert_eq!( + vec_ty.n_bits(), + 512, + "AVX-512 interleaved loads only specialize 512-bit vectors" + ); + let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); + let native_ty = self.arch_ty(vec_ty); + let len = vec_ty.len; + let permute = avx512_permutexvar_intrinsic(vec_ty); + let indices = avx512_index_vector( + vec_ty, + interleaved_load_indices(vec_ty.len, block_count as usize), + ); + + self.kernel_method(op, vec_ty, |token| { + quote! { + let lanes: #native_ty = + crate::transmute::checked_transmute_copy::<[#scalar_ty; #len], #native_ty>( + src, + ); + #permute(#indices, lanes).simd_into(#token) + } + }) + } + pub(crate) fn handle_store_interleaved( &self, op: Op, @@ -1924,10 +3131,18 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); + if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + return self.handle_avx512_store_interleaved(op, vec_ty, block_size, block_count); + } match vec_ty.scalar_bits { - 32 | 16 | 8 => { - let block_ty = - VecType::new(vec_ty.scalar, vec_ty.scalar_bits, 128 / vec_ty.scalar_bits); + 64 | 32 | 16 | 8 => { + let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_u64 { + 4 + } else { + block_size as usize / vec_ty.scalar_bits + }; + let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); @@ -1945,7 +3160,28 @@ impl X86 { ); let split_full = Ident::new(&format!("split_{}", vec_ty.rust_name()), Span::call_site()); - let block_len = block_size as usize / vec_ty.scalar_bits; + + if avx2_u64 { + return self.kernel_method(op, vec_ty, |token| { + quote! { + let (v0, v1) = #token.#split_full(a); + let v0: #native_ty = v0.into(); + let v1: #native_ty = v1.into(); + + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); // [0,4,2,6] + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); // [1,5,3,7] + let out0 = #unpacklo_64(lo, hi); // [0,1,2,3] + let out1 = #unpackhi_64(lo, hi); // [4,5,6,7] + + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + + crate::transmute::checked_transmute_store::<#native_ty, [#scalar_ty; 4]>(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::<#native_ty, [#scalar_ty; 4]>(out1, &mut chunks[1]); + } + }); + } let post_shuffle = match vec_ty.scalar_bits { 16 => Some(quote! { @@ -1975,36 +3211,53 @@ impl X86 { _ => None, }; - let final_unpack = if vec_ty.scalar == ScalarType::Float && vec_ty.scalar_bits == 32 - { - let cast_32 = cast_ident( - ScalarType::Float, - ScalarType::Float, - 64, - 32, - block_ty.n_bits(), - ); - let cast_64 = cast_ident( - ScalarType::Float, - ScalarType::Float, - 32, - 64, - block_ty.n_bits(), - ); + let initial_unpack = if vec_ty.scalar_bits == 64 { + None + } else { + Some(quote! { + let tmp0 = #unpacklo_32(v0, v1); // [0,4,1,5] + let tmp1 = #unpackhi_32(v0, v1); // [2,6,3,7] + let tmp2 = #unpacklo_32(v2, v3); // [8,12,9,13] + let tmp3 = #unpackhi_32(v2, v3); // [10,14,11,15] + }) + }; - quote! { - let out0 = #cast_32(#unpacklo_64(#cast_64(tmp0), #cast_64(tmp2))); // [0,4,8,12] - let out1 = #cast_32(#unpackhi_64(#cast_64(tmp0), #cast_64(tmp2))); // [1,5,9,13] - let out2 = #cast_32(#unpacklo_64(#cast_64(tmp1), #cast_64(tmp3))); // [2,6,10,14] - let out3 = #cast_32(#unpackhi_64(#cast_64(tmp1), #cast_64(tmp3))); // [3,7,11,15] + let final_unpack = match (vec_ty.scalar, vec_ty.scalar_bits) { + (_, 64) => quote! { + let out0 = #unpacklo_64(v0, v1); // [0,1] + let out1 = #unpacklo_64(v2, v3); // [2,3] + let out2 = #unpackhi_64(v0, v1); // [4,5] + let out3 = #unpackhi_64(v2, v3); // [6,7] + }, + (ScalarType::Float, 32) => { + let cast_32 = cast_ident( + ScalarType::Float, + ScalarType::Float, + 64, + 32, + block_ty.n_bits(), + ); + let cast_64 = cast_ident( + ScalarType::Float, + ScalarType::Float, + 32, + 64, + block_ty.n_bits(), + ); + + quote! { + let out0 = #cast_32(#unpacklo_64(#cast_64(tmp0), #cast_64(tmp2))); // [0,4,8,12] + let out1 = #cast_32(#unpackhi_64(#cast_64(tmp0), #cast_64(tmp2))); // [1,5,9,13] + let out2 = #cast_32(#unpacklo_64(#cast_64(tmp1), #cast_64(tmp3))); // [2,6,10,14] + let out3 = #cast_32(#unpackhi_64(#cast_64(tmp1), #cast_64(tmp3))); // [3,7,11,15] + } } - } else { - quote! { + _ => quote! { let out0 = #unpacklo_64(tmp0, tmp2); // [0,4,8,12] let out1 = #unpackhi_64(tmp0, tmp2); // [1,5,9,13] let out2 = #unpacklo_64(tmp1, tmp3); // [2,6,10,14] let out3 = #unpackhi_64(tmp1, tmp3); // [3,7,11,15] - } + }, }; self.kernel_method(op, vec_ty, |token| { @@ -2017,11 +3270,7 @@ impl X86 { let v2 = v2.into(); let v3 = v3.into(); - let tmp0 = #unpacklo_32(v0, v1); // [0,4,1,5] - let tmp1 = #unpackhi_32(v0, v1); // [2,6,3,7] - let tmp2 = #unpacklo_32(v2, v3); // [8,12,9,13] - let tmp3 = #unpackhi_32(v2, v3); // [10,14,11,15] - + #initial_unpack #final_unpack #post_shuffle @@ -2041,6 +3290,43 @@ impl X86 { } } + pub(crate) fn handle_avx512_store_interleaved( + &self, + op: Op, + vec_ty: &VecType, + block_size: u16, + block_count: u16, + ) -> TokenStream { + assert_eq!( + block_size, 128, + "only 128-bit blocks are currently supported" + ); + assert_eq!(block_count, 4, "only count of 4 is currently supported"); + assert_eq!( + vec_ty.n_bits(), + 512, + "AVX-512 interleaved stores only specialize 512-bit vectors" + ); + let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); + let native_ty = self.arch_ty(vec_ty); + let len = vec_ty.len; + let permute = avx512_permutexvar_intrinsic(vec_ty); + let indices = avx512_index_vector( + vec_ty, + interleaved_store_indices(vec_ty.len, block_count as usize), + ); + + self.kernel_method(op, vec_ty, |_| { + quote! { + let lanes = #permute(#indices, a.into()); + crate::transmute::checked_transmute_store::<#native_ty, [#scalar_ty; #len]>( + lanes, + dest, + ); + } + }) + } + /// Generates versions of the "alignr" intrinsics that take the shift amount as a regular argument instead of a /// const generic argument, to make them easier to use in higher-level operations. These are low-level helpers that /// inherit the semantics of the underlying `alignr` intrinsics, so the argument order is backwards from ARM's @@ -2052,6 +3338,7 @@ impl X86 { let vec_widths: &[usize] = match self { Self::Sse4_2 => &[128], Self::Avx2 => &[128, 256], + Self::Avx512 => &[128, 256, 512], }; for vec_ty in vec_widths diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 5bc189412..9dcb1bcfe 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -110,6 +110,8 @@ pub(crate) enum OpSig { MaskFromBitmask, /// Takes a mask vector type and returns its compact bitmask representation. MaskToBitmask, + /// Takes a mutable mask vector, a lane index, and a boolean, and updates the lane in place. + MaskSet, /// Takes an argument of an array of a certain scalar type, with the length (`block_size` * `block_count`) / [scalar /// type's byte size]. Returns a vector type of that scalar type and length. /// @@ -321,6 +323,10 @@ impl Op { OpSig::MaskReduce { .. } => (vec![vec], quote! { bool }), OpSig::MaskFromBitmask => (vec![quote! { u64 }], vec), OpSig::MaskToBitmask => (vec![vec], quote! { u64 }), + OpSig::MaskSet => ( + vec![quote! { &mut #vec }, quote! { usize }, quote! { bool }], + quote! { () }, + ), OpSig::Shift => (vec![vec.clone(), quote! { u32 }], vec), OpSig::Ternary => (vec![vec.clone(), vec.clone(), vec.clone()], vec), OpSig::Select => { @@ -388,7 +394,7 @@ impl Op { OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } | OpSig::StoreArray => { return None; } - OpSig::MaskFromBitmask | OpSig::MaskToBitmask => return None, + OpSig::MaskFromBitmask | OpSig::MaskToBitmask | OpSig::MaskSet => return None, OpSig::Unary | OpSig::Cvt { .. } | OpSig::Reinterpret { .. } @@ -618,6 +624,12 @@ const MASK_REPRESENTATION_OPS: &[Op] = &[ OpSig::MaskToBitmask, "Convert a SIMD mask to a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are cleared.", ), + Op::new( + "set", + OpKind::AssociatedOnly, + OpSig::MaskSet, + "Set one logical lane of a SIMD mask.", + ), ]; const FLOAT_OPS: &[Op] = &[ @@ -1211,7 +1223,7 @@ pub(crate) const F32_TO_U32: Op = Op::new( }, "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\n\ Out-of-range values or NaN will produce implementation-defined results.\n\n\ - On x86 platforms, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32` (at least until AVX-512, which is currently not supported).\n\ + On x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\n\ If you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards.", ); pub(crate) const F32_TO_U32_PRECISE: Op = Op::new( @@ -1355,11 +1367,11 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { }, "Load elements from an array with 4-way interleaving.\n\n\ This is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded \ - vectors, while this operation treats memory as four consecutive 128-bit blocks and transposes those blocks \ + vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them \ into one vector.\n\n\ For example, with 32-bit lanes, memory laid out as \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` loads as \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`.", + `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as \ + `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`.", )); } @@ -1373,11 +1385,11 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { }, "Store elements to an array with 4-way interleaving.\n\n\ This is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: \ - `interleave` combines two already-loaded vectors, while this operation transposes one vector into four \ - consecutive 128-bit blocks in memory.\n\n\ + `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit \ + vectors into lane-interleaved memory.\n\n\ For example, with 32-bit lanes, a vector containing \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` stores as \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`.", + `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as \ + `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`.", )); } @@ -1558,6 +1570,7 @@ impl OpSig { | Self::FromArray { .. } | Self::AsArray { .. } | Self::StoreArray + | Self::MaskSet | Self::Slide { granularity: SlideGranularity::AcrossBlocks, .. @@ -1587,6 +1600,7 @@ impl OpSig { match self { Self::Splat | Self::FromArray { .. } => &["val"], Self::MaskFromBitmask => &["bits"], + Self::MaskSet => &["a", "index", "value"], Self::Unary | Self::Split { .. } | Self::Cvt { .. } @@ -1619,6 +1633,7 @@ impl OpSig { | Self::FromArray { .. } | Self::MaskFromBitmask | Self::MaskToBitmask + | Self::MaskSet | Self::FromBytes { .. } | Self::StoreArray => &[], Self::Unary @@ -1681,6 +1696,7 @@ impl OpSig { | Self::Shift | Self::MaskFromBitmask | Self::MaskToBitmask + | Self::MaskSet | Self::LoadInterleaved { .. } | Self::StoreInterleaved { .. } | Self::FromArray { .. } diff --git a/fearless_simd_gen/src/types.rs b/fearless_simd_gen/src/types.rs index 670d6a5cb..8296652c7 100644 --- a/fearless_simd_gen/src/types.rs +++ b/fearless_simd_gen/src/types.rs @@ -281,6 +281,8 @@ pub(crate) const SIMD_TYPES: &[VecType] = &[ VecType::new(ScalarType::Unsigned, 32, 4), VecType::new(ScalarType::Mask, 32, 4), VecType::new(ScalarType::Float, 64, 2), + VecType::new(ScalarType::Int, 64, 2), + VecType::new(ScalarType::Unsigned, 64, 2), VecType::new(ScalarType::Mask, 64, 2), // 256 bit types VecType::new(ScalarType::Float, 32, 8), @@ -294,6 +296,8 @@ pub(crate) const SIMD_TYPES: &[VecType] = &[ VecType::new(ScalarType::Unsigned, 32, 8), VecType::new(ScalarType::Mask, 32, 8), VecType::new(ScalarType::Float, 64, 4), + VecType::new(ScalarType::Int, 64, 4), + VecType::new(ScalarType::Unsigned, 64, 4), VecType::new(ScalarType::Mask, 64, 4), // 512 bit types VecType::new(ScalarType::Float, 32, 16), @@ -307,6 +311,8 @@ pub(crate) const SIMD_TYPES: &[VecType] = &[ VecType::new(ScalarType::Unsigned, 32, 16), VecType::new(ScalarType::Mask, 32, 16), VecType::new(ScalarType::Float, 64, 8), + VecType::new(ScalarType::Int, 64, 8), + VecType::new(ScalarType::Unsigned, 64, 8), VecType::new(ScalarType::Mask, 64, 8), ]; diff --git a/fearless_simd_tests/tests/harness/int64.rs b/fearless_simd_tests/tests/harness/int64.rs new file mode 100644 index 000000000..10a851604 --- /dev/null +++ b/fearless_simd_tests/tests/harness/int64.rs @@ -0,0 +1,1468 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +fn mask_lane(value: bool) -> i64 { + if value { -1 } else { 0 } +} + +#[simd_test] +fn construct_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let mut stored = [0_i64; 2]; + a.store_slice(&mut stored); + assert_eq!(stored, [-9, 18]); + assert_eq!(*i64x2::splat(simd, -9), [-9, -9]); + assert_eq!(*i64x2::simd_from(simd, [-9, 18]), [-9, 18]); + assert_eq!(*i64x2::from_fn(simd, |i| [-9, 18][i]), [-9, 18]); + assert_eq!(*i64x2::from_bytes(a.to_bytes()), [-9, 18]); +} + +#[simd_test] +fn construct_i64x4(simd: S) { + let vals = [-9, 18, i64::MAX - 7, i64::MIN + 9]; + let a = i64x4::from_slice(simd, &vals); + let mut stored = [0_i64; 4]; + a.store_slice(&mut stored); + assert_eq!(stored, vals); + assert_eq!(*i64x4::splat(simd, -9), [-9, -9, -9, -9]); + assert_eq!(*i64x4::simd_from(simd, vals), vals); + assert_eq!(*i64x4::from_fn(simd, |i| vals[i]), vals); + assert_eq!(*i64x4::from_bytes(a.to_bytes()), vals); +} + +#[simd_test] +fn construct_i64x8(simd: S) { + let vals = [-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024]; + let a = i64x8::from_slice(simd, &vals); + let mut stored = [0_i64; 8]; + a.store_slice(&mut stored); + assert_eq!(stored, vals); + assert_eq!(*i64x8::splat(simd, -9), [-9, -9, -9, -9, -9, -9, -9, -9]); + assert_eq!(*i64x8::simd_from(simd, vals), vals); + assert_eq!(*i64x8::from_fn(simd, |i| vals[i]), vals); + assert_eq!(*i64x8::from_bytes(a.to_bytes()), vals); +} + +#[simd_test] +fn construct_u64x2(simd: S) { + let vals = [0, 1_u64 << 63]; + let a = u64x2::from_slice(simd, &vals); + let mut stored = [0_u64; 2]; + a.store_slice(&mut stored); + assert_eq!(stored, vals); + assert_eq!(*u64x2::splat(simd, vals[0]), [0, 0]); + assert_eq!(*u64x2::simd_from(simd, vals), vals); + assert_eq!(*u64x2::from_fn(simd, |i| vals[i]), vals); + assert_eq!(*u64x2::from_bytes(a.to_bytes()), vals); +} + +#[simd_test] +fn construct_u64x4(simd: S) { + let vals = [0, 1_u64 << 63, u64::MAX - 3, 42]; + let a = u64x4::from_slice(simd, &vals); + let mut stored = [0_u64; 4]; + a.store_slice(&mut stored); + assert_eq!(stored, vals); + assert_eq!(*u64x4::splat(simd, vals[0]), [0, 0, 0, 0]); + assert_eq!(*u64x4::simd_from(simd, vals), vals); + assert_eq!(*u64x4::from_fn(simd, |i| vals[i]), vals); + assert_eq!(*u64x4::from_bytes(a.to_bytes()), vals); +} + +#[simd_test] +fn construct_u64x8(simd: S) { + let vals = [ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ]; + let a = u64x8::from_slice(simd, &vals); + let mut stored = [0_u64; 8]; + a.store_slice(&mut stored); + assert_eq!(stored, vals); + assert_eq!(*u64x8::splat(simd, vals[0]), [0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(*u64x8::simd_from(simd, vals), vals); + assert_eq!(*u64x8::from_fn(simd, |i| vals[i]), vals); + assert_eq!(*u64x8::from_bytes(a.to_bytes()), vals); +} + +#[simd_test] +fn arithmetic_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let b = i64x2::from_slice(simd, &[3, -6]); + assert_eq!(*(a + b), [-6, 12]); + assert_eq!(*(a - b), [-12, 24]); + assert_eq!(*(a * b), [-27, -108]); +} + +#[simd_test] +fn arithmetic_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let b = i64x4::from_slice(simd, &[3, -6, 5, -7]); + assert_eq!( + *(a + b), + [-6, 12, 9223372036854775805, -9223372036854775806] + ); + assert_eq!( + *(a - b), + [-12, 24, 9223372036854775795, -9223372036854775792] + ); + assert_eq!( + *(a * b), + [-27, -108, 9223372036854775768, 9223372036854775745] + ); +} + +#[simd_test] +fn arithmetic_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let b = i64x8::from_slice(simd, &[3, -6, 5, -7, -11, 13, -17, 19]); + assert_eq!( + *(a + b), + [ + -6, + 12, + 9223372036854775805, + -9223372036854775806, + 112, + -443, + 772, + -1005 + ] + ); + assert_eq!( + *(a - b), + [ + -12, + 24, + 9223372036854775795, + -9223372036854775792, + 134, + -469, + 806, + -1043 + ] + ); + assert_eq!( + *(a * b), + [ + -27, + -108, + 9223372036854775768, + 9223372036854775745, + -1353, + -5928, + -13413, + -19456 + ] + ); +} + +#[simd_test] +fn arithmetic_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let b = u64x2::from_slice(simd, &[u64::MAX, 7]); + assert_eq!(*(a + b), [u64::MAX, 9223372036854775815]); + assert_eq!(*(a - b), [1, 9223372036854775801]); + assert_eq!(*(a * b), [0, 1_u64 << 63]); +} + +#[simd_test] +fn arithmetic_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let b = u64x4::from_slice(simd, &[u64::MAX, 7, 13, 999]); + assert_eq!(*(a + b), [u64::MAX, 9223372036854775815, 9, 1041]); + assert_eq!( + *(a - b), + [ + 1, + 9223372036854775801, + 18446744073709551599, + 18446744073709550659 + ] + ); + assert_eq!(*(a * b), [0, 1_u64 << 63, 18446744073709551564, 41958]); +} + +#[simd_test] +fn arithmetic_u64x8(simd: S) { + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let b = u64x8::from_slice(simd, &[u64::MAX, 7, 13, 999, 29, 11, 987654321, 1]); + assert_eq!( + *(a + b), + [ + u64::MAX, + 9223372036854775815, + 9, + 1041, + 46, + 110, + 1111111110, + 0 + ] + ); + assert_eq!( + *(a - b), + [ + 1, + 9223372036854775801, + 18446744073709551599, + 18446744073709550659, + 18446744073709551604, + 88, + 18446744072845354084, + 18446744073709551614 + ] + ); + assert_eq!( + *(a * b), + [ + 0, + 1_u64 << 63, + 18446744073709551564, + 41958, + 493, + 1089, + 121932631112635269, + u64::MAX + ] + ); +} + +#[simd_test] +fn neg_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[-1, 2, -3, 4, -5, 6, -7, 8]); + assert_eq!(*(-a), [1, -2, 3, -4, 5, -6, 7, -8]); +} + +#[simd_test] +fn bitwise_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let b = i64x2::from_slice(simd, &[3, -6]); + assert_eq!(*(a & b), [3, 18]); + assert_eq!(*(a | b), [-9, -6]); + assert_eq!(*(a ^ b), [-12, -24]); + assert_eq!(*(!a), [8, -19]); +} + +#[simd_test] +fn bitwise_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let b = i64x4::from_slice(simd, &[3, -6, 5, -7]); + assert_eq!(*(a & b), [3, 18, 0, -9223372036854775799]); + assert_eq!(*(a | b), [-9, -6, 9223372036854775805, -7]); + assert_eq!( + *(a ^ b), + [-12, -24, 9223372036854775805, 9223372036854775792] + ); + assert_eq!(*(!a), [8, -19, -9223372036854775801, 9223372036854775798]); +} + +#[simd_test] +fn bitwise_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let b = i64x8::from_slice(simd, &[3, -6, 5, -7, -11, 13, -17, 19]); + assert_eq!(*(a & b), [3, 18, 0, -9223372036854775799, 113, 8, 773, 0]); + assert_eq!( + *(a | b), + [-9, -6, 9223372036854775805, -7, -1, -451, -1, -1005] + ); + assert_eq!( + *(a ^ b), + [ + -12, + -24, + 9223372036854775805, + 9223372036854775792, + -114, + -459, + -774, + -1005 + ] + ); + assert_eq!( + *(!a), + [ + 8, + -19, + -9223372036854775801, + 9223372036854775798, + -124, + 455, + -790, + 1023 + ] + ); +} + +#[simd_test] +fn bitwise_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let b = u64x2::from_slice(simd, &[u64::MAX, 7]); + assert_eq!(*(a & b), [0, 0]); + assert_eq!(*(a | b), [u64::MAX, 9223372036854775815]); + assert_eq!(*(a ^ b), [u64::MAX, 9223372036854775815]); + assert_eq!(*(!a), [u64::MAX, 9223372036854775807]); +} + +#[simd_test] +fn bitwise_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let b = u64x4::from_slice(simd, &[u64::MAX, 7, 13, 999]); + assert_eq!(*(a & b), [0, 0, 12, 34]); + assert_eq!( + *(a | b), + [u64::MAX, 9223372036854775815, 18446744073709551613, 1007] + ); + assert_eq!( + *(a ^ b), + [u64::MAX, 9223372036854775815, 18446744073709551601, 973] + ); + assert_eq!( + *(!a), + [u64::MAX, 9223372036854775807, 3, 18446744073709551573] + ); +} + +#[simd_test] +fn bitwise_u64x8(simd: S) { + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let b = u64x8::from_slice(simd, &[u64::MAX, 7, 13, 999, 29, 11, 987654321, 1]); + assert_eq!(*(a & b), [0, 0, 12, 34, 17, 3, 39471121, 1]); + assert_eq!( + *(a | b), + [ + u64::MAX, + 9223372036854775815, + 18446744073709551613, + 1007, + 29, + 107, + 1071639989, + u64::MAX + ] + ); + assert_eq!( + *(a ^ b), + [ + u64::MAX, + 9223372036854775815, + 18446744073709551601, + 973, + 12, + 104, + 1032168868, + 18446744073709551614 + ] + ); + assert_eq!( + *(!a), + [ + u64::MAX, + 9223372036854775807, + 3, + 18446744073709551573, + 18446744073709551598, + 18446744073709551516, + 18446744073586094826, + 0 + ] + ); +} + +#[simd_test] +fn shifts_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let shifts = i64x2::from_slice(simd, &[0, 1]); + assert_eq!(*(a << 2_u32), [-36, 72]); + assert_eq!(*(a >> 2_u32), [-3, 4]); + assert_eq!(*(a << shifts), [-9, 36]); + assert_eq!(*(a >> shifts), [-9, 9]); +} + +#[simd_test] +fn shifts_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let shifts = i64x4::from_slice(simd, &[0, 1, 2, 3]); + assert_eq!(*(a << 2_u32), [-36, 72, -32, 36]); + assert_eq!( + *(a >> 2_u32), + [-3, 4, 2305843009213693950, -2305843009213693950] + ); + assert_eq!(*(a << shifts), [-9, 36, -32, 72]); + assert_eq!( + *(a >> shifts), + [-9, 9, 2305843009213693950, -1152921504606846975] + ); +} + +#[simd_test] +fn shifts_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let shifts = i64x8::from_slice(simd, &[0, 1, 2, 3, 0, 1, 2, 3]); + assert_eq!(*(a << 2_u32), [-36, 72, -32, 36, 492, -1824, 3156, -4096]); + assert_eq!( + *(a >> 2_u32), + [ + -3, + 4, + 2305843009213693950, + -2305843009213693950, + 30, + -114, + 197, + -256 + ] + ); + assert_eq!(*(a << shifts), [-9, 36, -32, 72, 123, -912, 3156, -8192]); + assert_eq!( + *(a >> shifts), + [ + -9, + 9, + 2305843009213693950, + -1152921504606846975, + 123, + -228, + 197, + -128 + ] + ); +} + +#[simd_test] +fn shifts_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let shifts = u64x2::from_slice(simd, &[0, 1]); + assert_eq!(*(a << 2_u32), [0, 0]); + assert_eq!(*(a >> 2_u32), [0, 2305843009213693952]); + assert_eq!(*(a << shifts), [0, 0]); + assert_eq!(*(a >> shifts), [0, 4611686018427387904]); +} + +#[simd_test] +fn shifts_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let shifts = u64x4::from_slice(simd, &[0, 1, 2, 3]); + assert_eq!(*(a << 2_u32), [0, 0, 18446744073709551600, 168]); + assert_eq!( + *(a >> 2_u32), + [0, 2305843009213693952, 4611686018427387903, 10] + ); + assert_eq!(*(a << shifts), [0, 0, 18446744073709551600, 336]); + assert_eq!( + *(a >> shifts), + [0, 4611686018427387904, 4611686018427387903, 5] + ); +} + +#[simd_test] +fn shifts_u64x8(simd: S) { + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let shifts = u64x8::from_slice(simd, &[0, 1, 2, 3, 0, 1, 2, 3]); + assert_eq!( + *(a << 2_u32), + [ + 0, + 0, + 18446744073709551600, + 168, + 68, + 396, + 493827156, + 18446744073709551612 + ] + ); + assert_eq!( + *(a >> 2_u32), + [ + 0, + 2305843009213693952, + 4611686018427387903, + 10, + 4, + 24, + 30864197, + 4611686018427387903 + ] + ); + assert_eq!( + *(a << shifts), + [ + 0, + 0, + 18446744073709551600, + 336, + 17, + 198, + 493827156, + 18446744073709551608 + ] + ); + assert_eq!( + *(a >> shifts), + [ + 0, + 4611686018427387904, + 4611686018427387903, + 5, + 17, + 49, + 30864197, + 2305843009213693951 + ] + ); +} + +#[simd_test] +fn compare_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let b = i64x2::from_slice(simd, &[3, -6]); + assert_eq!(<[i64; 2]>::from(a.simd_eq(b)), [0, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_lt(b)), [-1, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_le(b)), [-1, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_ge(b)), [0, -1]); + assert_eq!(<[i64; 2]>::from(a.simd_gt(b)), [0, -1]); +} + +#[simd_test] +fn compare_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let b = i64x4::from_slice(simd, &[3, -6, 5, -7]); + assert_eq!(<[i64; 4]>::from(a.simd_eq(b)), [0, 0, 0, 0]); + assert_eq!(<[i64; 4]>::from(a.simd_lt(b)), [-1, 0, 0, -1]); + assert_eq!(<[i64; 4]>::from(a.simd_le(b)), [-1, 0, 0, -1]); + assert_eq!(<[i64; 4]>::from(a.simd_ge(b)), [0, -1, -1, 0]); + assert_eq!(<[i64; 4]>::from(a.simd_gt(b)), [0, -1, -1, 0]); +} + +#[simd_test] +fn compare_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let b = i64x8::from_slice(simd, &[3, -6, 5, -7, -11, 13, -17, 19]); + assert_eq!(<[i64; 8]>::from(a.simd_eq(b)), [0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(<[i64; 8]>::from(a.simd_lt(b)), [-1, 0, 0, -1, 0, -1, 0, -1]); + assert_eq!(<[i64; 8]>::from(a.simd_le(b)), [-1, 0, 0, -1, 0, -1, 0, -1]); + assert_eq!(<[i64; 8]>::from(a.simd_ge(b)), [0, -1, -1, 0, -1, 0, -1, 0]); + assert_eq!(<[i64; 8]>::from(a.simd_gt(b)), [0, -1, -1, 0, -1, 0, -1, 0]); +} + +#[simd_test] +fn compare_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let b = u64x2::from_slice(simd, &[u64::MAX, 7]); + assert_eq!(<[i64; 2]>::from(a.simd_eq(b)), [0, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_lt(b)), [-1, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_le(b)), [-1, 0]); + assert_eq!(<[i64; 2]>::from(a.simd_ge(b)), [0, -1]); + assert_eq!(<[i64; 2]>::from(a.simd_gt(b)), [0, -1]); +} + +#[simd_test] +fn compare_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let b = u64x4::from_slice(simd, &[u64::MAX, 7, 13, 999]); + assert_eq!(<[i64; 4]>::from(a.simd_eq(b)), [0, 0, 0, 0]); + assert_eq!(<[i64; 4]>::from(a.simd_lt(b)), [-1, 0, 0, -1]); + assert_eq!(<[i64; 4]>::from(a.simd_le(b)), [-1, 0, 0, -1]); + assert_eq!(<[i64; 4]>::from(a.simd_ge(b)), [0, -1, -1, 0]); + assert_eq!(<[i64; 4]>::from(a.simd_gt(b)), [0, -1, -1, 0]); +} + +#[simd_test] +fn compare_u64x8(simd: S) { + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let b = u64x8::from_slice(simd, &[u64::MAX, 7, 13, 999, 29, 11, 987654321, 1]); + assert_eq!(<[i64; 8]>::from(a.simd_eq(b)), [0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(<[i64; 8]>::from(a.simd_lt(b)), [-1, 0, 0, -1, -1, 0, -1, 0]); + assert_eq!(<[i64; 8]>::from(a.simd_le(b)), [-1, 0, 0, -1, -1, 0, -1, 0]); + assert_eq!(<[i64; 8]>::from(a.simd_ge(b)), [0, -1, -1, 0, 0, -1, 0, -1]); + assert_eq!(<[i64; 8]>::from(a.simd_gt(b)), [0, -1, -1, 0, 0, -1, 0, -1]); +} + +#[simd_test] +fn min_max_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-9, 18]); + let b = i64x2::from_slice(simd, &[3, -6]); + assert_eq!(*a.min(b), [-9, -6]); + assert_eq!(*a.max(b), [3, 18]); +} + +#[simd_test] +fn min_max_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let b = i64x4::from_slice(simd, &[3, -6, 5, -7]); + assert_eq!(*a.min(b), [-9, -6, 5, -9223372036854775799]); + assert_eq!(*a.max(b), [3, 18, 9223372036854775800, -7]); +} + +#[simd_test] +fn min_max_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let b = i64x8::from_slice(simd, &[3, -6, 5, -7, -11, 13, -17, 19]); + assert_eq!( + *a.min(b), + [-9, -6, 5, -9223372036854775799, -11, -456, -17, -1024] + ); + assert_eq!( + *a.max(b), + [3, 18, 9223372036854775800, -7, 123, 13, 789, 19] + ); +} + +#[simd_test] +fn min_max_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let b = u64x2::from_slice(simd, &[u64::MAX, 7]); + assert_eq!(*a.min(b), [0, 7]); + assert_eq!(*a.max(b), [u64::MAX, 1_u64 << 63]); +} + +#[simd_test] +fn min_max_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let b = u64x4::from_slice(simd, &[u64::MAX, 7, 13, 999]); + assert_eq!(*a.min(b), [0, 7, 13, 42]); + assert_eq!(*a.max(b), [u64::MAX, 1_u64 << 63, u64::MAX - 3, 999]); +} + +#[simd_test] +fn min_max_u64x8(simd: S) { + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let b = u64x8::from_slice(simd, &[u64::MAX, 7, 13, 999, 29, 11, 987654321, 1]); + assert_eq!(*a.min(b), [0, 7, 13, 42, 17, 11, 123456789, 1]); + assert_eq!( + *a.max(b), + [ + u64::MAX, + 1_u64 << 63, + u64::MAX - 3, + 999, + 29, + 99, + 987654321, + u64::MAX + ] + ); +} + +#[simd_test] +fn select_i64x2(simd: S) { + let mask = mask64x2::from_slice(simd, &[-1, 0]); + let a = i64x2::from_slice(simd, &[-9, 18]); + let b = i64x2::from_slice(simd, &[3, -6]); + assert_eq!(*mask.select(a, b), [-9, -6]); +} + +#[simd_test] +fn select_i64x4(simd: S) { + let mask = mask64x4::from_slice(simd, &[-1, 0, -1, 0]); + let a = i64x4::from_slice(simd, &[-9, 18, i64::MAX - 7, i64::MIN + 9]); + let b = i64x4::from_slice(simd, &[3, -6, 5, -7]); + assert_eq!(*mask.select(a, b), [-9, -6, 9223372036854775800, -7]); +} + +#[simd_test] +fn select_i64x8(simd: S) { + let mask = mask64x8::from_slice(simd, &[-1, 0, -1, 0, -1, 0, -1, 0]); + let a = i64x8::from_slice( + simd, + &[-9, 18, i64::MAX - 7, i64::MIN + 9, 123, -456, 789, -1024], + ); + let b = i64x8::from_slice(simd, &[3, -6, 5, -7, -11, 13, -17, 19]); + assert_eq!( + *mask.select(a, b), + [-9, -6, 9223372036854775800, -7, 123, 13, 789, 19] + ); +} + +#[simd_test] +fn select_u64x2(simd: S) { + let mask = mask64x2::from_slice(simd, &[-1, 0]); + let a = u64x2::from_slice(simd, &[0, 1_u64 << 63]); + let b = u64x2::from_slice(simd, &[u64::MAX, 7]); + assert_eq!(*mask.select(a, b), [0, 7]); +} + +#[simd_test] +fn select_u64x4(simd: S) { + let mask = mask64x4::from_slice(simd, &[-1, 0, -1, 0]); + let a = u64x4::from_slice(simd, &[0, 1_u64 << 63, u64::MAX - 3, 42]); + let b = u64x4::from_slice(simd, &[u64::MAX, 7, 13, 999]); + assert_eq!(*mask.select(a, b), [0, 7, u64::MAX - 3, 999]); +} + +#[simd_test] +fn select_u64x8(simd: S) { + let mask = mask64x8::from_slice(simd, &[-1, 0, -1, 0, -1, 0, -1, 0]); + let a = u64x8::from_slice( + simd, + &[ + 0, + 1_u64 << 63, + u64::MAX - 3, + 42, + 17, + 99, + 123456789, + u64::MAX, + ], + ); + let b = u64x8::from_slice(simd, &[u64::MAX, 7, 13, 999, 29, 11, 987654321, 1]); + assert_eq!( + *mask.select(a, b), + [0, 7, u64::MAX - 3, 999, 17, 11, 123456789, 1] + ); +} + +#[simd_test] +fn slide_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[1, 2]); + let b = i64x2::from_slice(simd, &[3, 4]); + assert_eq!(*a.slide::<1>(b), [2, 3]); +} + +#[simd_test] +fn slide_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = i64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*a.slide::<1>(b), [2, 3, 4, 5]); +} + +#[simd_test] +fn slide_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = i64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*a.slide::<1>(b), [2, 3, 4, 5, 6, 7, 8, 9]); +} + +#[simd_test] +fn slide_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[1, 2]); + let b = u64x2::from_slice(simd, &[3, 4]); + assert_eq!(*a.slide::<1>(b), [2, 3]); +} + +#[simd_test] +fn slide_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = u64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*a.slide::<1>(b), [2, 3, 4, 5]); +} + +#[simd_test] +fn slide_u64x8(simd: S) { + let a = u64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = u64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*a.slide::<1>(b), [2, 3, 4, 5, 6, 7, 8, 9]); +} + +#[simd_test] +fn i64_split_combine(simd: S) { + let lo = i64x2::from_slice(simd, &[1, 2]); + let hi = i64x2::from_slice(simd, &[3, 4]); + let combined = lo.combine(hi); + assert_eq!(*combined, [1, 2, 3, 4]); + + let (lo, hi) = combined.split(); + assert_eq!(*lo, [1, 2]); + assert_eq!(*hi, [3, 4]); + + let tail = i64x4::from_slice(simd, &[5, 6, 7, 8]); + let wide = combined.combine(tail); + assert_eq!(*wide, [1, 2, 3, 4, 5, 6, 7, 8]); + + let (lo, hi) = wide.split(); + assert_eq!(*lo, [1, 2, 3, 4]); + assert_eq!(*hi, [5, 6, 7, 8]); +} + +#[simd_test] +fn u64_split_combine(simd: S) { + let lo = u64x2::from_slice(simd, &[1, 2]); + let hi = u64x2::from_slice(simd, &[3, 4]); + let combined = lo.combine(hi); + assert_eq!(*combined, [1, 2, 3, 4]); + + let (lo, hi) = combined.split(); + assert_eq!(*lo, [1, 2]); + assert_eq!(*hi, [3, 4]); + + let tail = u64x4::from_slice(simd, &[5, 6, 7, 8]); + let wide = combined.combine(tail); + assert_eq!(*wide, [1, 2, 3, 4, 5, 6, 7, 8]); + + let (lo, hi) = wide.split(); + assert_eq!(*lo, [1, 2, 3, 4]); + assert_eq!(*hi, [5, 6, 7, 8]); +} + +#[simd_test] +fn native_width_i64_u64(simd: S) { + let mask_vals: Vec = (0..S::mask64s::N).map(|i| mask_lane(i % 2 == 0)).collect(); + let mask = S::mask64s::from_slice(simd, &mask_vals); + + let u_true: Vec = (0..S::u64s::N).map(|i| (1_u64 << 63) + i as u64).collect(); + let u_false: Vec = (0..S::u64s::N).map(|i| i as u64).collect(); + let u_selected = mask.select( + S::u64s::from_slice(simd, &u_true), + S::u64s::from_slice(simd, &u_false), + ); + let u_expected: Vec = (0..S::u64s::N) + .map(|i| if i % 2 == 0 { u_true[i] } else { u_false[i] }) + .collect(); + assert_eq!(u_selected.as_slice(), u_expected); + assert_eq!( + (S::u64s::splat(simd, 3) * 7).as_slice(), + vec![21; S::u64s::N] + ); + + let i_true: Vec = (0..S::i64s::N) + .map(|i| -i64::try_from(i).expect("native vector length fits in i64") - 1) + .collect(); + let i_false: Vec = (0..S::i64s::N) + .map(|i| i64::try_from(i).expect("native vector length fits in i64") + 1) + .collect(); + let i_selected = mask.select( + S::i64s::from_slice(simd, &i_true), + S::i64s::from_slice(simd, &i_false), + ); + let i_expected: Vec = (0..S::i64s::N) + .map(|i| if i % 2 == 0 { i_true[i] } else { i_false[i] }) + .collect(); + assert_eq!(i_selected.as_slice(), i_expected); + assert_eq!( + (S::i64s::block_splat(i64x2::from_slice(simd, &[11, -12]))).as_slice(), + [11, -12].repeat(S::i64s::N / 2) + ); + assert_eq!( + (S::u64s::block_splat(u64x2::from_slice(simd, &[13, 14]))).as_slice(), + [13, 14].repeat(S::u64s::N / 2) + ); +} + +#[simd_test] +fn array_methods_i64x2(simd: S) { + let a = simd.load_array_i64x2([1, 2]); + assert_eq!(simd.as_array_i64x2(a), [1, 2]); + + let b_vals = [3, 4]; + let mut b = simd.load_array_ref_i64x2(&b_vals); + assert_eq!(simd.as_array_ref_i64x2(&b), &[3, 4]); + + simd.as_array_mut_i64x2(&mut b)[1] = 9; + assert_eq!(*b, [3, 9]); + + let mut dest = [0_i64; 2]; + simd.store_array_i64x2(b, &mut dest); + assert_eq!(dest, [3, 9]); +} + +#[simd_test] +fn array_methods_i64x4(simd: S) { + let a = simd.load_array_i64x4([1, 2, 3, 4]); + assert_eq!(simd.as_array_i64x4(a), [1, 2, 3, 4]); + + let b_vals = [5, 6, 7, 8]; + let mut b = simd.load_array_ref_i64x4(&b_vals); + assert_eq!(simd.as_array_ref_i64x4(&b), &[5, 6, 7, 8]); + + simd.as_array_mut_i64x4(&mut b)[2] = 99; + assert_eq!(*b, [5, 6, 99, 8]); + + let mut dest = [0_i64; 4]; + simd.store_array_i64x4(b, &mut dest); + assert_eq!(dest, [5, 6, 99, 8]); +} + +#[simd_test] +fn array_methods_i64x8(simd: S) { + let a = simd.load_array_i64x8([1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(simd.as_array_i64x8(a), [1, 2, 3, 4, 5, 6, 7, 8]); + + let b_vals = [9, 10, 11, 12, 13, 14, 15, 16]; + let mut b = simd.load_array_ref_i64x8(&b_vals); + assert_eq!( + simd.as_array_ref_i64x8(&b), + &[9, 10, 11, 12, 13, 14, 15, 16] + ); + + simd.as_array_mut_i64x8(&mut b)[4] = 99; + assert_eq!(*b, [9, 10, 11, 12, 99, 14, 15, 16]); + + let mut dest = [0_i64; 8]; + simd.store_array_i64x8(b, &mut dest); + assert_eq!(dest, [9, 10, 11, 12, 99, 14, 15, 16]); +} + +#[simd_test] +fn array_methods_u64x2(simd: S) { + let a = simd.load_array_u64x2([1, 2]); + assert_eq!(simd.as_array_u64x2(a), [1, 2]); + + let b_vals = [3, 4]; + let mut b = simd.load_array_ref_u64x2(&b_vals); + assert_eq!(simd.as_array_ref_u64x2(&b), &[3, 4]); + + simd.as_array_mut_u64x2(&mut b)[1] = 9; + assert_eq!(*b, [3, 9]); + + let mut dest = [0_u64; 2]; + simd.store_array_u64x2(b, &mut dest); + assert_eq!(dest, [3, 9]); +} + +#[simd_test] +fn array_methods_u64x4(simd: S) { + let a = simd.load_array_u64x4([1, 2, 3, 4]); + assert_eq!(simd.as_array_u64x4(a), [1, 2, 3, 4]); + + let b_vals = [5, 6, 7, 8]; + let mut b = simd.load_array_ref_u64x4(&b_vals); + assert_eq!(simd.as_array_ref_u64x4(&b), &[5, 6, 7, 8]); + + simd.as_array_mut_u64x4(&mut b)[2] = 99; + assert_eq!(*b, [5, 6, 99, 8]); + + let mut dest = [0_u64; 4]; + simd.store_array_u64x4(b, &mut dest); + assert_eq!(dest, [5, 6, 99, 8]); +} + +#[simd_test] +fn array_methods_u64x8(simd: S) { + let a = simd.load_array_u64x8([1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(simd.as_array_u64x8(a), [1, 2, 3, 4, 5, 6, 7, 8]); + + let b_vals = [9, 10, 11, 12, 13, 14, 15, 16]; + let mut b = simd.load_array_ref_u64x8(&b_vals); + assert_eq!( + simd.as_array_ref_u64x8(&b), + &[9, 10, 11, 12, 13, 14, 15, 16] + ); + + simd.as_array_mut_u64x8(&mut b)[4] = 99; + assert_eq!(*b, [9, 10, 11, 12, 99, 14, 15, 16]); + + let mut dest = [0_u64; 8]; + simd.store_array_u64x8(b, &mut dest); + assert_eq!(dest, [9, 10, 11, 12, 99, 14, 15, 16]); +} + +#[simd_test] +fn neg_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[-1, 2]); + assert_eq!(*(-a), [1, -2]); +} + +#[simd_test] +fn neg_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[-1, 2, -3, 4]); + assert_eq!(*(-a), [1, -2, 3, -4]); +} + +#[simd_test] +fn slide_within_blocks_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[1, 2]); + let b = i64x2::from_slice(simd, &[3, 4]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 3]); + assert_eq!(*a.slide_within_blocks::<2>(b), [3, 4]); +} + +#[simd_test] +fn slide_within_blocks_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = i64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2, 3, 4]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 5, 4, 7]); + assert_eq!(*a.slide_within_blocks::<2>(b), [5, 6, 7, 8]); +} + +#[simd_test] +fn slide_within_blocks_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = i64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 9, 4, 11, 6, 13, 8, 15]); + assert_eq!( + *a.slide_within_blocks::<2>(b), + [9, 10, 11, 12, 13, 14, 15, 16] + ); +} + +#[simd_test] +fn slide_within_blocks_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[1, 2]); + let b = u64x2::from_slice(simd, &[3, 4]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 3]); + assert_eq!(*a.slide_within_blocks::<2>(b), [3, 4]); +} + +#[simd_test] +fn slide_within_blocks_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = u64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2, 3, 4]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 5, 4, 7]); + assert_eq!(*a.slide_within_blocks::<2>(b), [5, 6, 7, 8]); +} + +#[simd_test] +fn slide_within_blocks_u64x8(simd: S) { + let a = u64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = u64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*a.slide_within_blocks::<0>(b), [1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(*a.slide_within_blocks::<1>(b), [2, 9, 4, 11, 6, 13, 8, 15]); + assert_eq!( + *a.slide_within_blocks::<2>(b), + [9, 10, 11, 12, 13, 14, 15, 16] + ); +} + +#[simd_test] +fn zip_unzip_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[1, 2]); + let b = i64x2::from_slice(simd, &[3, 4]); + assert_eq!(*simd.zip_low_i64x2(a, b), [1, 3]); + assert_eq!(*simd.zip_high_i64x2(a, b), [2, 4]); + assert_eq!(*simd.unzip_low_i64x2(a, b), [1, 3]); + assert_eq!(*simd.unzip_high_i64x2(a, b), [2, 4]); +} + +#[simd_test] +fn zip_unzip_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = i64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*simd.zip_low_i64x4(a, b), [1, 5, 2, 6]); + assert_eq!(*simd.zip_high_i64x4(a, b), [3, 7, 4, 8]); + assert_eq!(*simd.unzip_low_i64x4(a, b), [1, 3, 5, 7]); + assert_eq!(*simd.unzip_high_i64x4(a, b), [2, 4, 6, 8]); +} + +#[simd_test] +fn zip_unzip_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = i64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*simd.zip_low_i64x8(a, b), [1, 9, 2, 10, 3, 11, 4, 12]); + assert_eq!(*simd.zip_high_i64x8(a, b), [5, 13, 6, 14, 7, 15, 8, 16]); + assert_eq!(*simd.unzip_low_i64x8(a, b), [1, 3, 5, 7, 9, 11, 13, 15]); + assert_eq!(*simd.unzip_high_i64x8(a, b), [2, 4, 6, 8, 10, 12, 14, 16]); +} + +#[simd_test] +fn zip_unzip_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[1, 2]); + let b = u64x2::from_slice(simd, &[3, 4]); + assert_eq!(*simd.zip_low_u64x2(a, b), [1, 3]); + assert_eq!(*simd.zip_high_u64x2(a, b), [2, 4]); + assert_eq!(*simd.unzip_low_u64x2(a, b), [1, 3]); + assert_eq!(*simd.unzip_high_u64x2(a, b), [2, 4]); +} + +#[simd_test] +fn zip_unzip_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = u64x4::from_slice(simd, &[5, 6, 7, 8]); + assert_eq!(*simd.zip_low_u64x4(a, b), [1, 5, 2, 6]); + assert_eq!(*simd.zip_high_u64x4(a, b), [3, 7, 4, 8]); + assert_eq!(*simd.unzip_low_u64x4(a, b), [1, 3, 5, 7]); + assert_eq!(*simd.unzip_high_u64x4(a, b), [2, 4, 6, 8]); +} + +#[simd_test] +fn zip_unzip_u64x8(simd: S) { + let a = u64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = u64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + assert_eq!(*simd.zip_low_u64x8(a, b), [1, 9, 2, 10, 3, 11, 4, 12]); + assert_eq!(*simd.zip_high_u64x8(a, b), [5, 13, 6, 14, 7, 15, 8, 16]); + assert_eq!(*simd.unzip_low_u64x8(a, b), [1, 3, 5, 7, 9, 11, 13, 15]); + assert_eq!(*simd.unzip_high_u64x8(a, b), [2, 4, 6, 8, 10, 12, 14, 16]); +} + +#[simd_test] +fn interleave_deinterleave_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[1, 2]); + let b = i64x2::from_slice(simd, &[3, 4]); + let (lo, hi) = simd.interleave_i64x2(a, b); + assert_eq!(*lo, [1, 3]); + assert_eq!(*hi, [2, 4]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_i64x2(lo, hi); + assert_eq!(*a_roundtrip, [1, 2]); + assert_eq!(*b_roundtrip, [3, 4]); +} + +#[simd_test] +fn interleave_deinterleave_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = i64x4::from_slice(simd, &[5, 6, 7, 8]); + let (lo, hi) = simd.interleave_i64x4(a, b); + assert_eq!(*lo, [1, 5, 2, 6]); + assert_eq!(*hi, [3, 7, 4, 8]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_i64x4(lo, hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4]); + assert_eq!(*b_roundtrip, [5, 6, 7, 8]); + + let (lo, hi) = a.interleave(b); + let (a_roundtrip, b_roundtrip) = lo.deinterleave(hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4]); + assert_eq!(*b_roundtrip, [5, 6, 7, 8]); +} + +#[simd_test] +fn interleave_deinterleave_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = i64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + let (lo, hi) = simd.interleave_i64x8(a, b); + assert_eq!(*lo, [1, 9, 2, 10, 3, 11, 4, 12]); + assert_eq!(*hi, [5, 13, 6, 14, 7, 15, 8, 16]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_i64x8(lo, hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(*b_roundtrip, [9, 10, 11, 12, 13, 14, 15, 16]); +} + +#[simd_test] +fn interleave_deinterleave_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[1, 2]); + let b = u64x2::from_slice(simd, &[3, 4]); + let (lo, hi) = simd.interleave_u64x2(a, b); + assert_eq!(*lo, [1, 3]); + assert_eq!(*hi, [2, 4]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_u64x2(lo, hi); + assert_eq!(*a_roundtrip, [1, 2]); + assert_eq!(*b_roundtrip, [3, 4]); +} + +#[simd_test] +fn interleave_deinterleave_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[1, 2, 3, 4]); + let b = u64x4::from_slice(simd, &[5, 6, 7, 8]); + let (lo, hi) = simd.interleave_u64x4(a, b); + assert_eq!(*lo, [1, 5, 2, 6]); + assert_eq!(*hi, [3, 7, 4, 8]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_u64x4(lo, hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4]); + assert_eq!(*b_roundtrip, [5, 6, 7, 8]); + + let (lo, hi) = a.interleave(b); + let (a_roundtrip, b_roundtrip) = lo.deinterleave(hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4]); + assert_eq!(*b_roundtrip, [5, 6, 7, 8]); +} + +#[simd_test] +fn interleave_deinterleave_u64x8(simd: S) { + let a = u64x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); + let b = u64x8::from_slice(simd, &[9, 10, 11, 12, 13, 14, 15, 16]); + let (lo, hi) = simd.interleave_u64x8(a, b); + assert_eq!(*lo, [1, 9, 2, 10, 3, 11, 4, 12]); + assert_eq!(*hi, [5, 13, 6, 14, 7, 15, 8, 16]); + let (a_roundtrip, b_roundtrip) = simd.deinterleave_u64x8(lo, hi); + assert_eq!(*a_roundtrip, [1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(*b_roundtrip, [9, 10, 11, 12, 13, 14, 15, 16]); +} + +#[simd_test] +fn load_store_interleaved_128_u64x8(simd: S) { + let data = [1, 2, 101, 102, 201, 202, 301, 302]; + let loaded = simd.load_interleaved_128_u64x8(&data); + assert_eq!(*loaded, [1, 201, 2, 202, 101, 301, 102, 302]); + + let a = u64x8::from_slice(simd, &[1, 201, 2, 202, 101, 301, 102, 302]); + let mut dest = [0_u64; 8]; + simd.store_interleaved_128_u64x8(a, &mut dest); + assert_eq!(dest, data); +} + +#[simd_test] +fn reinterpret_i64x2(simd: S) { + let a = i64x2::from_slice(simd, &[1, -2]); + let bytes: u8x16 = a.bitcast(); + let words: u32x4 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_i64x2(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_i64x2(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn reinterpret_i64x4(simd: S) { + let a = i64x4::from_slice(simd, &[1, -2, 3, -4]); + let bytes: u8x32 = a.bitcast(); + let words: u32x8 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_i64x4(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_i64x4(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn reinterpret_i64x8(simd: S) { + let a = i64x8::from_slice(simd, &[1, -2, 3, -4, 5, -6, 7, -8]); + let bytes: u8x64 = a.bitcast(); + let words: u32x16 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_i64x8(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_i64x8(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn reinterpret_u64x2(simd: S) { + let a = u64x2::from_slice(simd, &[1, u64::MAX - 1]); + let bytes: u8x16 = a.bitcast(); + let words: u32x4 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_u64x2(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_u64x2(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn reinterpret_u64x4(simd: S) { + let a = u64x4::from_slice(simd, &[1, u64::MAX - 1, 3, u64::MAX - 3]); + let bytes: u8x32 = a.bitcast(); + let words: u32x8 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_u64x4(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_u64x4(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn reinterpret_u64x8(simd: S) { + let a = u64x8::from_slice(simd, &[1, u64::MAX - 1, 3, u64::MAX - 3, 5, 6, 7, 8]); + let bytes: u8x64 = a.bitcast(); + let words: u32x16 = a.bitcast(); + assert_eq!(simd.reinterpret_u8_u64x8(a).as_slice(), bytes.as_slice()); + assert_eq!(simd.reinterpret_u32_u64x8(a).as_slice(), words.as_slice()); +} + +#[simd_test] +fn mask64x2_ops(simd: S) { + let t = simd.splat_mask64x2(true); + let f = simd.splat_mask64x2(false); + assert_eq!(simd.as_array_mask64x2(t), [-1, -1]); + assert_eq!(simd.as_array_mask64x2(f), [0, 0]); + + let a = simd.load_array_mask64x2([-1, 0]); + let b = simd.load_array_mask64x2([0, -1]); + assert_eq!(simd.as_array_mask64x2(a), [-1, 0]); + assert_eq!(simd.as_array_mask64x2(simd.and_mask64x2(a, b)), [0, 0]); + assert_eq!(simd.as_array_mask64x2(simd.or_mask64x2(a, b)), [-1, -1]); + assert_eq!(simd.as_array_mask64x2(simd.xor_mask64x2(a, b)), [-1, -1]); + assert_eq!(simd.as_array_mask64x2(simd.not_mask64x2(a)), [0, -1]); + assert_eq!( + simd.as_array_mask64x2(simd.select_mask64x2(a, t, f)), + [-1, 0] + ); + assert_eq!( + simd.as_array_mask64x2(simd.simd_eq_mask64x2(a, a)), + [-1, -1] + ); + assert_eq!(simd.as_array_mask64x2(simd.simd_eq_mask64x2(a, b)), [0, 0]); + + let mut bitmask = simd.from_bitmask_mask64x2(0b01); + assert_eq!(simd.as_array_mask64x2(bitmask), [-1, 0]); + assert_eq!(simd.to_bitmask_mask64x2(bitmask), 0b01); + simd.set_mask64x2(&mut bitmask, 1, true); + assert_eq!(simd.to_bitmask_mask64x2(bitmask), 0b11); + + assert!(simd.any_true_mask64x2(a)); + assert!(!simd.all_true_mask64x2(a)); + assert!(simd.any_false_mask64x2(a)); + assert!(!simd.all_false_mask64x2(a)); + assert!(simd.all_true_mask64x2(t)); + assert!(simd.all_false_mask64x2(f)); +} + +#[simd_test] +fn mask64x4_ops(simd: S) { + let t = simd.splat_mask64x4(true); + let f = simd.splat_mask64x4(false); + assert_eq!(simd.as_array_mask64x4(t), [-1, -1, -1, -1]); + assert_eq!(simd.as_array_mask64x4(f), [0, 0, 0, 0]); + + let a = simd.load_array_mask64x4([-1, 0, -1, 0]); + let b = simd.load_array_mask64x4([0, -1, -1, 0]); + assert_eq!( + simd.as_array_mask64x4(simd.and_mask64x4(a, b)), + [0, 0, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x4(simd.or_mask64x4(a, b)), + [-1, -1, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x4(simd.xor_mask64x4(a, b)), + [-1, -1, 0, 0] + ); + assert_eq!(simd.as_array_mask64x4(simd.not_mask64x4(a)), [0, -1, 0, -1]); + assert_eq!( + simd.as_array_mask64x4(simd.select_mask64x4(a, t, f)), + [-1, 0, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x4(simd.simd_eq_mask64x4(a, b)), + [0, 0, -1, -1] + ); + + let mut bitmask = simd.from_bitmask_mask64x4(0b1010); + assert_eq!(simd.as_array_mask64x4(bitmask), [0, -1, 0, -1]); + assert_eq!(simd.to_bitmask_mask64x4(bitmask), 0b1010); + simd.set_mask64x4(&mut bitmask, 0, true); + assert_eq!(simd.to_bitmask_mask64x4(bitmask), 0b1011); + + let combined = simd.combine_mask64x2( + simd.load_array_mask64x2([-1, 0]), + simd.load_array_mask64x2([0, -1]), + ); + assert_eq!(simd.as_array_mask64x4(combined), [-1, 0, 0, -1]); + let (lo, hi) = simd.split_mask64x4(combined); + assert_eq!(simd.as_array_mask64x2(lo), [-1, 0]); + assert_eq!(simd.as_array_mask64x2(hi), [0, -1]); + + assert!(simd.any_true_mask64x4(a)); + assert!(!simd.all_true_mask64x4(a)); + assert!(simd.any_false_mask64x4(a)); + assert!(!simd.all_false_mask64x4(a)); + assert!(simd.all_true_mask64x4(t)); + assert!(simd.all_false_mask64x4(f)); +} + +#[simd_test] +fn mask64x8_ops(simd: S) { + let t = simd.splat_mask64x8(true); + let f = simd.splat_mask64x8(false); + assert_eq!(simd.as_array_mask64x8(t), [-1, -1, -1, -1, -1, -1, -1, -1]); + assert_eq!(simd.as_array_mask64x8(f), [0, 0, 0, 0, 0, 0, 0, 0]); + + let a = simd.load_array_mask64x8([-1, 0, -1, 0, -1, 0, -1, 0]); + let b = simd.load_array_mask64x8([0, -1, -1, 0, 0, -1, -1, 0]); + assert_eq!( + simd.as_array_mask64x8(simd.and_mask64x8(a, b)), + [0, 0, -1, 0, 0, 0, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x8(simd.or_mask64x8(a, b)), + [-1, -1, -1, 0, -1, -1, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x8(simd.xor_mask64x8(a, b)), + [-1, -1, 0, 0, -1, -1, 0, 0] + ); + assert_eq!( + simd.as_array_mask64x8(simd.not_mask64x8(a)), + [0, -1, 0, -1, 0, -1, 0, -1] + ); + assert_eq!( + simd.as_array_mask64x8(simd.select_mask64x8(a, t, f)), + [-1, 0, -1, 0, -1, 0, -1, 0] + ); + assert_eq!( + simd.as_array_mask64x8(simd.simd_eq_mask64x8(a, b)), + [0, 0, -1, -1, 0, 0, -1, -1] + ); + + let mut bitmask = simd.from_bitmask_mask64x8(0b1010_0101); + assert_eq!( + simd.as_array_mask64x8(bitmask), + [-1, 0, -1, 0, 0, -1, 0, -1] + ); + assert_eq!(simd.to_bitmask_mask64x8(bitmask), 0b1010_0101); + simd.set_mask64x8(&mut bitmask, 1, true); + assert_eq!(simd.to_bitmask_mask64x8(bitmask), 0b1010_0111); + + let combined = simd.combine_mask64x4( + simd.load_array_mask64x4([-1, 0, -1, 0]), + simd.load_array_mask64x4([0, -1, 0, -1]), + ); + assert_eq!( + simd.as_array_mask64x8(combined), + [-1, 0, -1, 0, 0, -1, 0, -1] + ); + let (lo, hi) = simd.split_mask64x8(combined); + assert_eq!(simd.as_array_mask64x4(lo), [-1, 0, -1, 0]); + assert_eq!(simd.as_array_mask64x4(hi), [0, -1, 0, -1]); + + assert!(simd.any_true_mask64x8(a)); + assert!(!simd.all_true_mask64x8(a)); + assert!(simd.any_false_mask64x8(a)); + assert!(!simd.all_false_mask64x8(a)); + assert!(simd.all_true_mask64x8(t)); + assert!(simd.all_false_mask64x8(f)); +} diff --git a/fearless_simd_tests/tests/harness/lm_generated.rs b/fearless_simd_tests/tests/harness/lm_generated.rs index 789a8eb99..34de5b16e 100644 --- a/fearless_simd_tests/tests/harness/lm_generated.rs +++ b/fearless_simd_tests/tests/harness/lm_generated.rs @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT mod extended_512; -mod mask_methods; +#[cfg(not(miri))] // too slow +mod mask_roundtrip; +#[cfg(not(miri))] // too slow +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod mask_roundtrip_x86; mod mod_256; mod mod_512; diff --git a/fearless_simd_tests/tests/harness/lm_generated/extended_512.rs b/fearless_simd_tests/tests/harness/lm_generated/extended_512.rs index 2de317d3e..f1e03a25b 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/extended_512.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/extended_512.rs @@ -512,6 +512,61 @@ fn fract_f32x16(simd: S) { ); } +#[simd_test] +fn fract_f64x8(simd: S) { + let a = f64x8::from_slice(simd, &[1.7, -2.3, 3.9, -4.1, 5.5, -6.6, 7.2, -8.8]); + let result = simd.fract_f64x8(a); + assert_eq!( + *result, + [ + 0.7, + -0.2999999999999998, + 0.8999999999999999, + -0.09999999999999964, + 0.5, + -0.5999999999999996, + 0.20000000000000018, + -0.8000000000000007 + ] + ); +} + +#[simd_test] +fn approximate_recip_f32x16(simd: S) { + let a = f32x16::from_slice( + simd, + &[ + 1.0, -2.0, 23.0, 9.0, 0.5, -0.25, 128.0, -1024.0, 3.0, -7.0, 11.0, -13.0, 19.0, -29.0, + 37.0, -41.0, + ], + ); + let result = a.approximate_recip(); + for i in 0..16 { + let expected = 1.0 / a[i]; + let rel_error = ((result[i] - expected) / expected).abs(); + assert!( + rel_error < 0.005, + "approximate_recip({}) rel_error = {rel_error}", + a[i] + ); + } +} + +#[simd_test] +fn approximate_recip_f64x8(simd: S) { + let a = f64x8::from_slice(simd, &[1.0, -2.0, 23.0, 9.0, 0.5, -0.25, 128.0, -1024.0]); + let result = a.approximate_recip(); + for i in 0..8 { + let expected = 1.0 / a[i]; + let rel_error = ((result[i] - expected) / expected).abs(); + assert!( + rel_error < 0.005, + "approximate_recip({}) rel_error = {rel_error}", + a[i] + ); + } +} + // ============================================================================= // max_precise and min_precise tests (512-bit floats) // ============================================================================= @@ -688,6 +743,82 @@ fn min_precise_f32x16_with_nan(simd: S) { assert_eq!(result[15], 5.0); } +#[simd_test] +fn max_precise_f64x8(simd: S) { + let a = f64x8::from_slice(simd, &[2.0, -3.0, 0.0, 0.5, 1.0, 5.0, 3.0, 7.0]); + let b = f64x8::from_slice(simd, &[1.0, -2.0, 7.0, 3.0, 2.0, 4.0, 6.0, 5.0]); + assert_eq!(*a.max_precise(b), [2.0, -2.0, 7.0, 3.0, 2.0, 5.0, 6.0, 7.0]); +} + +#[simd_test] +fn min_precise_f64x8(simd: S) { + let a = f64x8::from_slice(simd, &[2.0, -3.0, 0.0, 0.5, 1.0, 5.0, 3.0, 7.0]); + let b = f64x8::from_slice(simd, &[1.0, -2.0, 7.0, 3.0, 2.0, 4.0, 6.0, 5.0]); + assert_eq!(*a.min_precise(b), [1.0, -3.0, 0.0, 0.5, 1.0, 4.0, 3.0, 5.0]); +} + +#[simd_test] +fn max_precise_f64x8_with_nan(simd: S) { + let a = f64x8::from_slice( + simd, + &[f64::NAN, -3.0, f64::INFINITY, 0.5, 1.0, f64::NAN, 3.0, 7.0], + ); + let b = f64x8::from_slice( + simd, + &[ + 1.0, + f64::NAN, + 7.0, + f64::NEG_INFINITY, + f64::NAN, + 4.0, + 6.0, + 5.0, + ], + ); + let result = a.max_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); + assert_eq!(result[2], f64::INFINITY); + assert_eq!(result[3], 0.5); + assert_eq!(result[4], 1.0); + assert_eq!(result[5], 4.0); + assert_eq!(result[6], 6.0); + assert_eq!(result[7], 7.0); +} + +#[simd_test] +fn min_precise_f64x8_with_nan(simd: S) { + let a = f64x8::from_slice( + simd, + &[f64::NAN, -3.0, f64::INFINITY, 0.5, 1.0, f64::NAN, 3.0, 7.0], + ); + let b = f64x8::from_slice( + simd, + &[ + 1.0, + f64::NAN, + 7.0, + f64::NEG_INFINITY, + f64::NAN, + 4.0, + 6.0, + 5.0, + ], + ); + let result = a.min_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); + assert_eq!(result[2], 7.0); + assert_eq!(result[3], f64::NEG_INFINITY); + assert_eq!(result[4], 1.0); + assert_eq!(result[5], 4.0); + assert_eq!(result[6], 3.0); + assert_eq!(result[7], 5.0); +} + // ============================================================================= // Shift operations tests (512-bit) // ============================================================================= @@ -941,6 +1072,114 @@ fn shl_u32x16(simd: S) { } // Vector shift tests (shlv/shrv) +#[simd_test] +fn shlv_i8x64(simd: S) { + const A: [i8; 16] = [64, 65, -64, -65, 1, 2, 3, 4, -1, -2, -3, -4, 15, 16, 31, 32]; + const SHIFTS: [i8; 16] = [1, 2, 1, 2, 0, 1, 2, 3, 1, 2, 3, 4, 3, 2, 1, 0]; + const EXPECTED: [i8; 16] = [ + -128, 4, -128, -4, 1, 4, 12, 32, -2, -8, -24, -64, 120, 64, 62, 32, + ]; + let a_vals: [i8; 64] = core::array::from_fn(|i| A[i % 16]); + let shift_vals: [i8; 64] = core::array::from_fn(|i| SHIFTS[i % 16]); + let expected: [i8; 64] = core::array::from_fn(|i| EXPECTED[i % 16]); + let a = i8x64::from_slice(simd, &a_vals); + let shifts = i8x64::from_slice(simd, &shift_vals); + assert_eq!(*(a << shifts), expected); +} + +#[simd_test] +fn shrv_i8x64(simd: S) { + const A: [i8; 16] = [ + -128, -64, -33, -1, 127, 64, 33, 1, -2, -4, -8, -16, 0, 2, 4, 8, + ]; + const SHIFTS: [i8; 16] = [1, 2, 3, 7, 1, 2, 3, 0, 1, 2, 3, 4, 0, 1, 2, 3]; + const EXPECTED: [i8; 16] = [-64, -16, -5, -1, 63, 16, 4, 1, -1, -1, -1, -1, 0, 1, 1, 1]; + let a_vals: [i8; 64] = core::array::from_fn(|i| A[i % 16]); + let shift_vals: [i8; 64] = core::array::from_fn(|i| SHIFTS[i % 16]); + let expected: [i8; 64] = core::array::from_fn(|i| EXPECTED[i % 16]); + let a = i8x64::from_slice(simd, &a_vals); + let shifts = i8x64::from_slice(simd, &shift_vals); + assert_eq!(*(a >> shifts), expected); +} + +#[simd_test] +fn shlv_u8x64(simd: S) { + const A: [u8; 16] = [255, 128, 64, 32, 16, 8, 4, 2, 1, 3, 5, 7, 15, 31, 63, 127]; + const SHIFTS: [u8; 16] = [4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 3, 2, 1]; + const EXPECTED: [u8; 16] = [240, 0, 0, 0, 0, 0, 0, 0, 1, 6, 20, 56, 240, 248, 252, 254]; + let a_vals: [u8; 64] = core::array::from_fn(|i| A[i % 16]); + let shift_vals: [u8; 64] = core::array::from_fn(|i| SHIFTS[i % 16]); + let expected: [u8; 64] = core::array::from_fn(|i| EXPECTED[i % 16]); + let a = u8x64::from_slice(simd, &a_vals); + let shifts = u8x64::from_slice(simd, &shift_vals); + assert_eq!(*(a << shifts), expected); +} + +#[simd_test] +fn shrv_u8x64(simd: S) { + const A: [u8; 16] = [255, 128, 64, 32, 16, 8, 4, 2, 1, 3, 5, 7, 15, 31, 63, 127]; + const SHIFTS: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 1, 0, 1, 2, 3, 4, 3, 2, 1]; + const EXPECTED: [u8; 16] = [127, 32, 8, 2, 0, 0, 0, 1, 1, 1, 1, 0, 0, 3, 15, 63]; + let a_vals: [u8; 64] = core::array::from_fn(|i| A[i % 16]); + let shift_vals: [u8; 64] = core::array::from_fn(|i| SHIFTS[i % 16]); + let expected: [u8; 64] = core::array::from_fn(|i| EXPECTED[i % 16]); + let a = u8x64::from_slice(simd, &a_vals); + let shifts = u8x64::from_slice(simd, &shift_vals); + assert_eq!(*(a >> shifts), expected); +} + +#[simd_test] +fn shlv_i16x32(simd: S) { + const A: [i16; 8] = [16384, 8192, -16384, -8192, 1, -1, 255, -256]; + const SHIFTS: [i16; 8] = [1, 2, 1, 2, 15, 1, 4, 3]; + const EXPECTED: [i16; 8] = [-32768, -32768, -32768, -32768, -32768, -2, 4080, -2048]; + let a_vals: [i16; 32] = core::array::from_fn(|i| A[i % 8]); + let shift_vals: [i16; 32] = core::array::from_fn(|i| SHIFTS[i % 8]); + let expected: [i16; 32] = core::array::from_fn(|i| EXPECTED[i % 8]); + let a = i16x32::from_slice(simd, &a_vals); + let shifts = i16x32::from_slice(simd, &shift_vals); + assert_eq!(*(a << shifts), expected); +} + +#[simd_test] +fn shrv_i16x32(simd: S) { + const A: [i16; 8] = [-32768, -16384, -1025, -1, 32767, 16384, 1025, 1]; + const SHIFTS: [i16; 8] = [1, 2, 3, 15, 1, 2, 3, 0]; + const EXPECTED: [i16; 8] = [-16384, -4096, -129, -1, 16383, 4096, 128, 1]; + let a_vals: [i16; 32] = core::array::from_fn(|i| A[i % 8]); + let shift_vals: [i16; 32] = core::array::from_fn(|i| SHIFTS[i % 8]); + let expected: [i16; 32] = core::array::from_fn(|i| EXPECTED[i % 8]); + let a = i16x32::from_slice(simd, &a_vals); + let shifts = i16x32::from_slice(simd, &shift_vals); + assert_eq!(*(a >> shifts), expected); +} + +#[simd_test] +fn shlv_u16x32(simd: S) { + const A: [u16; 8] = [65535, 32768, 16384, 8192, 1, 255, 1024, 4096]; + const SHIFTS: [u16; 8] = [4, 1, 2, 3, 15, 4, 5, 0]; + const EXPECTED: [u16; 8] = [65520, 0, 0, 0, 32768, 4080, 32768, 4096]; + let a_vals: [u16; 32] = core::array::from_fn(|i| A[i % 8]); + let shift_vals: [u16; 32] = core::array::from_fn(|i| SHIFTS[i % 8]); + let expected: [u16; 32] = core::array::from_fn(|i| EXPECTED[i % 8]); + let a = u16x32::from_slice(simd, &a_vals); + let shifts = u16x32::from_slice(simd, &shift_vals); + assert_eq!(*(a << shifts), expected); +} + +#[simd_test] +fn shrv_u16x32(simd: S) { + const A: [u16; 8] = [65535, 32768, 16384, 8192, 1, 255, 1024, 4096]; + const SHIFTS: [u16; 8] = [1, 2, 3, 4, 0, 4, 5, 12]; + const EXPECTED: [u16; 8] = [32767, 8192, 2048, 512, 1, 15, 32, 1]; + let a_vals: [u16; 32] = core::array::from_fn(|i| A[i % 8]); + let shift_vals: [u16; 32] = core::array::from_fn(|i| SHIFTS[i % 8]); + let expected: [u16; 32] = core::array::from_fn(|i| EXPECTED[i % 8]); + let a = u16x32::from_slice(simd, &a_vals); + let shifts = u16x32::from_slice(simd, &shift_vals); + assert_eq!(*(a >> shifts), expected); +} + #[simd_test] fn shrv_i32x16(simd: S) { let a = i32x16::from_slice( @@ -1359,6 +1598,166 @@ fn unzip_high_u32x16(simd: S) { ); } +#[simd_test] +fn zip_unzip_i16x32(simd: S) { + let a = i16x32::from_slice( + simd, + &[ + -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + ], + ); + let b = i16x32::from_slice( + simd, + &[ + 1000, 999, 998, 997, 996, 995, 994, 993, 992, 991, 990, 989, 988, 987, 986, 985, 984, + 983, 982, 981, 980, 979, 978, 977, 976, 975, 974, 973, 972, 971, 970, 969, + ], + ); + + assert_eq!( + *simd.zip_low_i16x32(a, b), + [ + -16, 1000, -15, 999, -14, 998, -13, 997, -12, 996, -11, 995, -10, 994, -9, 993, -8, + 992, -7, 991, -6, 990, -5, 989, -4, 988, -3, 987, -2, 986, -1, 985 + ] + ); + assert_eq!( + *simd.zip_high_i16x32(a, b), + [ + 0, 984, 1, 983, 2, 982, 3, 981, 4, 980, 5, 979, 6, 978, 7, 977, 8, 976, 9, 975, 10, + 974, 11, 973, 12, 972, 13, 971, 14, 970, 15, 969 + ] + ); + assert_eq!( + *simd.unzip_low_i16x32(a, b), + [ + -16, -14, -12, -10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10, 12, 14, 1000, 998, 996, 994, + 992, 990, 988, 986, 984, 982, 980, 978, 976, 974, 972, 970 + ] + ); + assert_eq!( + *simd.unzip_high_i16x32(a, b), + [ + -15, -13, -11, -9, -7, -5, -3, -1, 1, 3, 5, 7, 9, 11, 13, 15, 999, 997, 995, 993, 991, + 989, 987, 985, 983, 981, 979, 977, 975, 973, 971, 969 + ] + ); + + let (interleaved_low, interleaved_high) = simd.interleave_i16x32(a, b); + assert_eq!( + *interleaved_low, + [ + -16, 1000, -15, 999, -14, 998, -13, 997, -12, 996, -11, 995, -10, 994, -9, 993, -8, + 992, -7, 991, -6, 990, -5, 989, -4, 988, -3, 987, -2, 986, -1, 985 + ] + ); + assert_eq!( + *interleaved_high, + [ + 0, 984, 1, 983, 2, 982, 3, 981, 4, 980, 5, 979, 6, 978, 7, 977, 8, 976, 9, 975, 10, + 974, 11, 973, 12, 972, 13, 971, 14, 970, 15, 969 + ] + ); + + let (roundtrip_a, roundtrip_b) = simd.deinterleave_i16x32(interleaved_low, interleaved_high); + assert_eq!( + *roundtrip_a, + [ + -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ] + ); + assert_eq!( + *roundtrip_b, + [ + 1000, 999, 998, 997, 996, 995, 994, 993, 992, 991, 990, 989, 988, 987, 986, 985, 984, + 983, 982, 981, 980, 979, 978, 977, 976, 975, 974, 973, 972, 971, 970, 969 + ] + ); +} + +#[simd_test] +fn zip_unzip_u16x32(simd: S) { + let a = u16x32::from_slice( + simd, + &[ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + ], + ); + let b = u16x32::from_slice( + simd, + &[ + 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, + 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, + 1028, 1029, 1030, 1031, + ], + ); + + assert_eq!( + *simd.zip_low_u16x32(a, b), + [ + 0, 1000, 1, 1001, 2, 1002, 3, 1003, 4, 1004, 5, 1005, 6, 1006, 7, 1007, 8, 1008, 9, + 1009, 10, 1010, 11, 1011, 12, 1012, 13, 1013, 14, 1014, 15, 1015 + ] + ); + assert_eq!( + *simd.zip_high_u16x32(a, b), + [ + 16, 1016, 17, 1017, 18, 1018, 19, 1019, 20, 1020, 21, 1021, 22, 1022, 23, 1023, 24, + 1024, 25, 1025, 26, 1026, 27, 1027, 28, 1028, 29, 1029, 30, 1030, 31, 1031 + ] + ); + assert_eq!( + *simd.unzip_low_u16x32(a, b), + [ + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 1000, 1002, 1004, 1006, + 1008, 1010, 1012, 1014, 1016, 1018, 1020, 1022, 1024, 1026, 1028, 1030 + ] + ); + assert_eq!( + *simd.unzip_high_u16x32(a, b), + [ + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 1001, 1003, 1005, 1007, + 1009, 1011, 1013, 1015, 1017, 1019, 1021, 1023, 1025, 1027, 1029, 1031 + ] + ); + + let (interleaved_low, interleaved_high) = simd.interleave_u16x32(a, b); + assert_eq!( + *interleaved_low, + [ + 0, 1000, 1, 1001, 2, 1002, 3, 1003, 4, 1004, 5, 1005, 6, 1006, 7, 1007, 8, 1008, 9, + 1009, 10, 1010, 11, 1011, 12, 1012, 13, 1013, 14, 1014, 15, 1015 + ] + ); + assert_eq!( + *interleaved_high, + [ + 16, 1016, 17, 1017, 18, 1018, 19, 1019, 20, 1020, 21, 1021, 22, 1022, 23, 1023, 24, + 1024, 25, 1025, 26, 1026, 27, 1027, 28, 1028, 29, 1029, 30, 1030, 31, 1031 + ] + ); + + let (roundtrip_a, roundtrip_b) = simd.deinterleave_u16x32(interleaved_low, interleaved_high); + assert_eq!( + *roundtrip_a, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 + ] + ); + assert_eq!( + *roundtrip_b, + [ + 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, + 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, + 1028, 1029, 1030, 1031 + ] + ); +} + // ============================================================================= // interleave tests (512-bit) // ============================================================================= diff --git a/fearless_simd_tests/tests/harness/lm_generated/mask_methods.rs b/fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip.rs similarity index 76% rename from fearless_simd_tests/tests/harness/lm_generated/mask_methods.rs rename to fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip.rs index 15963b2a3..5433ce2a6 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/mask_methods.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip.rs @@ -4,6 +4,86 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +/// Verifies that `SimdMask::set` can set and clear every lane while keeping +/// `to_bitmask` and `test` in sync with the expected compact bitmask. +fn assert_mask_set_roundtrip>(simd: S) { + let mut mask = M::from_bitmask(simd, 0); + let mut expected = 0_u64; + for i in 0..M::N { + mask.set(i, true); + expected |= 1_u64 << i; + assert_eq!(mask.to_bitmask(), expected); + assert!(mask.test(i)); + } + + for i in 0..M::N { + mask.set(i, false); + expected &= !(1_u64 << i); + assert_eq!(mask.to_bitmask(), expected); + assert!(!mask.test(i)); + } +} + +#[simd_test] +fn mask8x16_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask16x8_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask32x4_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask64x2_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask8x32_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask16x16_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask32x8_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask64x4_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask8x64_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask16x32_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask32x16_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + +#[simd_test] +fn mask64x8_set_roundtrip(simd: S) { + assert_mask_set_roundtrip::>(simd); +} + #[simd_test] fn mask8x16_bitmask_roundtrip(simd: S) { for bits in 0..=0xffff_u64 { diff --git a/fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip_x86.rs b/fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip_x86.rs new file mode 100644 index 000000000..cade583d3 --- /dev/null +++ b/fearless_simd_tests/tests/harness/lm_generated/mask_roundtrip_x86.rs @@ -0,0 +1,254 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; +use core::convert::TryFrom; +use core::mem::size_of; + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +fn lane_mask(lanes: usize) -> u64 { + if lanes == u64::BITS as usize { + u64::MAX + } else { + (1_u64 << lanes) - 1 + } +} + +fn lanes_from_bits(bits: u64) -> [L; LANES] +where + L: Copy + From, +{ + let bits = bits & lane_mask(LANES); + core::array::from_fn(|i| { + if ((bits >> i) & 1) != 0 { + L::from(-1) + } else { + L::from(0) + } + }) +} + +fn assert_native_vector_roundtrip(simd: S, bits: u64) +where + S: Simd, + M: SimdMask + SimdFrom + Into, + A: Copy, + L: Copy + Eq + core::fmt::Debug + From, +{ + let expected_bits = bits & lane_mask(LANES); + let expected_lanes = lanes_from_bits::(bits); + + assert_eq!(size_of::(), size_of::<[L; LANES]>()); + + let mask = M::from_bitmask(simd, bits); + let arch: A = mask.into(); + // Safety: the size assertion above verifies that the x86 vector type has + // the same size as the signed integer lane representation used for masks. + let lanes = unsafe { core::mem::transmute_copy::(&arch) }; + assert_eq!(lanes, expected_lanes); + + // Safety: this builds the native x86 vector value from the lane + // representation expected by the public mask conversion. + let arch = unsafe { core::mem::transmute_copy::<[L; LANES], A>(&expected_lanes) }; + let mask = M::simd_from(simd, arch); + assert_eq!(mask.to_bitmask(), expected_bits); +} + +fn assert_native_mask_roundtrip(simd: S, bits: u64) +where + S: Simd, + M: SimdMask + SimdFrom + Into, + A: Copy + Eq + core::fmt::Debug + TryFrom, + A::Error: core::fmt::Debug, +{ + let expected_bits = bits & lane_mask(LANES); + let expected_arch = A::try_from(expected_bits).expect("masked bits fit in native mask type"); + + let mask = M::from_bitmask(simd, bits); + let arch: A = mask.into(); + assert_eq!(arch, expected_arch); + + let mask = M::simd_from(simd, expected_arch); + assert_eq!(mask.to_bitmask(), expected_bits); + + let arch: A = mask.into(); + assert_eq!(arch, expected_arch); +} + +#[simd_test] +fn mask8x16_m128i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m128i, i8, 16>(simd, bits); + } +} + +#[simd_test] +fn mask16x8_m128i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m128i, i16, 8>(simd, bits); + } +} + +#[simd_test] +fn mask32x4_m128i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m128i, i32, 4>(simd, bits); + } +} + +#[simd_test] +fn mask64x2_m128i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m128i, i64, 2>(simd, bits); + } +} + +#[simd_test] +fn mask8x32_m256i_roundtrip(simd: S) { + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x0000_0000); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x0000_0001); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x8000_0000); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x0000_ffff); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xffff_0000); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x5555_5555); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xaaaa_aaaa); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0x8000_aa55); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xffff_ffff); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xffff_ffff_0000_0000); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xffff_ffff_8000_aa55); + assert_native_vector_roundtrip::, __m256i, i8, 32>(simd, 0xffff_ffff_ffff_ffff); +} + +#[simd_test] +fn mask16x16_m256i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m256i, i16, 16>(simd, bits); + } +} + +#[simd_test] +fn mask32x8_m256i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m256i, i32, 8>(simd, bits); + } +} + +#[simd_test] +fn mask64x4_m256i_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_vector_roundtrip::, __m256i, i64, 4>(simd, bits); + } +} + +#[simd_test] +fn mask8x16_mmask16_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask16, 16>(simd, bits); + } +} + +#[simd_test] +fn mask16x8_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 8>(simd, bits); + } +} + +#[simd_test] +fn mask32x4_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 4>(simd, bits); + } +} + +#[simd_test] +fn mask64x2_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 2>(simd, bits); + } +} + +#[simd_test] +fn mask8x32_mmask32_roundtrip(simd: S) { + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_0001); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x8000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_ffff); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x5555_5555); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xaaaa_aaaa); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x8000_aa55); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_0000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_8000_aa55); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_ffff_ffff); +} + +#[simd_test] +fn mask16x16_mmask16_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask16, 16>(simd, bits); + } +} + +#[simd_test] +fn mask32x8_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 8>(simd, bits); + } +} + +#[simd_test] +fn mask64x4_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 4>(simd, bits); + } +} + +#[simd_test] +fn mask8x64_mmask64_roundtrip(simd: S) { + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x0000_0000_0000_0000); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x0000_0000_0000_0001); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x8000_0000_0000_0000); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x0000_0000_ffff_ffff); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0xffff_ffff_0000_0000); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x5555_5555_5555_5555); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0xaaaa_aaaa_aaaa_aaaa); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0x8000_0001_5555_aaab); + assert_native_mask_roundtrip::, __mmask64, 64>(simd, 0xffff_ffff_ffff_ffff); +} + +#[simd_test] +fn mask16x32_mmask32_roundtrip(simd: S) { + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_0001); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x8000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x0000_ffff); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x5555_5555); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xaaaa_aaaa); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0x8000_aa55); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_0000_0000); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_8000_aa55); + assert_native_mask_roundtrip::, __mmask32, 32>(simd, 0xffff_ffff_ffff_ffff); +} + +#[simd_test] +fn mask32x16_mmask16_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask16, 16>(simd, bits); + } +} + +#[simd_test] +fn mask64x8_mmask8_roundtrip(simd: S) { + for bits in 0..=0xffff_u64 { + assert_native_mask_roundtrip::, __mmask8, 8>(simd, bits); + } +} diff --git a/fearless_simd_tests/tests/harness/lm_generated/mod_256.rs b/fearless_simd_tests/tests/harness/lm_generated/mod_256.rs index 2d01f40e9..f9736013c 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/mod_256.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/mod_256.rs @@ -65,6 +65,30 @@ fn sqrt_f32x8(simd: S) { ); } +#[simd_test] +fn approximate_recip_f32x8(simd: S) { + let a = f32x8::from_slice(simd, &[1.0, -2.0, 23.0, 9.0, 3.5, -7.25, 13.0, 0.25]); + let result = a.approximate_recip(); + let expected = [ + 1.0, + -0.5, + 1. / 23., + 1. / 9., + 1. / 3.5, + 1. / -7.25, + 1. / 13., + 4.0, + ]; + for i in 0..8 { + let rel_error = ((result[i] - expected[i]) / expected[i]).abs(); + assert!( + rel_error < 0.005, + "approximate_recip({}) rel_error = {rel_error}", + a[i] + ); + } +} + #[simd_test] fn div_f32x8(simd: S) { let a = f32x8::from_slice(simd, &[4.0, 2.0, 1.0, 0.0, 10.0, 12.0, 15.0, 20.0]); @@ -235,6 +259,44 @@ fn min_precise_f32x8_with_nan(simd: S) { assert_eq!(result[7], 5.0); } +#[simd_test] +fn max_precise_f64x4(simd: S) { + let a = f64x4::from_slice(simd, &[2.0, -3.0, 0.0, 0.5]); + let b = f64x4::from_slice(simd, &[1.0, -2.0, 7.0, 3.0]); + assert_eq!(*a.max_precise(b), [2.0, -2.0, 7.0, 3.0]); +} + +#[simd_test] +fn min_precise_f64x4(simd: S) { + let a = f64x4::from_slice(simd, &[2.0, -3.0, 0.0, 0.5]); + let b = f64x4::from_slice(simd, &[1.0, -2.0, 7.0, 3.0]); + assert_eq!(*a.min_precise(b), [1.0, -3.0, 0.0, 0.5]); +} + +#[simd_test] +fn max_precise_f64x4_with_nan(simd: S) { + let a = f64x4::from_slice(simd, &[f64::NAN, -3.0, f64::INFINITY, 0.5]); + let b = f64x4::from_slice(simd, &[1.0, f64::NAN, 7.0, f64::NEG_INFINITY]); + let result = a.max_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); + assert_eq!(result[2], f64::INFINITY); + assert_eq!(result[3], 0.5); +} + +#[simd_test] +fn min_precise_f64x4_with_nan(simd: S) { + let a = f64x4::from_slice(simd, &[f64::NAN, -3.0, f64::INFINITY, 0.5]); + let b = f64x4::from_slice(simd, &[1.0, f64::NAN, 7.0, f64::NEG_INFINITY]); + let result = a.min_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); + assert_eq!(result[2], 7.0); + assert_eq!(result[3], f64::NEG_INFINITY); +} + #[simd_test] fn floor_f32x8(simd: S) { let a = f32x8::from_slice(simd, &[2.0, -3.2, 0.0, 0.5, 1.7, -2.8, 3.1, -4.9]); @@ -315,6 +377,92 @@ fn trunc_f32x8_special_values(simd: S) { ); } +#[simd_test] +fn cvt_u32_f32x8(simd: S) { + let a = f32x8::from_slice(simd, &[1.0, 42.7, 3e9, -0.3, 0.0, 17.9, 255.99, 1024.1]); + assert_eq!( + *a.to_int::>(), + [1, 42, 3000000000, 0, 0, 17, 255, 1024] + ); +} + +#[simd_test] +fn cvt_u32_precise_f32x8(simd: S) { + let a = f32x8::from_slice( + simd, + &[-1.0, 42.7, 5e9, f32::NAN, 0.0, 1.9, 3000000000.0, -5e9], + ); + assert_eq!( + *a.to_int_precise::>(), + [0, 42, u32::MAX, 0, 0, 1, 3000000000, 0] + ); +} + +#[simd_test] +fn cvt_i32_precise_f32x8(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + -10.3, + f32::NAN, + 5e9, + -5e9, + f32::INFINITY, + f32::NEG_INFINITY, + 42.7, + -0.9, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [-10, 0, i32::MAX, i32::MIN, i32::MAX, i32::MIN, 42, 0] + ); +} + +#[simd_test] +fn cvt_u32_f32x8_rounding(simd: S) { + let a = f32x8::from_slice(simd, &[0.0, 0.49, 0.51, 0.99, 1.01, 1.99, 2.5, 3.75]); + assert_eq!(*a.to_int::>(), [0, 0, 0, 0, 1, 1, 2, 3]); +} + +#[simd_test] +fn cvt_f32_u32x8(simd: S) { + let values = [ + 0, + 42, + 1_000_000, + i32::MAX as u32, + 0x8000_0000, + 0xffff_ff00, + u32::MAX - 1, + u32::MAX, + ]; + let a = u32x8::from_slice(simd, &values); + assert_eq!(*a.to_float::>(), values.map(|x| x as f32)); +} + +#[simd_test] +fn cvt_u32_precise_f32x8_inf(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + -10.3, + f32::NAN, + f32::INFINITY, + f32::NEG_INFINITY, + u32::MAX as f32, + 4294967040.0, + 4294967296.0, + -0.5, + ], + ); + + assert_eq!( + *a.to_int_precise::>(), + [0, 0, u32::MAX, u32::MIN, u32::MAX, 4294967040, u32::MAX, 0] + ); +} + #[simd_test] fn select_f32x8(simd: S) { let a = f32x8::from_slice(simd, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index 4ef949267..ce22f42ce 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -11,6 +11,7 @@ //! Tests for `fearless_simd`. +mod int64; mod lm_generated; use fearless_simd::*; @@ -204,6 +205,40 @@ fn min_precise_f32x4_with_nan(simd: S) { assert_eq!(result[3], f32::NEG_INFINITY); } +#[simd_test] +fn max_precise_f64x2(simd: S) { + let a = f64x2::from_slice(simd, &[2.0, -3.0]); + let b = f64x2::from_slice(simd, &[1.0, -2.0]); + assert_eq!(*a.max_precise(b), [2.0, -2.0]); +} + +#[simd_test] +fn min_precise_f64x2(simd: S) { + let a = f64x2::from_slice(simd, &[2.0, -3.0]); + let b = f64x2::from_slice(simd, &[1.0, -2.0]); + assert_eq!(*a.min_precise(b), [1.0, -3.0]); +} + +#[simd_test] +fn max_precise_f64x2_with_nan(simd: S) { + let a = f64x2::from_slice(simd, &[f64::NAN, -3.0]); + let b = f64x2::from_slice(simd, &[1.0, f64::NAN]); + let result = a.max_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); +} + +#[simd_test] +fn min_precise_f64x2_with_nan(simd: S) { + let a = f64x2::from_slice(simd, &[f64::NAN, -3.0]); + let b = f64x2::from_slice(simd, &[1.0, f64::NAN]); + let result = a.min_precise(b); + + assert_eq!(result[0], 1.0); + assert_eq!(result[1], -3.0); +} + #[simd_test] fn floor_f32x4(simd: S) { let a = f32x4::from_slice(simd, &[2.0, -3.2, 0.0, 0.5]); @@ -694,47 +729,63 @@ fn combine_u8x16(simd: S) { #[simd_test] fn and_mask8x16(simd: S) { - let a = mask8x16::from_slice(simd, &[1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0]); + let a = mask8x16::from_slice( + simd, + &[-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0], + ); let b = mask8x16::from_slice( simd, &[ - 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, ], ); assert_eq!( <[i8; 16]>::from(a & b), - [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0] + [-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0] ); } #[simd_test] fn or_mask8x16(simd: S) { - let a = mask8x16::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8]); - let b = mask8x16::from_slice(simd, &[1, 1, 1, 1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0]); + let a = mask8x16::from_slice( + simd, + &[0, -1, 0, -1, 0, -1, 0, -1, -1, 0, -1, 0, -1, 0, -1, 0], + ); + let b = mask8x16::from_slice( + simd, + &[0, 0, -1, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, -1, 0, -1], + ); assert_eq!( <[i8; 16]>::from(a | b), - [1, 1, 3, 3, 6, 7, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8] + [0, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] ); } #[simd_test] fn xor_mask8x16(simd: S) { - let a = mask8x16::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7, 1, 1, 1, 1, 0, 0, 0, 0]); - let b = mask8x16::from_slice(simd, &[1, 1, 0, 0, 5, 4, 7, 6, 1, 0, 1, 0, 1, 0, 1, 0]); + let a = mask8x16::from_slice( + simd, + &[0, -1, -1, 0, -1, 0, 0, -1, -1, -1, 0, 0, -1, -1, 0, 0], + ); + let b = mask8x16::from_slice( + simd, + &[-1, -1, 0, 0, -1, -1, 0, 0, -1, 0, -1, 0, -1, 0, -1, 0], + ); assert_eq!( <[i8; 16]>::from(a ^ b), - [1, 0, 2, 3, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0] + [-1, 0, -1, 0, 0, -1, 0, -1, 0, -1, -1, 0, 0, -1, -1, 0] ); } #[simd_test] fn not_mask8x16(simd: S) { - let a = mask8x16::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8]); + let a = mask8x16::from_slice( + simd, + &[0, -1, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, 0, -1, 0, -1], + ); assert_eq!( <[i8; 16]>::from(!a), - [ - -1, -2, -3, -4, -5, -6, -7, -8, -2, -3, -4, -5, -6, -7, -8, -9 - ] + [-1, 0, 0, -1, 0, -1, -1, 0, 0, -1, 0, -1, -1, 0, -1, 0] ); } @@ -2463,6 +2514,139 @@ fn shlv_u32x4_varied(simd: S) { ); } +#[simd_test] +fn shlv_i8x16(simd: S) { + let a = i8x16::from_slice( + simd, + &[64, 65, -64, -65, 1, 2, 3, 4, -1, -2, -3, -4, 15, 16, 31, 32], + ); + let shifts = i8x16::from_slice(simd, &[1, 2, 1, 2, 0, 1, 2, 3, 1, 2, 3, 4, 3, 2, 1, 0]); + assert_eq!( + *(a << shifts), + [ + -128, 4, -128, -4, 1, 4, 12, 32, -2, -8, -24, -64, 120, 64, 62, 32 + ] + ); +} + +#[simd_test] +fn shrv_i8x16(simd: S) { + let a = i8x16::from_slice( + simd, + &[ + -128, -64, -33, -1, 127, 64, 33, 1, -2, -4, -8, -16, 0, 2, 4, 8, + ], + ); + let shifts = i8x16::from_slice(simd, &[1, 2, 3, 7, 1, 2, 3, 0, 1, 2, 3, 4, 0, 1, 2, 3]); + assert_eq!( + *(a >> shifts), + [-64, -16, -5, -1, 63, 16, 4, 1, -1, -1, -1, -1, 0, 1, 1, 1] + ); +} + +#[simd_test] +fn shlv_u8x16(simd: S) { + let a = u8x16::from_slice( + simd, + &[255, 128, 64, 32, 16, 8, 4, 2, 1, 3, 5, 7, 15, 31, 63, 127], + ); + let shifts = u8x16::from_slice(simd, &[4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 3, 2, 1]); + assert_eq!( + *(a << shifts), + [240, 0, 0, 0, 0, 0, 0, 0, 1, 6, 20, 56, 240, 248, 252, 254] + ); +} + +#[simd_test] +fn shrv_u8x16(simd: S) { + let a = u8x16::from_slice( + simd, + &[255, 128, 64, 32, 16, 8, 4, 2, 1, 3, 5, 7, 15, 31, 63, 127], + ); + let shifts = u8x16::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 1, 0, 1, 2, 3, 4, 3, 2, 1]); + assert_eq!( + *(a >> shifts), + [127, 32, 8, 2, 0, 0, 0, 1, 1, 1, 1, 0, 0, 3, 15, 63] + ); +} + +#[simd_test] +fn shlv_i16x8(simd: S) { + let a = i16x8::from_slice(simd, &[16384, 8192, -16384, -8192, 1, -1, 255, -256]); + let shifts = i16x8::from_slice(simd, &[1, 2, 1, 2, 15, 1, 4, 3]); + assert_eq!( + *(a << shifts), + [-32768, -32768, -32768, -32768, -32768, -2, 4080, -2048] + ); +} + +#[simd_test] +fn shrv_i16x8(simd: S) { + let a = i16x8::from_slice(simd, &[-32768, -16384, -1025, -1, 32767, 16384, 1025, 1]); + let shifts = i16x8::from_slice(simd, &[1, 2, 3, 15, 1, 2, 3, 0]); + assert_eq!( + *(a >> shifts), + [-16384, -4096, -129, -1, 16383, 4096, 128, 1] + ); +} + +#[simd_test] +fn shlv_u16x8(simd: S) { + let a = u16x8::from_slice(simd, &[65535, 32768, 16384, 8192, 1, 255, 1024, 4096]); + let shifts = u16x8::from_slice(simd, &[4, 1, 2, 3, 15, 4, 5, 0]); + assert_eq!(*(a << shifts), [65520, 0, 0, 0, 32768, 4080, 32768, 4096]); +} + +#[simd_test] +fn shrv_u16x8(simd: S) { + let a = u16x8::from_slice(simd, &[65535, 32768, 16384, 8192, 1, 255, 1024, 4096]); + let shifts = u16x8::from_slice(simd, &[1, 2, 3, 4, 0, 4, 5, 12]); + assert_eq!(*(a >> shifts), [32767, 8192, 2048, 512, 1, 15, 32, 1]); +} + +#[simd_test] +fn shlv_u8x32(simd: S) { + let a = u8x32::from_slice( + simd, + &[ + 255, 128, 64, 32, 16, 8, 4, 2, 1, 3, 5, 7, 15, 31, 63, 127, 255, 128, 64, 32, 16, 8, 4, + 2, 1, 3, 5, 7, 15, 31, 63, 127, + ], + ); + let shifts = u8x32::from_slice( + simd, + &[ + 4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 3, 2, 1, 4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, + 3, 2, 1, + ], + ); + assert_eq!( + *(a << shifts), + [ + 240, 0, 0, 0, 0, 0, 0, 0, 1, 6, 20, 56, 240, 248, 252, 254, 240, 0, 0, 0, 0, 0, 0, 0, + 1, 6, 20, 56, 240, 248, 252, 254 + ] + ); +} + +#[simd_test] +fn shlv_u16x16(simd: S) { + let a = u16x16::from_slice( + simd, + &[ + 65535, 32768, 16384, 8192, 1, 255, 1024, 4096, 65535, 32768, 16384, 8192, 1, 255, 1024, + 4096, + ], + ); + let shifts = u16x16::from_slice(simd, &[4, 1, 2, 3, 15, 4, 5, 0, 4, 1, 2, 3, 15, 4, 5, 0]); + assert_eq!( + *(a << shifts), + [ + 65520, 0, 0, 0, 32768, 4080, 32768, 4096, 65520, 0, 0, 0, 32768, 4080, 32768, 4096 + ] + ); +} + #[simd_test] fn add_i16x8(simd: S) { let a = i16x8::from_slice(simd, &[1, 2, 3, 4, 5, 6, 7, 8]); @@ -3245,10 +3429,25 @@ fn sqrt_f64x2(simd: S) { #[simd_test] fn approximate_recip_f64x2(simd: S) { + let a = f64x2::from_slice(simd, &[1.0, -2.0]); + let result = a.approximate_recip(); + let expected = [1.0, -0.5]; + for i in 0..2 { + let rel_error = ((result[i] - expected[i]) / expected[i]).abs(); + assert!( + rel_error < 0.005, + "approximate_recip({}) rel_error = {rel_error}", + a[i] + ); + } +} + +#[simd_test] +fn approximate_recip_f64x4(simd: S) { let a = f64x4::from_slice(simd, &[1.0, -2.0, 23.0, 9.0]); let result = a.approximate_recip(); let expected = [1.0, -0.5, 1. / 23., 1. / 9.]; - for i in 0..2 { + for i in 0..4 { let rel_error = ((result[i] - expected[i]) / expected[i]).abs(); assert!( rel_error < 0.005, diff --git a/fearless_simd_tests/tests/harness/slide_exhaustive.rs b/fearless_simd_tests/tests/harness/slide_exhaustive.rs index 1b82d4548..78e30d7e2 100644 --- a/fearless_simd_tests/tests/harness/slide_exhaustive.rs +++ b/fearless_simd_tests/tests/harness/slide_exhaustive.rs @@ -225,6 +225,8 @@ macro_rules! test_slide_exhaustive { // 128-bit vectors (block size == vector size, so within_blocks uses same range as vector-wide) test_slide_exhaustive!(slide_exhaustive_f32x4, f32x4, f32, 4, vec4, block4); test_slide_exhaustive!(slide_exhaustive_f64x2, f64x2, f64, 2, vec2, block2); +test_slide_exhaustive!(slide_exhaustive_i64x2, i64x2, i64, 2, vec2, block2); +test_slide_exhaustive!(slide_exhaustive_u64x2, u64x2, u64, 2, vec2, block2); test_slide_exhaustive!(slide_exhaustive_i8x16, i8x16, i8, 16, vec16, block16); test_slide_exhaustive!(slide_exhaustive_u8x16, u8x16, u8, 16, vec16, block16); test_slide_exhaustive!(slide_exhaustive_i16x8, i16x8, i16, 8, vec8, block8); @@ -235,6 +237,8 @@ test_slide_exhaustive!(slide_exhaustive_u32x4, u32x4, u32, 4, vec4, block4); // 256-bit vectors (block size = 128 bits = half the vector size) test_slide_exhaustive!(slide_exhaustive_f32x8, f32x8, f32, 8, vec8, block4); test_slide_exhaustive!(slide_exhaustive_f64x4, f64x4, f64, 4, vec4, block2); +test_slide_exhaustive!(slide_exhaustive_i64x4, i64x4, i64, 4, vec4, block2); +test_slide_exhaustive!(slide_exhaustive_u64x4, u64x4, u64, 4, vec4, block2); test_slide_exhaustive!(slide_exhaustive_i8x32, i8x32, i8, 32, vec32, block16); test_slide_exhaustive!(slide_exhaustive_u8x32, u8x32, u8, 32, vec32, block16); test_slide_exhaustive!(slide_exhaustive_i16x16, i16x16, i16, 16, vec16, block8); @@ -245,48 +249,11 @@ test_slide_exhaustive!(slide_exhaustive_u32x8, u32x8, u32, 8, vec8, block4); // 512-bit vectors (block size = 128 bits = quarter the vector size) test_slide_exhaustive!(slide_exhaustive_f32x16, f32x16, f32, 16, vec16, block4); test_slide_exhaustive!(slide_exhaustive_f64x8, f64x8, f64, 8, vec8, block2); +test_slide_exhaustive!(slide_exhaustive_i64x8, i64x8, i64, 8, vec8, block2); +test_slide_exhaustive!(slide_exhaustive_u64x8, u64x8, u64, 8, vec8, block2); test_slide_exhaustive!(slide_exhaustive_i8x64, i8x64, i8, 64, vec64, block16); test_slide_exhaustive!(slide_exhaustive_u8x64, u8x64, u8, 64, vec64, block16); test_slide_exhaustive!(slide_exhaustive_i16x32, i16x32, i16, 32, vec32, block8); test_slide_exhaustive!(slide_exhaustive_u16x32, u16x32, u16, 32, vec32, block8); test_slide_exhaustive!(slide_exhaustive_i32x16, i32x16, i32, 16, vec16, block4); test_slide_exhaustive!(slide_exhaustive_u32x16, u32x16, u32, 16, vec16, block4); - -// Mask types (128-bit) -test_slide_exhaustive!(slide_exhaustive_mask8x16, mask8x16, i8, 16, vec16, block16); -test_slide_exhaustive!(slide_exhaustive_mask16x8, mask16x8, i16, 8, vec8, block8); -test_slide_exhaustive!(slide_exhaustive_mask32x4, mask32x4, i32, 4, vec4, block4); -test_slide_exhaustive!(slide_exhaustive_mask64x2, mask64x2, i64, 2, vec2, block2); - -// Mask types (256-bit) -test_slide_exhaustive!(slide_exhaustive_mask8x32, mask8x32, i8, 32, vec32, block16); -test_slide_exhaustive!( - slide_exhaustive_mask16x16, - mask16x16, - i16, - 16, - vec16, - block8 -); -test_slide_exhaustive!(slide_exhaustive_mask32x8, mask32x8, i32, 8, vec8, block4); -test_slide_exhaustive!(slide_exhaustive_mask64x4, mask64x4, i64, 4, vec4, block2); - -// Mask types (512-bit) -test_slide_exhaustive!(slide_exhaustive_mask8x64, mask8x64, i8, 64, vec64, block16); -test_slide_exhaustive!( - slide_exhaustive_mask16x32, - mask16x32, - i16, - 32, - vec32, - block8 -); -test_slide_exhaustive!( - slide_exhaustive_mask32x16, - mask32x16, - i32, - 16, - vec16, - block4 -); -test_slide_exhaustive!(slide_exhaustive_mask64x8, mask64x8, i64, 8, vec8, block2); diff --git a/fearless_simd_tests/tests/mod.rs b/fearless_simd_tests/tests/mod.rs index 4d2f053d8..bd64c14c9 100644 --- a/fearless_simd_tests/tests/mod.rs +++ b/fearless_simd_tests/tests/mod.rs @@ -5,13 +5,53 @@ missing_docs, reason = "TODO: https://github.com/linebender/fearless_simd/issues/40" )] +#![allow( + clippy::disallowed_methods, + reason = "fearless_simd_tests has test-only transmute helpers that should not be forced through the library's private checked transmute machinery" +)] use fearless_simd::*; use fearless_simd_dev_macros::simd_test; mod harness; +#[cfg(not(miri))] // too slow mod soundness; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn x86_detects_icelake_avx512() -> bool { + std::arch::is_x86_feature_detected!("adx") + && std::arch::is_x86_feature_detected!("aes") + && std::arch::is_x86_feature_detected!("avx512bitalg") + && std::arch::is_x86_feature_detected!("avx512bw") + && std::arch::is_x86_feature_detected!("avx512cd") + && std::arch::is_x86_feature_detected!("avx512dq") + && std::arch::is_x86_feature_detected!("avx512f") + && std::arch::is_x86_feature_detected!("avx512ifma") + && std::arch::is_x86_feature_detected!("avx512vbmi") + && std::arch::is_x86_feature_detected!("avx512vbmi2") + && std::arch::is_x86_feature_detected!("avx512vl") + && std::arch::is_x86_feature_detected!("avx512vnni") + && std::arch::is_x86_feature_detected!("avx512vpopcntdq") + && std::arch::is_x86_feature_detected!("bmi1") + && std::arch::is_x86_feature_detected!("bmi2") + && std::arch::is_x86_feature_detected!("cmpxchg16b") + && std::arch::is_x86_feature_detected!("fma") + && std::arch::is_x86_feature_detected!("gfni") + && std::arch::is_x86_feature_detected!("lzcnt") + && std::arch::is_x86_feature_detected!("movbe") + && std::arch::is_x86_feature_detected!("pclmulqdq") + && std::arch::is_x86_feature_detected!("popcnt") + && std::arch::is_x86_feature_detected!("rdrand") + && std::arch::is_x86_feature_detected!("rdseed") + && std::arch::is_x86_feature_detected!("sha") + && std::arch::is_x86_feature_detected!("vaes") + && std::arch::is_x86_feature_detected!("vpclmulqdq") + && std::arch::is_x86_feature_detected!("xsave") + && std::arch::is_x86_feature_detected!("xsavec") + && std::arch::is_x86_feature_detected!("xsaveopt") + && std::arch::is_x86_feature_detected!("xsaves") +} + // Ensure that we can cast between generic native-width vectors #[expect(dead_code, reason = "Compile only test")] fn generic_cast(x: S::f32s) -> S::u32s { @@ -45,7 +85,7 @@ fn supports_highest_level() { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] assert!( level.as_avx2().is_some(), - "This machine does not support every `Level` supported by Fearless SIMD (currently AVX2 and below).\n{UNSUPPORTED_LEVEL_MESSAGE}", + "This machine does not support every routinely local-tested x86 `Level` supported by Fearless SIMD (currently AVX2 and below; AVX-512 is covered by the SDE CI job).\n{UNSUPPORTED_LEVEL_MESSAGE}", ); #[cfg(target_arch = "aarch64")] @@ -62,6 +102,53 @@ fn supports_highest_level() { ); } +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[test] +fn detects_avx512_when_available() { + if !x86_detects_icelake_avx512() { + return; + } + + let level = Level::new(); + assert!( + level.as_avx512().is_some(), + "Ice Lake AVX-512 should be selected when all required features are available" + ); + assert!( + level.as_avx2().is_some(), + "AVX-512 should downgrade to an AVX2 proof" + ); + assert!( + level.as_sse4_2().is_some(), + "AVX-512 should downgrade to an SSE4.2 proof" + ); +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[test] +fn avx512_masks_are_compact() { + #[cfg(target_arch = "x86")] + use core::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::*; + use std::mem::size_of; + + type A = Avx512; + + assert_eq!(size_of::>(), size_of::<__mmask16>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); + assert_eq!(size_of::>(), size_of::<__mmask32>()); + assert_eq!(size_of::>(), size_of::<__mmask16>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); + assert_eq!(size_of::>(), size_of::<__mmask64>()); + assert_eq!(size_of::>(), size_of::<__mmask32>()); + assert_eq!(size_of::>(), size_of::<__mmask16>()); + assert_eq!(size_of::>(), size_of::<__mmask8>()); +} + #[simd_test] #[ignore] fn test_f32_to_i32_precise_exhaustive(simd: S) {