diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index d6f622d..a09aa2f 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -22,3 +22,12 @@ jobs: tests: uses: "SciML/.github/.github/workflows/grouped-tests.yml@v1" secrets: "inherit" + tests-32bit: + name: "Core (julia 1, ubuntu-latest, x86)" + uses: "SciML/.github/.github/workflows/tests.yml@v1" + with: + julia-version: "1" + julia-arch: "x86" + group: "Core" + coverage: false + secrets: "inherit" diff --git a/src/equality.jl b/src/equality.jl index 14feb03..89099dc 100644 --- a/src/equality.jl +++ b/src/equality.jl @@ -14,15 +14,16 @@ This function does **not** assume `A` is sorted. For sorted vectors, see `DenseVector{Int64}`) or [`findequal`](@ref) (the strategy-framework equality wrapper that returns an `Int` with a sentinel). -The `(x::Int64, A::DenseVector{Int64})` method uses a custom LLVM IR SIMD -scan (load 8 lanes, `icmp eq`, `cttz` on the mask) — about 8× faster than -the scalar `findfirst(==(x), v)` on modern x86-64. Every other element-type +On 64-bit platforms the `(x::Int64, A::DenseVector{Int64})` method uses a +custom LLVM IR SIMD scan (load 8 lanes, `icmp eq`, `cttz` on the mask) — +about 8× faster than the scalar `findfirst(==(x), v)` on modern x86-64; +on 32-bit platforms it uses a scalar loop. Every other element-type and array-storage combination falls back to `findfirst(isequal(x), A)`. """ findfirstequal(vpivot, ivars) = findfirst(isequal(vpivot), ivars) function findfirstequal(vpivot::Int64, ivars::DenseVector{Int64}) GC.@preserve ivars begin - ret = _findfirstequal(vpivot, pointer(ivars), length(ivars)) + ret = _findfirstequal(vpivot, pointer(ivars), Int64(length(ivars))) end return ret < 0 ? nothing : ret + 1 end @@ -67,7 +68,7 @@ function findfirstsortedequal( end # maybe occurs in vars[offset+1:offset+len] GC.@preserve vars begin - ret = _findfirstequal(var, pointer(vars) + 8offset, len) + ret = _findfirstequal(var, pointer(vars) + 8offset, Int64(len)) end return ret < 0 ? nothing : ret + offset + 1 end diff --git a/src/simd_ir.jl b/src/simd_ir.jl index 3880a9a..97bc9f3 100644 --- a/src/simd_ir.jl +++ b/src/simd_ir.jl @@ -82,93 +82,120 @@ function _simd_scan_ir(t, pred) """ end -const FFE_IR = _simd_scan_ir("i64", "eq") - -function _findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64) - return Base.llvmcall( - (FFE_IR, "entry"), - Int64, - Tuple{Int64, Ptr{Int64}, Int64}, - vpivot, - ptr, - len - ) +# Portable scalar implementation of the primitives' contract: the 0-based +# offset of the first element with `pred(v[i], x)`, or -1 if there is none. +# Backs the primitives on non-64-bit platforms, where the SIMD kernels +# (64-bit lengths and pointer lowering) are unavailable. +function _scalar_first(pred::P, x::T, ptr::Ptr{T}, len::Int64) where {P, T} + for i in 1:len + pred(unsafe_load(ptr, i), x) && return i - 1 + end + return Int64(-1) end -const _SIMD_GT_I64_IR = _simd_scan_ir("i64", "sgt") -const _SIMD_GE_I64_IR = _simd_scan_ir("i64", "sge") -const _SIMD_GT_F64_IR = _simd_scan_ir("double", "ogt") -const _SIMD_GE_F64_IR = _simd_scan_ir("double", "oge") - -# Reverse-direction predicates: used by `SIMDLinearScan` under -# `Base.Order.Reverse` ordering, where the array is decreasing and we want -# to find the first lane where `v[i] < x` (searchsortedlast) or `v[i] <= x` -# (searchsortedfirst). -const _SIMD_LT_I64_IR = _simd_scan_ir("i64", "slt") -const _SIMD_LE_I64_IR = _simd_scan_ir("i64", "sle") -const _SIMD_LT_F64_IR = _simd_scan_ir("double", "olt") -const _SIMD_LE_F64_IR = _simd_scan_ir("double", "ole") - -# Backing primitives for SIMDLinearScan. Each returns the 0-based offset of -# the first lane satisfying the predicate, or -1 if none. Caveat: NaN inputs -# always compare false under the ordered `o*` float predicates, so NaN in `v` -# or `x` produces "no match" rather than an exception — consistent with the -# undefined-input contract for sorted Float64 vectors containing NaN. -function _simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64) - return Base.llvmcall( - (_SIMD_GT_I64_IR, "entry"), - Int64, Tuple{Int64, Ptr{Int64}, Int64}, - x, ptr, len - ) -end -function _simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64) - return Base.llvmcall( - (_SIMD_GE_I64_IR, "entry"), - Int64, Tuple{Int64, Ptr{Int64}, Int64}, - x, ptr, len - ) -end -function _simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64) - return Base.llvmcall( - (_SIMD_GT_F64_IR, "entry"), - Int64, Tuple{Float64, Ptr{Float64}, Int64}, - x, ptr, len - ) -end -function _simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64) - return Base.llvmcall( - (_SIMD_GE_F64_IR, "entry"), - Int64, Tuple{Float64, Ptr{Float64}, Int64}, - x, ptr, len - ) -end +@static if Sys.WORD_SIZE == 64 + + const FFE_IR = _simd_scan_ir("i64", "eq") + + function _findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64) + return Base.llvmcall( + (FFE_IR, "entry"), + Int64, + Tuple{Int64, Ptr{Int64}, Int64}, + vpivot, + ptr, + len + ) + end + + const _SIMD_GT_I64_IR = _simd_scan_ir("i64", "sgt") + const _SIMD_GE_I64_IR = _simd_scan_ir("i64", "sge") + const _SIMD_GT_F64_IR = _simd_scan_ir("double", "ogt") + const _SIMD_GE_F64_IR = _simd_scan_ir("double", "oge") + + # Reverse-direction predicates: used by `SIMDLinearScan` under + # `Base.Order.Reverse` ordering, where the array is decreasing and we want + # to find the first lane where `v[i] < x` (searchsortedlast) or `v[i] <= x` + # (searchsortedfirst). + const _SIMD_LT_I64_IR = _simd_scan_ir("i64", "slt") + const _SIMD_LE_I64_IR = _simd_scan_ir("i64", "sle") + const _SIMD_LT_F64_IR = _simd_scan_ir("double", "olt") + const _SIMD_LE_F64_IR = _simd_scan_ir("double", "ole") + + # Backing primitives for SIMDLinearScan. Each returns the 0-based offset of + # the first lane satisfying the predicate, or -1 if none. Caveat: NaN inputs + # always compare false under the ordered `o*` float predicates, so NaN in `v` + # or `x` produces "no match" rather than an exception — consistent with the + # undefined-input contract for sorted Float64 vectors containing NaN. + function _simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64) + return Base.llvmcall( + (_SIMD_GT_I64_IR, "entry"), + Int64, Tuple{Int64, Ptr{Int64}, Int64}, + x, ptr, len + ) + end + function _simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64) + return Base.llvmcall( + (_SIMD_GE_I64_IR, "entry"), + Int64, Tuple{Int64, Ptr{Int64}, Int64}, + x, ptr, len + ) + end + function _simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64) + return Base.llvmcall( + (_SIMD_GT_F64_IR, "entry"), + Int64, Tuple{Float64, Ptr{Float64}, Int64}, + x, ptr, len + ) + end + function _simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64) + return Base.llvmcall( + (_SIMD_GE_F64_IR, "entry"), + Int64, Tuple{Float64, Ptr{Float64}, Int64}, + x, ptr, len + ) + end + + # Reverse-direction primitives. + function _simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64) + return Base.llvmcall( + (_SIMD_LT_I64_IR, "entry"), + Int64, Tuple{Int64, Ptr{Int64}, Int64}, + x, ptr, len + ) + end + function _simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64) + return Base.llvmcall( + (_SIMD_LE_I64_IR, "entry"), + Int64, Tuple{Int64, Ptr{Int64}, Int64}, + x, ptr, len + ) + end + function _simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64) + return Base.llvmcall( + (_SIMD_LT_F64_IR, "entry"), + Int64, Tuple{Float64, Ptr{Float64}, Int64}, + x, ptr, len + ) + end + function _simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64) + return Base.llvmcall( + (_SIMD_LE_F64_IR, "entry"), + Int64, Tuple{Float64, Ptr{Float64}, Int64}, + x, ptr, len + ) + end + +else + + _findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(==, vpivot, ptr, len) + _simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(>, x, ptr, len) + _simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(>=, x, ptr, len) + _simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(<, x, ptr, len) + _simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(<=, x, ptr, len) + _simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(>, x, ptr, len) + _simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(>=, x, ptr, len) + _simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(<, x, ptr, len) + _simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(<=, x, ptr, len) -# Reverse-direction primitives. -function _simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64) - return Base.llvmcall( - (_SIMD_LT_I64_IR, "entry"), - Int64, Tuple{Int64, Ptr{Int64}, Int64}, - x, ptr, len - ) -end -function _simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64) - return Base.llvmcall( - (_SIMD_LE_I64_IR, "entry"), - Int64, Tuple{Int64, Ptr{Int64}, Int64}, - x, ptr, len - ) -end -function _simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64) - return Base.llvmcall( - (_SIMD_LT_F64_IR, "entry"), - Int64, Tuple{Float64, Ptr{Float64}, Int64}, - x, ptr, len - ) -end -function _simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64) - return Base.llvmcall( - (_SIMD_LE_F64_IR, "entry"), - Int64, Tuple{Float64, Ptr{Float64}, Int64}, - x, ptr, len - ) end diff --git a/test/core_tests.jl b/test/core_tests.jl index 451aaaa..702a2b1 100644 --- a/test/core_tests.jl +++ b/test/core_tests.jl @@ -1353,3 +1353,27 @@ end end end end + +@safetestset "Scalar primitive parity" begin + using FindFirstFunctions, StableRNGs + F = FindFirstFunctions + + # `_scalar_first` backs the SIMD primitives on non-64-bit platforms; + # check it against `findfirst` on all platforms. + rng = StableRNG(2028) + for _ in 1:2_000 + n = rand(rng, 0:64) + vi = rand(rng, Int64(-50):Int64(50), n) + xi = rand(rng, Int64(-60):Int64(60)) + vf = randn(rng, n) + xf = randn(rng) + for (pred, v, x) in ( + (==, vi, xi), (>, vi, xi), (>=, vi, xi), (<, vi, xi), (<=, vi, xi), + (>, vf, xf), (>=, vf, xf), (<, vf, xf), (<=, vf, xf), + ) + ref = findfirst(e -> pred(e, x), v) + got = GC.@preserve v F._scalar_first(pred, x, pointer(v), Int64(length(v))) + @test got == (ref === nothing ? -1 : ref - 1) + end + end +end