Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/Tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,12 @@ jobs:
tests:
uses: "SciML/.github/.github/workflows/grouped-tests.yml@v1"
secrets: "inherit"
tests-32bit:
name: "Core (julia 1, ubuntu-latest, x86)"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
with:
julia-version: "1"
julia-arch: "x86"
group: "Core"
coverage: false
secrets: "inherit"
11 changes: 6 additions & 5 deletions src/equality.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,16 @@ This function does **not** assume `A` is sorted. For sorted vectors, see
`DenseVector{Int64}`) or [`findequal`](@ref) (the strategy-framework
equality wrapper that returns an `Int` with a sentinel).

The `(x::Int64, A::DenseVector{Int64})` method uses a custom LLVM IR SIMD
scan (load 8 lanes, `icmp eq`, `cttz` on the mask) — about 8× faster than
the scalar `findfirst(==(x), v)` on modern x86-64. Every other element-type
On 64-bit platforms the `(x::Int64, A::DenseVector{Int64})` method uses a
custom LLVM IR SIMD scan (load 8 lanes, `icmp eq`, `cttz` on the mask) —
about 8× faster than the scalar `findfirst(==(x), v)` on modern x86-64;
on 32-bit platforms it uses a scalar loop. Every other element-type
and array-storage combination falls back to `findfirst(isequal(x), A)`.
"""
findfirstequal(vpivot, ivars) = findfirst(isequal(vpivot), ivars)
function findfirstequal(vpivot::Int64, ivars::DenseVector{Int64})
GC.@preserve ivars begin
ret = _findfirstequal(vpivot, pointer(ivars), length(ivars))
ret = _findfirstequal(vpivot, pointer(ivars), Int64(length(ivars)))
end
return ret < 0 ? nothing : ret + 1
end
Expand Down Expand Up @@ -67,7 +68,7 @@ function findfirstsortedequal(
end
# maybe occurs in vars[offset+1:offset+len]
GC.@preserve vars begin
ret = _findfirstequal(var, pointer(vars) + 8offset, len)
ret = _findfirstequal(var, pointer(vars) + 8offset, Int64(len))
end
return ret < 0 ? nothing : ret + offset + 1
end
199 changes: 113 additions & 86 deletions src/simd_ir.jl
Original file line number Diff line number Diff line change
Expand Up @@ -82,93 +82,120 @@ function _simd_scan_ir(t, pred)
"""
end

const FFE_IR = _simd_scan_ir("i64", "eq")

function _findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(FFE_IR, "entry"),
Int64,
Tuple{Int64, Ptr{Int64}, Int64},
vpivot,
ptr,
len
)
# Portable scalar implementation of the primitives' contract: the 0-based
# offset of the first element with `pred(v[i], x)`, or -1 if there is none.
# Backs the primitives on non-64-bit platforms, where the SIMD kernels
# (64-bit lengths and pointer lowering) are unavailable.
function _scalar_first(pred::P, x::T, ptr::Ptr{T}, len::Int64) where {P, T}
for i in 1:len
pred(unsafe_load(ptr, i), x) && return i - 1
end
return Int64(-1)
end

const _SIMD_GT_I64_IR = _simd_scan_ir("i64", "sgt")
const _SIMD_GE_I64_IR = _simd_scan_ir("i64", "sge")
const _SIMD_GT_F64_IR = _simd_scan_ir("double", "ogt")
const _SIMD_GE_F64_IR = _simd_scan_ir("double", "oge")

# Reverse-direction predicates: used by `SIMDLinearScan` under
# `Base.Order.Reverse` ordering, where the array is decreasing and we want
# to find the first lane where `v[i] < x` (searchsortedlast) or `v[i] <= x`
# (searchsortedfirst).
const _SIMD_LT_I64_IR = _simd_scan_ir("i64", "slt")
const _SIMD_LE_I64_IR = _simd_scan_ir("i64", "sle")
const _SIMD_LT_F64_IR = _simd_scan_ir("double", "olt")
const _SIMD_LE_F64_IR = _simd_scan_ir("double", "ole")

# Backing primitives for SIMDLinearScan. Each returns the 0-based offset of
# the first lane satisfying the predicate, or -1 if none. Caveat: NaN inputs
# always compare false under the ordered `o*` float predicates, so NaN in `v`
# or `x` produces "no match" rather than an exception — consistent with the
# undefined-input contract for sorted Float64 vectors containing NaN.
function _simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_GT_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_GE_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_GT_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
function _simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_GE_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
@static if Sys.WORD_SIZE == 64

const FFE_IR = _simd_scan_ir("i64", "eq")

function _findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(FFE_IR, "entry"),
Int64,
Tuple{Int64, Ptr{Int64}, Int64},
vpivot,
ptr,
len
)
end

const _SIMD_GT_I64_IR = _simd_scan_ir("i64", "sgt")
const _SIMD_GE_I64_IR = _simd_scan_ir("i64", "sge")
const _SIMD_GT_F64_IR = _simd_scan_ir("double", "ogt")
const _SIMD_GE_F64_IR = _simd_scan_ir("double", "oge")

# Reverse-direction predicates: used by `SIMDLinearScan` under
# `Base.Order.Reverse` ordering, where the array is decreasing and we want
# to find the first lane where `v[i] < x` (searchsortedlast) or `v[i] <= x`
# (searchsortedfirst).
const _SIMD_LT_I64_IR = _simd_scan_ir("i64", "slt")
const _SIMD_LE_I64_IR = _simd_scan_ir("i64", "sle")
const _SIMD_LT_F64_IR = _simd_scan_ir("double", "olt")
const _SIMD_LE_F64_IR = _simd_scan_ir("double", "ole")

# Backing primitives for SIMDLinearScan. Each returns the 0-based offset of
# the first lane satisfying the predicate, or -1 if none. Caveat: NaN inputs
# always compare false under the ordered `o*` float predicates, so NaN in `v`
# or `x` produces "no match" rather than an exception — consistent with the
# undefined-input contract for sorted Float64 vectors containing NaN.
function _simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_GT_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_GE_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_GT_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
function _simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_GE_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end

# Reverse-direction primitives.
function _simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_LT_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_LE_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_LT_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
function _simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_LE_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end

else

_findfirstequal(vpivot::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(==, vpivot, ptr, len)
_simd_first_gt(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(>, x, ptr, len)
_simd_first_ge(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(>=, x, ptr, len)
_simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(<, x, ptr, len)
_simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64) = _scalar_first(<=, x, ptr, len)
_simd_first_gt(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(>, x, ptr, len)
_simd_first_ge(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(>=, x, ptr, len)
_simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(<, x, ptr, len)
_simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64) = _scalar_first(<=, x, ptr, len)

# Reverse-direction primitives.
function _simd_first_lt(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_LT_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_le(x::Int64, ptr::Ptr{Int64}, len::Int64)
return Base.llvmcall(
(_SIMD_LE_I64_IR, "entry"),
Int64, Tuple{Int64, Ptr{Int64}, Int64},
x, ptr, len
)
end
function _simd_first_lt(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_LT_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
function _simd_first_le(x::Float64, ptr::Ptr{Float64}, len::Int64)
return Base.llvmcall(
(_SIMD_LE_F64_IR, "entry"),
Int64, Tuple{Float64, Ptr{Float64}, Int64},
x, ptr, len
)
end
24 changes: 24 additions & 0 deletions test/core_tests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1353,3 +1353,27 @@ end
end
end
end

@safetestset "Scalar primitive parity" begin
using FindFirstFunctions, StableRNGs
F = FindFirstFunctions

# `_scalar_first` backs the SIMD primitives on non-64-bit platforms;
# check it against `findfirst` on all platforms.
rng = StableRNG(2028)
for _ in 1:2_000
n = rand(rng, 0:64)
vi = rand(rng, Int64(-50):Int64(50), n)
xi = rand(rng, Int64(-60):Int64(60))
vf = randn(rng, n)
xf = randn(rng)
for (pred, v, x) in (
(==, vi, xi), (>, vi, xi), (>=, vi, xi), (<, vi, xi), (<=, vi, xi),
(>, vf, xf), (>=, vf, xf), (<, vf, xf), (<=, vf, xf),
)
ref = findfirst(e -> pred(e, x), v)
got = GC.@preserve v F._scalar_first(pred, x, pointer(v), Int64(length(v)))
@test got == (ref === nothing ? -1 : ref - 1)
end
end
end
Loading