From 8d94763a64d69b2a0778ebd46b8f65d0ba785a05 Mon Sep 17 00:00:00 2001 From: Daniel Karrasch Date: Wed, 28 Feb 2024 19:45:24 +0100 Subject: [PATCH 1/2] Avoid constructing `MulAddMul`s on Julia v1.12+ --- lib/cublas/linalg.jl | 47 +++++++++++++++++++++++++++-- lib/cusparse/interfaces.jl | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/lib/cublas/linalg.jl b/lib/cublas/linalg.jl index 763b32aafa..68da7e0445 100644 --- a/lib/cublas/linalg.jl +++ b/lib/cublas/linalg.jl @@ -173,7 +173,7 @@ end # # GEMV - +if VERSION < v"1.12.0-" function LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, _add::MulAddMul) mA, nA = tA == 'N' ? size(A) : reverse(size(A)) @@ -208,6 +208,44 @@ function LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::Stri end LinearAlgebra.generic_matmatmul!(Y, tA, 'N', A, B, MulAddMul(alpha, beta)) end +else # VERSION >= v"1.12.0-" +# legacy method +LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, _add::MulAddMul) = + LinearAlgebra.generic_matvecmul!(Y, tA, A, B, _add.alpha, _add.beta) +function LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, alpha::Number, beta::Number) + mA, nA = tA == 'N' ? size(A) : reverse(size(A)) + + if nA != length(B) + throw(DimensionMismatch("second dimension of A, $nA, does not match length of B, $(length(B))")) + end + + if mA != length(Y) + throw(DimensionMismatch("first dimension of A, $mA, does not match length of Y, $(length(Y))")) + end + + if mA == 0 + return Y + end + + if nA == 0 + return rmul!(Y, 0) + end + + T = eltype(Y) + if alpha isa Union{Bool,T} && beta isa Union{Bool,T} + if T <: CublasFloat && eltype(A) == eltype(B) == T + if tA in ('N', 'T', 'C') + return gemv!(tA, alpha, A, B, beta, Y) + elseif tA in ('S', 's') + return symv!(tA == 'S' ? 'U' : 'L', alpha, A, B, beta, Y) + elseif tA in ('H', 'h') + return hemv!(tA == 'H' ? 'U' : 'L', alpha, A, B, beta, Y) + end + end + end + LinearAlgebra.generic_matmatmul!(Y, tA, 'N', A, B, alpha, beta) +end +end # VERSION if VERSION < v"1.10.0-DEV.1365" @inline LinearAlgebra.gemv!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, a::Number, b::Number) = @@ -282,10 +320,13 @@ end # VERSION # # GEMM +if VERSION < v"1.12.0-" +LinearAlgebra.generic_matmatmul!(C::StridedCuVecOrMat, tA, tB, A::StridedCuVecOrMat, B::StridedCuVecOrMat, _add::MulAddMul) = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) +end -function LinearAlgebra.generic_matmatmul!(C::StridedCuVecOrMat, tA, tB, A::StridedCuVecOrMat, B::StridedCuVecOrMat, _add::MulAddMul) +function LinearAlgebra.generic_matmatmul!(C::StridedCuVecOrMat, tA, tB, A::StridedCuVecOrMat, B::StridedCuVecOrMat, alpha::Number, beta::Number) T = eltype(C) - alpha, beta = _add.alpha, _add.beta mA, nA = size(A, tA == 'N' ? 1 : 2), size(A, tA == 'N' ? 2 : 1) mB, nB = size(B, tB == 'N' ? 1 : 2), size(B, tB == 'N' ? 2 : 1) diff --git a/lib/cusparse/interfaces.jl b/lib/cusparse/interfaces.jl index 686a4a9c22..dacde00f04 100644 --- a/lib/cusparse/interfaces.jl +++ b/lib/cusparse/interfaces.jl @@ -61,6 +61,7 @@ op_wrappers = ((identity, T -> 'N', identity), (T -> :(Adjoint{T, <:$T}), T -> T <: Real ? 'T' : 'C', A -> :(parent($A))), (T -> :(HermOrSym{T, <:$T}), T -> 'N', A -> :(parent($A)))) +if VERSION < v"1.12.0-" function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::DenseCuVector{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA mv_wrapper(tA, _add.alpha, A, B, _add.beta, C) @@ -75,6 +76,22 @@ function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::CuSparseMat tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB mm_wrapper(tA, tB, _add.alpha, A, B, _add.beta, C) end +else +function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::DenseCuVector{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + mv_wrapper(tA, alpha, A, B, beta, C) +end +function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::CuSparseVector{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + mv_wrapper(tA, alpha, A, CuVector{T}(B), beta, C) +end + +function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::CuSparseMatrix{T}, B::DenseCuMatrix{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + mm_wrapper(tA, tB, alpha, A, B, beta, C) +end +end for (wrapa, transa, unwrapa) in op_wrappers TypeA = wrapa(:(CuSparseMatrix{T})) @@ -87,6 +104,7 @@ for (wrapa, transa, unwrapa) in op_wrappers end end +if VERSION < v"1.12.0-" function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::DenseCuMatrix{T}, B::CuSparseVector{T}, _add::MulAddMul) where {T <: BlasFloat} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA gemvi!(tA, _add.alpha, A, B, _add.beta, C, 'O') @@ -107,6 +125,28 @@ function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatr tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB mm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') end +else +function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::DenseCuMatrix{T}, B::CuSparseVector{T}, alpha::Number, beta::Number) where {T <: BlasFloat} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + gemvi!(tA, alpha, A, B, beta, C, 'O') +end + +function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSC{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + mm!(tA, tB, alpha, A, B, beta, C, 'O') +end +function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSR{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + mm!(tA, tB, alpha, A, B, beta, C, 'O') +end +function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCOO{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + mm!(tA, tB, alpha, A, B, beta, C, 'O') +end +end for (wrapa, transa, unwrapa) in op_wrappers TypeA = wrapa(:(DenseCuMatrix{T})) @@ -143,6 +183,7 @@ for (wrapa, transa, unwrapa) in op_wrappers end end +if VERSION < v"1.12.0-" function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSC{T}, tA, tB, A::CuSparseMatrixCSC{T}, B::CuSparseMatrixCSC{T}, _add::MulAddMul) where {T <: BlasFloat} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB @@ -162,6 +203,27 @@ function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCOO{T}, tA, tB, A::Cu generic_matmatmul!(C_csr, tA, tB, A_csr, B_csr, _add.alpha, _add.beta) C = CuSparseMatrixCOO(C_csr) # is this in-place of the original C? end +else +function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSC{T}, tA, tB, A::CuSparseMatrixCSC{T}, B::CuSparseMatrixCSC{T}, alpha::Number, beta::Number) where {T <: BlasFloat} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + gemm!(tA, tB, alpha, A, B, beta, C, 'O') +end +function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSR{T}, tA, tB, A::CuSparseMatrixCSR{T}, B::CuSparseMatrixCSR{T}, alpha::Number, beta::Number) where {T <: BlasFloat} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + gemm!(tA, tB, alpha, A, B, beta, C, 'O') +end +function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCOO{T}, tA, tB, A::CuSparseMatrixCOO{T}, B::CuSparseMatrixCOO{T}, alpha::Number, beta::Number) where {T <: BlasFloat} + tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA + tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB + A_csr = CuSparseMatrixCSR(A) + B_csr = CuSparseMatrixCSR(B) + C_csr = CuSparseMatrixCSR(C) + generic_matmatmul!(C_csr, tA, tB, A_csr, B_csr, alpha, beta) + C = CuSparseMatrixCOO(C_csr) # is this in-place of the original C? +end +end for SparseMatrixType in (:CuSparseMatrixCSC, :CuSparseMatrixCSR) @eval function LinearAlgebra.:(*)(A::$SparseMatrixType{T}, B::$SparseMatrixType{T}) where {T <: BlasFloat} From d4529df08153aea4d84dddc9d4ad741d1ff56da0 Mon Sep 17 00:00:00 2001 From: Daniel Karrasch Date: Sat, 2 Mar 2024 11:01:45 +0100 Subject: [PATCH 2/2] make changes non-breaking --- lib/cublas/linalg.jl | 40 ------------------ lib/cusparse/interfaces.jl | 85 +++++++++++--------------------------- 2 files changed, 25 insertions(+), 100 deletions(-) diff --git a/lib/cublas/linalg.jl b/lib/cublas/linalg.jl index 68da7e0445..df368178b0 100644 --- a/lib/cublas/linalg.jl +++ b/lib/cublas/linalg.jl @@ -173,42 +173,6 @@ end # # GEMV -if VERSION < v"1.12.0-" -function LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, _add::MulAddMul) - mA, nA = tA == 'N' ? size(A) : reverse(size(A)) - - if nA != length(B) - throw(DimensionMismatch("second dimension of A, $nA, does not match length of B, $(length(B))")) - end - - if mA != length(Y) - throw(DimensionMismatch("first dimension of A, $mA, does not match length of Y, $(length(Y))")) - end - - if mA == 0 - return Y - end - - if nA == 0 - return rmul!(Y, 0) - end - - T = eltype(Y) - alpha, beta = _add.alpha, _add.beta - if alpha isa Union{Bool,T} && beta isa Union{Bool,T} - if T <: CublasFloat && eltype(A) == eltype(B) == T - if tA in ('N', 'T', 'C') - return gemv!(tA, alpha, A, B, beta, Y) - elseif tA in ('S', 's') - return symv!(tA == 'S' ? 'U' : 'L', alpha, A, B, beta, Y) - elseif tA in ('H', 'h') - return hemv!(tA == 'H' ? 'U' : 'L', alpha, A, B, beta, Y) - end - end - end - LinearAlgebra.generic_matmatmul!(Y, tA, 'N', A, B, MulAddMul(alpha, beta)) -end -else # VERSION >= v"1.12.0-" # legacy method LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, _add::MulAddMul) = LinearAlgebra.generic_matvecmul!(Y, tA, A, B, _add.alpha, _add.beta) @@ -245,7 +209,6 @@ function LinearAlgebra.generic_matvecmul!(Y::CuVector, tA::AbstractChar, A::Stri end LinearAlgebra.generic_matmatmul!(Y, tA, 'N', A, B, alpha, beta) end -end # VERSION if VERSION < v"1.10.0-DEV.1365" @inline LinearAlgebra.gemv!(Y::CuVector, tA::AbstractChar, A::StridedCuMatrix, B::StridedCuVector, a::Number, b::Number) = @@ -320,11 +283,8 @@ end # VERSION # # GEMM -if VERSION < v"1.12.0-" LinearAlgebra.generic_matmatmul!(C::StridedCuVecOrMat, tA, tB, A::StridedCuVecOrMat, B::StridedCuVecOrMat, _add::MulAddMul) = LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) -end - function LinearAlgebra.generic_matmatmul!(C::StridedCuVecOrMat, tA, tB, A::StridedCuVecOrMat, B::StridedCuVecOrMat, alpha::Number, beta::Number) T = eltype(C) mA, nA = size(A, tA == 'N' ? 1 : 2), size(A, tA == 'N' ? 2 : 1) diff --git a/lib/cusparse/interfaces.jl b/lib/cusparse/interfaces.jl index dacde00f04..e6bbe22a1b 100644 --- a/lib/cusparse/interfaces.jl +++ b/lib/cusparse/interfaces.jl @@ -61,22 +61,14 @@ op_wrappers = ((identity, T -> 'N', identity), (T -> :(Adjoint{T, <:$T}), T -> T <: Real ? 'T' : 'C', A -> :(parent($A))), (T -> :(HermOrSym{T, <:$T}), T -> 'N', A -> :(parent($A)))) -if VERSION < v"1.12.0-" -function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::DenseCuVector{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - mv_wrapper(tA, _add.alpha, A, B, _add.beta, C) -end -function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::CuSparseVector{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - mv_wrapper(tA, _add.alpha, A, CuVector{T}(B), _add.beta, C) -end +# legacy methods with final MulAddMul argument +LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::DenseCuVector{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matvecmul!(C, tA, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::CuSparseVector{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matvecmul!(C, tA, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::CuSparseMatrix{T}, B::DenseCuMatrix{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) -function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::CuSparseMatrix{T}, B::DenseCuMatrix{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - mm_wrapper(tA, tB, _add.alpha, A, B, _add.beta, C) -end -else function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::CuSparseMatrix{T}, B::DenseCuVector{T}, alpha::Number, beta::Number) where {T <: Union{Float16, ComplexF16, BlasFloat}} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA mv_wrapper(tA, alpha, A, B, beta, C) @@ -91,7 +83,6 @@ function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::CuSparseMat tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB mm_wrapper(tA, tB, alpha, A, B, beta, C) end -end for (wrapa, transa, unwrapa) in op_wrappers TypeA = wrapa(:(CuSparseMatrix{T})) @@ -104,28 +95,17 @@ for (wrapa, transa, unwrapa) in op_wrappers end end -if VERSION < v"1.12.0-" -function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::DenseCuMatrix{T}, B::CuSparseVector{T}, _add::MulAddMul) where {T <: BlasFloat} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - gemvi!(tA, _add.alpha, A, B, _add.beta, C, 'O') -end +# legacy methods with final MulAddMul argument +LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::DenseCuMatrix{T}, B::CuSparseVector{T}, _add::MulAddMul) where {T <: BlasFloat} = + LinearAlgebra.generic_matvecmul!(C, tA, A, B, _add.alpha, _add.beta) + +LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSC{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSR{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCOO{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) -function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSC{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - mm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') -end -function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCSR{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - mm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') -end -function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatrix{T}, B::CuSparseMatrixCOO{T}, _add::MulAddMul) where {T <: Union{Float16, ComplexF16, BlasFloat}} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - mm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') -end -else function LinearAlgebra.generic_matvecmul!(C::CuVector{T}, tA::AbstractChar, A::DenseCuMatrix{T}, B::CuSparseVector{T}, alpha::Number, beta::Number) where {T <: BlasFloat} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA gemvi!(tA, alpha, A, B, beta, C, 'O') @@ -146,7 +126,6 @@ function LinearAlgebra.generic_matmatmul!(C::CuMatrix{T}, tA, tB, A::DenseCuMatr tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB mm!(tA, tB, alpha, A, B, beta, C, 'O') end -end for (wrapa, transa, unwrapa) in op_wrappers TypeA = wrapa(:(DenseCuMatrix{T})) @@ -183,27 +162,14 @@ for (wrapa, transa, unwrapa) in op_wrappers end end -if VERSION < v"1.12.0-" -function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSC{T}, tA, tB, A::CuSparseMatrixCSC{T}, B::CuSparseMatrixCSC{T}, _add::MulAddMul) where {T <: BlasFloat} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - gemm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') -end -function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSR{T}, tA, tB, A::CuSparseMatrixCSR{T}, B::CuSparseMatrixCSR{T}, _add::MulAddMul) where {T <: BlasFloat} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - gemm!(tA, tB, _add.alpha, A, B, _add.beta, C, 'O') -end -function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCOO{T}, tA, tB, A::CuSparseMatrixCOO{T}, B::CuSparseMatrixCOO{T}, _add::MulAddMul) where {T <: BlasFloat} - tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA - tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB - A_csr = CuSparseMatrixCSR(A) - B_csr = CuSparseMatrixCSR(B) - C_csr = CuSparseMatrixCSR(C) - generic_matmatmul!(C_csr, tA, tB, A_csr, B_csr, _add.alpha, _add.beta) - C = CuSparseMatrixCOO(C_csr) # is this in-place of the original C? -end -else +# legacy methods with final MulAddMul argument +LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSC{T}, tA, tB, A::CuSparseMatrixCSC{T}, B::CuSparseMatrixCSC{T}, _add::MulAddMul) where {T <: BlasFloat} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSR{T}, tA, tB, A::CuSparseMatrixCSR{T}, B::CuSparseMatrixCSR{T}, _add::MulAddMul) where {T <: BlasFloat} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) +LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCOO{T}, tA, tB, A::CuSparseMatrixCOO{T}, B::CuSparseMatrixCOO{T}, _add::MulAddMul) where {T <: BlasFloat} = + LinearAlgebra.generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta) + function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCSC{T}, tA, tB, A::CuSparseMatrixCSC{T}, B::CuSparseMatrixCSC{T}, alpha::Number, beta::Number) where {T <: BlasFloat} tA = tA in ('S', 's', 'H', 'h') ? 'N' : tA tB = tB in ('S', 's', 'H', 'h') ? 'N' : tB @@ -223,7 +189,6 @@ function LinearAlgebra.generic_matmatmul!(C::CuSparseMatrixCOO{T}, tA, tB, A::Cu generic_matmatmul!(C_csr, tA, tB, A_csr, B_csr, alpha, beta) C = CuSparseMatrixCOO(C_csr) # is this in-place of the original C? end -end for SparseMatrixType in (:CuSparseMatrixCSC, :CuSparseMatrixCSR) @eval function LinearAlgebra.:(*)(A::$SparseMatrixType{T}, B::$SparseMatrixType{T}) where {T <: BlasFloat}