diff --git a/tmva/sofie/inc/TMVA/ROperator_Selu.hxx b/tmva/sofie/inc/TMVA/ROperator_Selu.hxx index 676c8871b226b..d2458733e68d8 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Selu.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Selu.hxx @@ -5,6 +5,8 @@ #include "TMVA/ROperator.hxx" #include "TMVA/RModel.hxx" +#include +#include #include namespace TMVA{ @@ -16,18 +18,20 @@ class ROperator_Selu final : public ROperator { private: - + float fAlpha = 1.67326319217681884765625; + float fGamma = 1.05070102214813232421875; std::string fNX; std::string fNY; std::vector fShape; public: ROperator_Selu(){} - ROperator_Selu(std::string nameX, std::string nameY): - fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY)){ - fInputTensorNames = { fNX }; - fOutputTensorNames = { fNY }; - } + ROperator_Selu(float alpha, float gamma, std::string nameX, std::string nameY) + : fAlpha(alpha), fGamma(gamma), fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY)) + { + fInputTensorNames = {fNX}; + fOutputTensorNames = {fNY}; + } void Initialize(RModel& model) override { if (model.CheckIfTensorAlreadyExist(fNX) == false){ //input must be a graph input, or already initialized intermediate tensor @@ -48,8 +52,12 @@ public: for(auto& i: fShape){ length *= i; } + out << std::setprecision(std::numeric_limits::max_digits10); + out << "\t" << "float " << OpName << "_alpha = " << fAlpha << ";\n"; + out << "\t" << "float " << OpName << "_gamma = " << fGamma << ";\n"; out << "\t" << "for (int id = 0; id < " << length << " ; id++){\n"; - out << "\t\t" << "tensor_" << fNY << "[id] = 1.0507009873554804934193349852946 * (std::max(float(0.0), tensor_" << fNX << "[id]) + std::min(0.0, 1.6732632423543772848170429916717 * (std::exp(" << "tensor_" << fNX << "[id]" <<")-1)));\n"; + out << "\t\t" << "tensor_" << fNY << "[id] = " << OpName << "_gamma * (std::max(float(0.0), tensor_" << fNX + << "[id]) + std::min(float(0.0), " << OpName << "_alpha * (std::exp(tensor_" << fNX << "[id]) - 1)));\n"; out << "\t}\n"; return out.str(); } diff --git a/tmva/sofie/test/TestCustomModelsFromONNX.cxx b/tmva/sofie/test/TestCustomModelsFromONNX.cxx index 4ea6d41c95081..b65c98ae29cc4 100644 --- a/tmva/sofie/test/TestCustomModelsFromONNX.cxx +++ b/tmva/sofie/test/TestCustomModelsFromONNX.cxx @@ -166,6 +166,15 @@ TEST(ONNX, Tanh) expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE); } +TEST(ONNX, SeluAttributes) +{ + SofieReference ref = readReference("SeluAttributes"); + + ASSERT_INCLUDE_AND_RUN(std::vector, "SeluAttributes", ref.f32("input0")); + + expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE); +} + TEST(ONNX, Erf) { SofieReference ref = readReference("Erf"); diff --git a/tmva/sofie/test/generate_input_models.py b/tmva/sofie/test/generate_input_models.py index 403dd52a9c6cb..0d649c4cd0c1f 100644 --- a/tmva/sofie/test/generate_input_models.py +++ b/tmva/sofie/test/generate_input_models.py @@ -2570,6 +2570,18 @@ def make_LinearWithSelu(): return _model(graph, opset=9, ir_version=6, producer_name='pytorch', producer_version='1.9') +def make_SeluAttributes(): + """Ops: Selu with non-default alpha and gamma.""" + nodes = [helper.make_node("Selu", ["x"], ["y"], name="selu_0", alpha=2.0, gamma=3.0)] + graph = helper.make_graph( + nodes, + "selu_attributes", + inputs=[_vi("x", FLOAT, [1, 4])], + outputs=[_vi("y", FLOAT, [1, 4])], + ) + return _model(graph, opset=13, ir_version=10, producer_name="onnx-example") + + def make_LinearWithSigmoid(): """Ops: Gemm, Sigmoid, Gemm, Sigmoid""" nodes = [ @@ -6323,6 +6335,7 @@ def make_AliasDynShapeAcrossNewTensor(): "ScatterND_1": make_ScatterND_1, "ScatterND_2": make_ScatterND_2, "ScatterND_3": make_ScatterND_3, + "SeluAttributes": make_SeluAttributes, "Shape": make_Shape, "Sin": make_Sin, "Slice": make_Slice, @@ -7568,6 +7581,7 @@ def rand_f32(seed, shape): "ReduceProd": [f32([5.0, 2.0, 3.0, 5.0, 5.0, 4.0], (1, 2, 3))], "ReshapeAlias": [rand_f32(42, (2, 3))], "ReshapeAliasGraphOutput": [rand_f32(43, (2, 3))], + "SeluAttributes": [f32([-2.0, -0.5, 0.5, 2.0], (1, 4))], "Shape": [f32([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], (1, 2, 3))], "Slice": [rand_f32(1, (20, 10, 5))], "SliceIdentityAlias": [rand_f32(44, (2, 3))], diff --git a/tmva/sofie_parsers/src/ParseSelu.cxx b/tmva/sofie_parsers/src/ParseSelu.cxx index 87993252035f6..8faa42dce5b03 100644 --- a/tmva/sofie_parsers/src/ParseSelu.cxx +++ b/tmva/sofie_parsers/src/ParseSelu.cxx @@ -19,9 +19,21 @@ ParserFuncSignature ParseSelu = [](RModelParser_ONNX &parser, const onnx::NodePr std::unique_ptr op; + // default values from the ONNX specification + float attr_alpha = 1.67326319217681884765625; + float attr_gamma = 1.05070102214813232421875; + + for (int_t i = 0; i < nodeproto.attribute_size(); i++) { + std::string attribute_name = nodeproto.attribute(i).name(); + if (attribute_name == "alpha") + attr_alpha = nodeproto.attribute(i).f(); + else if (attribute_name == "gamma") + attr_gamma = nodeproto.attribute(i).f(); + } + std::string output_name = nodeproto.output(0); switch (input_type) { - case ETensorType::FLOAT: op.reset(new ROperator_Selu(input_name, output_name)); break; + case ETensorType::FLOAT: op.reset(new ROperator_Selu(attr_alpha, attr_gamma, input_name, output_name)); break; default: throw std::runtime_error("TMVA::SOFIE - Unsupported - Operator Selu does not yet support input type " + std::to_string(static_cast(input_type)));