[tmva][sofie] Read the alpha and gamma attributes of Selu

tmva/sofie/inc/TMVA/ROperator_Selu.hxx

@@ -5,6 +5,8 @@
 #include "TMVA/ROperator.hxx"
 #include "TMVA/RModel.hxx"
 
+#include <iomanip>
+#include <limits>
 #include <sstream>
 
 namespace TMVA{
@@ -16,18 +18,20 @@ class ROperator_Selu final : public ROperator
 {
 
 private:
-
+   float fAlpha = 1.67326319217681884765625;
+   float fGamma = 1.05070102214813232421875;
    std::string fNX;
    std::string fNY;
    std::vector<size_t> fShape;
 
 public:
    ROperator_Selu(){}
-   ROperator_Selu(std::string nameX, std::string nameY):
-      fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY)){
-         fInputTensorNames = { fNX };
-         fOutputTensorNames = { fNY };
-      }
+   ROperator_Selu(float alpha, float gamma, std::string nameX, std::string nameY)
+      : fAlpha(alpha), fGamma(gamma), fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY))
+   {
+      fInputTensorNames = {fNX};
+      fOutputTensorNames = {fNY};
+   }
 
    void Initialize(RModel& model) override {
       if (model.CheckIfTensorAlreadyExist(fNX) == false){   //input must be a graph input, or already initialized intermediate tensor
@@ -48,8 +52,12 @@ public:
       for(auto& i: fShape){
          length *= i;
       }
+      out << std::setprecision(std::numeric_limits<float>::max_digits10);
+      out << "\t" << "float " << OpName << "_alpha = " << fAlpha << ";\n";
+      out << "\t" << "float " << OpName << "_gamma = " << fGamma << ";\n";
       out << "\t" << "for (int id = 0; id < " << length << " ; id++){\n";
-      out << "\t\t" << "tensor_" << fNY << "[id] = 1.0507009873554804934193349852946 * (std::max(float(0.0), tensor_"  << fNX << "[id]) + std::min(0.0, 1.6732632423543772848170429916717 * (std::exp(" << "tensor_" << fNX << "[id]" <<")-1)));\n";
+      out << "\t\t" << "tensor_" << fNY << "[id] = " << OpName << "_gamma * (std::max(float(0.0), tensor_" << fNX
+          << "[id]) + std::min(float(0.0), " << OpName << "_alpha * (std::exp(tensor_" << fNX << "[id]) - 1)));\n";
       out << "\t}\n";
       return out.str();
    }

tmva/sofie/test/TestCustomModelsFromONNX.cxx

@@ -166,6 +166,15 @@ TEST(ONNX, Tanh)
    expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE);
 }
 
+TEST(ONNX, SeluAttributes)
+{
+   SofieReference ref = readReference("SeluAttributes");
+
+   ASSERT_INCLUDE_AND_RUN(std::vector<float>, "SeluAttributes", ref.f32("input0"));
+
+   expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE);
+}
+
 TEST(ONNX, Erf)
 {
    SofieReference ref = readReference("Erf");

tmva/sofie/test/generate_input_models.py

@@ -2570,6 +2570,18 @@ def make_LinearWithSelu():
     return _model(graph, opset=9, ir_version=6, producer_name='pytorch', producer_version='1.9')
 
 
+def make_SeluAttributes():
+    """Ops: Selu with non-default alpha and gamma."""
+    nodes = [helper.make_node("Selu", ["x"], ["y"], name="selu_0", alpha=2.0, gamma=3.0)]
+    graph = helper.make_graph(
+        nodes,
+        "selu_attributes",
+        inputs=[_vi("x", FLOAT, [1, 4])],
+        outputs=[_vi("y", FLOAT, [1, 4])],
+    )
+    return _model(graph, opset=13, ir_version=10, producer_name="onnx-example")
+
+
 def make_LinearWithSigmoid():
     """Ops: Gemm, Sigmoid, Gemm, Sigmoid"""
     nodes = [
@@ -6323,6 +6335,7 @@ def make_AliasDynShapeAcrossNewTensor():
     "ScatterND_1": make_ScatterND_1,
     "ScatterND_2": make_ScatterND_2,
     "ScatterND_3": make_ScatterND_3,
+    "SeluAttributes": make_SeluAttributes,
     "Shape": make_Shape,
     "Sin": make_Sin,
     "Slice": make_Slice,
@@ -7568,6 +7581,7 @@ def rand_f32(seed, shape):
     "ReduceProd": [f32([5.0, 2.0, 3.0, 5.0, 5.0, 4.0], (1, 2, 3))],
     "ReshapeAlias": [rand_f32(42, (2, 3))],
     "ReshapeAliasGraphOutput": [rand_f32(43, (2, 3))],
+    "SeluAttributes": [f32([-2.0, -0.5, 0.5, 2.0], (1, 4))],
     "Shape": [f32([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], (1, 2, 3))],
     "Slice": [rand_f32(1, (20, 10, 5))],
     "SliceIdentityAlias": [rand_f32(44, (2, 3))],

tmva/sofie_parsers/src/ParseSelu.cxx

@@ -19,9 +19,21 @@ ParserFuncSignature ParseSelu = [](RModelParser_ONNX &parser, const onnx::NodePr
 
    std::unique_ptr<ROperator> op;
 
+   // default values from the ONNX specification
+   float attr_alpha = 1.67326319217681884765625;
+   float attr_gamma = 1.05070102214813232421875;
+
+   for (int_t i = 0; i < nodeproto.attribute_size(); i++) {
+      std::string attribute_name = nodeproto.attribute(i).name();
+      if (attribute_name == "alpha")
+         attr_alpha = nodeproto.attribute(i).f();
+      else if (attribute_name == "gamma")
+         attr_gamma = nodeproto.attribute(i).f();
+   }
+
    std::string output_name = nodeproto.output(0);
    switch (input_type) {
-   case ETensorType::FLOAT: op.reset(new ROperator_Selu<float>(input_name, output_name)); break;
+   case ETensorType::FLOAT: op.reset(new ROperator_Selu<float>(attr_alpha, attr_gamma, input_name, output_name)); break;
    default:
       throw std::runtime_error("TMVA::SOFIE - Unsupported - Operator Selu does not yet support input type " +
                                std::to_string(static_cast<int>(input_type)));