diff --git a/tmva/sofie/CMakeLists.txt b/tmva/sofie/CMakeLists.txt index 89cf24c6eca67..36ddba3fb1fae 100644 --- a/tmva/sofie/CMakeLists.txt +++ b/tmva/sofie/CMakeLists.txt @@ -24,12 +24,9 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie # target. NO_GLOB_HEADERS HEADERS - TMVA/RModel_Base.hxx TMVA/RModel.hxx TMVA/SOFIE_common.hxx - TMVA/SOFIEHelpers.hxx SOURCES - src/RModel_Base.cxx src/RModel.cxx src/SOFIE_common.cxx src/SOFIE_common_helpers.cxx diff --git a/tmva/sofie/README.md b/tmva/sofie/README.md index 3312f46ee0330..7c2c55a1345e9 100644 --- a/tmva/sofie/README.md +++ b/tmva/sofie/README.md @@ -66,7 +66,6 @@ With the default settings, the weights are contained in a separate binary file, model.Generate(Options::kNoWeightFile); ``` -Other such options includes `Options::kNoSession` (for not generating the Session class, and instead keeping the infer function independent). SOFIE also supports generating inference code with RDataFrame as inputs, refer to the tutorials below for examples. ## Supported ONNX operators diff --git a/tmva/sofie/inc/LinkDef.h b/tmva/sofie/inc/LinkDef.h index 0d747b2786a08..52715254a7456 100644 --- a/tmva/sofie/inc/LinkDef.h +++ b/tmva/sofie/inc/LinkDef.h @@ -8,7 +8,6 @@ #pragma link C++ nestedclass; #pragma link C++ namespace TMVA::Experimental::SOFIE; -#pragma link C++ class TMVA::Experimental::SOFIE::RModel_Base+; #pragma link C++ class TMVA::Experimental::SOFIE::RModel+; #pragma link C++ struct TMVA::Experimental::SOFIE::InitializedTensor+; #pragma link C++ struct TMVA::Experimental::SOFIE::TensorInfo+; diff --git a/tmva/sofie/inc/TMVA/RModel.hxx b/tmva/sofie/inc/TMVA/RModel.hxx index eaa8f78d0a9d7..5d59af24ca5fa 100644 --- a/tmva/sofie/inc/TMVA/RModel.hxx +++ b/tmva/sofie/inc/TMVA/RModel.hxx @@ -1,24 +1,83 @@ #ifndef TMVA_SOFIE_RMODEL #define TMVA_SOFIE_RMODEL -#include "TMVA/RModel_Base.hxx" #include "TMVA/SOFIE_common.hxx" #include "Rtypes.h" // for ClassDefNV +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include namespace TMVA::Experimental::SOFIE { +enum class Options { + kDefault = 0x0, + kNoWeightFile = 0x2, + kRootBinaryWeightFile = 0x4, +}; + +// Optimization levels inspired by ONNXRuntime. +// We only get Operator Fusion with the Basic, and +// memory reuse with Extended. kExtended is enabled +// by default +enum class OptimizationLevel { + kBasic = 0x0, + kExtended = 0x1, +}; + +enum class WeightFileType { None, RootBinary, Text }; + +std::underlying_type_t operator|(Options opA, Options opB); +std::underlying_type_t operator|(std::underlying_type_t opA, Options opB); + // The ROperator interface is an implementation detail of the code generation // and deliberately not exposed to the public: only this forward declaration // is visible here, the full definition (TMVA/ROperator.hxx) is a private // header of the SOFIE libraries. class ROperator; -class RModel final : public RModel_Base { +class RModel final { private: + std::string fFileName; // file name of original model file for identification + std::string fParseTime; // UTC date and time string at parsing + + WeightFileType fWeightFile = WeightFileType::Text; + + std::unordered_set fNeededBlasRoutines; + // Set to true once GenerateHeaderInfo has emitted the extern "C" declaration + // of the BLAS sgemm_ routine (from fNeededBlasRoutines). It lets the + // standalone Gemm_Call helper skip emitting a second, duplicate declaration. + bool fBlasSgemmDeclared = false; + + std::unordered_set fNeededStdLib = {"vector"}; + std::unordered_set fCustomOpHeaders; + + // Inference helper functions (from SOFIE_common) that the generated code + // needs. Their standalone definitions are emitted into the generated header + // so that it does not depend on including TMVA/SOFIE_common.hxx. + std::set fNeededHelperFunctions; + + std::string fName = "UnnamedModel"; + std::string fGC; // generated code + bool fUseWeightFile = true; + + // Placeholder tokens emitted by GenerateHeaderInfo and later replaced by + // EmitHelperFunctionsCode with the actual helper includes / definitions. + // This two-step approach is needed because the full set of required helpers + // is only known once all operators (and sub-graphs) have been generated. + static constexpr const char *kHelperIncludesMarker = "//@SOFIE_HELPER_INCLUDES@\n"; + static constexpr const char *kHelperFunctionsMarker = "//@SOFIE_HELPER_FUNCTIONS@\n"; + bool fIsInitialized = false; bool fIsSubGraph = false; bool fUseVDT = false; @@ -68,7 +127,11 @@ public: https://root.cern/manual/io_custom_classes/#restrictions-on-types-root-io-can-handle */ RModel() = default; - RModel(std::string name, std::string parsedtime) : RModel_Base(name, parsedtime) {} + RModel(std::string name, std::string parsedtime) : fFileName(std::move(name)), fParseTime(std::move(parsedtime)) + { + fName = fFileName.substr(0, fFileName.rfind(".")); + fName = UTILITY::Clean_name(fName); + } // Defined out of line because ROperator is an incomplete type in this // header (the definition is a private implementation header). @@ -233,13 +296,44 @@ public: void PrintOutputTensors() const; void OutputGenerated(std::string filename = "", bool append = false); void SetFilename(std::string filename) { fName = filename; } + std::string GetFilename() { return fName; } + const std::string &GetName() const { return fName; } + + void AddBlasRoutines(std::vector routines) + { + for (auto &routine : routines) { + fNeededBlasRoutines.insert(routine); + } + } + void AddNeededStdLib(std::string libname) + { + // if the library is already in the set, insert does nothing, so we don't need to check before inserting + fNeededStdLib.insert(std::move(libname)); + } + void AddNeededCustomHeader(std::string filename) + { + fCustomOpHeaders.insert(std::move(filename)); + } + // Register an inference helper function that the generated code needs. See + // EmitHelperFunctionsCode for the list of recognised keys. + void AddNeededHelperFunction(std::string name) + { + fNeededHelperFunctions.insert(std::move(name)); + } + const std::set &GetNeededHelperFunctions() const { return fNeededHelperFunctions; } + + void GenerateHeaderInfo(std::string &hgname); + // Replace the helper markers in the generated code with the standalone + // definitions of the helper functions collected in fNeededHelperFunctions. + void EmitHelperFunctionsCode(); + void PrintGenerated(std::ostream &os = std::cout) { os << fGC; } + std::string ReturnGenerated() { return fGC; } void PrintRequiredInputTensors() const; void PrintInitializedTensors() const; void PrintDynamicTensors() const; void HeadInitializedTensors(std::string name, int n_print = 50); - bool UseSession() const { return fUseSession; } // flag to use vdt for fast math functions (e.g. exp in softmax) void SetUseVDT(bool on) { fUseVDT = on; diff --git a/tmva/sofie/inc/TMVA/RModel_Base.hxx b/tmva/sofie/inc/TMVA/RModel_Base.hxx deleted file mode 100644 index 6215692548565..0000000000000 --- a/tmva/sofie/inc/TMVA/RModel_Base.hxx +++ /dev/null @@ -1,120 +0,0 @@ -#ifndef TMVA_SOFIE_RMODEL_BASE -#define TMVA_SOFIE_RMODEL_BASE - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace TMVA::Experimental::SOFIE { - -enum class Options { - kDefault = 0x0, - kNoSession = 0x1, - kNoWeightFile = 0x2, - kRootBinaryWeightFile = 0x4, -}; - -// Optimization levels inspired by ONNXRuntime. -// We only get Operator Fusion with the Basic, and -// memory reuse with Extended. kExtended is enabled -// by default -enum class OptimizationLevel { - kBasic = 0x0, - kExtended = 0x1, -}; - -enum class WeightFileType { None, RootBinary, Text }; - -std::underlying_type_t operator|(Options opA, Options opB); -std::underlying_type_t operator|(std::underlying_type_t opA, Options opB); - -class RModel_Base { - -protected: - std::string fFileName; // file name of original model file for identification - std::string fParseTime; // UTC date and time string at parsing - - WeightFileType fWeightFile = WeightFileType::Text; - - std::unordered_set fNeededBlasRoutines; - // Set to true once GenerateHeaderInfo has emitted the extern "C" declaration - // of the BLAS sgemm_ routine (from fNeededBlasRoutines). It lets the - // standalone Gemm_Call helper skip emitting a second, duplicate declaration. - bool fBlasSgemmDeclared = false; //! - - std::unordered_set fNeededStdLib = {"vector"}; - std::unordered_set fCustomOpHeaders; - - // Inference helper functions (from SOFIE_common) that the generated code - // needs. Their standalone definitions are emitted into the generated header - // so that it does not depend on including TMVA/SOFIE_common.hxx. - std::set fNeededHelperFunctions; - - std::string fName = "UnnamedModel"; - std::string fGC; // generated code - bool fUseWeightFile = true; - bool fUseSession = true; - -public: - /** - Default constructor. Needed to allow serialization of ROOT objects. See - https://root.cern/manual/io_custom_classes/#restrictions-on-types-root-io-can-handle - */ - RModel_Base() = default; - - RModel_Base(std::string name, std::string parsedtime); - - void AddBlasRoutines(std::vector routines) - { - for (auto &routine : routines) { - fNeededBlasRoutines.insert(routine); - } - } - void AddNeededStdLib(std::string libname) - { - // if the library is already in the set, insert does nothing, so we don't need to check before inserting - fNeededStdLib.insert(libname); - } - void AddNeededCustomHeader(std::string filename) - { - fCustomOpHeaders.insert(filename); - } - // Register an inference helper function that the generated code needs. See - // GenerateHelperFunctionsCode for the list of recognised keys. - void AddNeededHelperFunction(std::string name) - { - fNeededHelperFunctions.insert(std::move(name)); - } - const std::set &GetNeededHelperFunctions() const { return fNeededHelperFunctions; } - - // Placeholder tokens emitted by GenerateHeaderInfo and later replaced by - // EmitHelperFunctionsCode with the actual helper includes / definitions. - // This two-step approach is needed because the full set of required helpers - // is only known once all operators (and sub-graphs) have been generated. - static constexpr const char *kHelperIncludesMarker = "//@SOFIE_HELPER_INCLUDES@\n"; - static constexpr const char *kHelperFunctionsMarker = "//@SOFIE_HELPER_FUNCTIONS@\n"; - - void GenerateHeaderInfo(std::string &hgname); - // Replace the helper markers in the generated code with the standalone - // definitions of the helper functions collected in fNeededHelperFunctions. - void EmitHelperFunctionsCode(); - void PrintGenerated(std::ostream &os=std::cout) { os << fGC; } - - std::string ReturnGenerated() { return fGC; } - void OutputGenerated(std::string filename = "", bool append = false); - void SetFilename(std::string filename) { fName = filename; } - std::string GetFilename() { return fName; } - const std::string & GetName() const { return fName;} -}; - -} // namespace TMVA::Experimental::SOFIE - -#endif // TMVA_SOFIE_RMODEL_BASE diff --git a/tmva/sofie/inc/TMVA/ROperator.hxx b/tmva/sofie/inc/TMVA/ROperator.hxx index da8cef4584539..27ca4bf8e3202 100644 --- a/tmva/sofie/inc/TMVA/ROperator.hxx +++ b/tmva/sofie/inc/TMVA/ROperator.hxx @@ -40,7 +40,6 @@ public: protected: const std::string SP = " "; ///< space used to correctly indent the generated C++ code - bool fUseSession = false; ///< flag to identify if using the session class bool fIsOutputConstant = false; ///< flag to identify if operator has a constant output (no need to generate code) bool fIsOutputParamShape = false; ///< flag to identify of the output represents a parametric shape (can be known at compile time) diff --git a/tmva/sofie/inc/TMVA/ROperator_Conv.hxx b/tmva/sofie/inc/TMVA/ROperator_Conv.hxx index ecadb48b3508c..3f45855231dbb 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Conv.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Conv.hxx @@ -236,7 +236,6 @@ public: } void Initialize(RModel& model) override { - fUseSession = model.UseSession(); if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE Conv op Input Tensor " + fNX + " is not found in model"); @@ -272,7 +271,6 @@ public: auto shapeDimB = model.GetDimTensorShape(fNB); bool broadcast_needed = !UTILITY::AreSameShape(shapeDimB, targetShape); if (broadcast_needed) { - auto original_data = model.GetInitializedTensorData(fNB); // make bias shape equal to Y shape by adding 1 if (fShapeB.size() < 1) throw std::runtime_error("TMVA SOFIE Conv op: Bias Tensor has empty shape"); @@ -283,19 +281,8 @@ public: ConvertShapeToString(fShapeB)); if (fType != "float") throw std::runtime_error("TMVA SOFIE Conv op: Broadcasting for non-float type tensors is not supported"); - // here is the actual broadcasting + // here is the actual broadcasting (done in the Session constructor via GenerateInitCode) fBroadcastBias = true; - if (!fUseSession) { - // do here broadcasting - std::vector shape(fDim + 1, 1); - shape[0] = fShapeB[0]; - auto intTargetShape = ConvertShapeToInt(targetShape); - std::shared_ptr new_data_ptr( - UTILITY::UnidirectionalBroadcast(static_cast(original_data.get()), shape, intTargetShape), - std::default_delete()); - model.UpdateInitializedTensor(fNB, model.GetTensorType(fNB), intTargetShape, new_data_ptr); - fShapeB = model.GetTensorShape(fNB); - } } } // output channel size can be parametric and is an expression diff --git a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx index 3ec7682b68da4..e29d26b68efdf 100644 --- a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx @@ -219,7 +219,6 @@ template void ROperator_ConvTranspose::Initialize(RModel &model) { - fUseSession = model.UseSession(); if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE Conv Transpose op Input Tensor " + fNX + " is not found in model"); } @@ -262,27 +261,13 @@ void ROperator_ConvTranspose::Initialize(RModel &model) throw std::runtime_error("TMVA SOFIE ConvTrans op: Bias Tensor has wrong shape: " + ConvertShapeToString(fShapeB)); - auto original_data = model.GetInitializedTensorData(fNB); - if (fType != "float") throw std::runtime_error( "TMVA SOFIE ConvTrans op: Broadcasting for non-float type tensors is not supported"); - // here the acual broadcasting - if (!fUseSession) { - // Broadcast B from M to N x M x Od x Oh x Ow - std::shared_ptr new_data_ptr( - UTILITY::BroadcastConvBias(static_cast(original_data.get()), bsize, fShapeY), - std::default_delete()); - - model.UpdateInitializedTensor(fNB, model.GetTensorType(fNB), fShapeY, new_data_ptr); - fShapeB = model.GetTensorShape(fNB); - fNBroadcastedB = fNB; // use same name - } else { - // In case of session add broadcasting code in Session constructor and in GenerateInitCode - // we need to add a new intermediate tensor for broadcasted bias tensor - fNBroadcastedB = "Broadcasted" + fNB; - model.AddIntermediateTensor(fNBroadcastedB, model.GetTensorType(fNB), fShapeY); - } + // Add broadcasting code in Session constructor and in GenerateInitCode: + // we need to add a new intermediate tensor for broadcasted bias tensor + fNBroadcastedB = "Broadcasted" + fNB; + model.AddIntermediateTensor(fNBroadcastedB, model.GetTensorType(fNB), fShapeY); } else { // bias tensor is already correct shape, no need to broadcast if (fShapeY != fShapeB) @@ -362,14 +347,6 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) out << "\n//---- operator ConvTranspose " << OpName << "\n"; - // create first matrix with convolution kernels - if (!fUseSession) { - size_t kernelSize = fAttrKernelShape[0]; - if (fDim > 1) - kernelSize *= fAttrKernelShape[1]; - out << SP << fType << " tensor_" << fConvK << "[" << fShapeW[0] * fShapeW[1] * kernelSize << "] = {0};\n"; - } - // vectorize the (dilated)convolution kernels into a matrix // The shape of the kernel is W for 1d image, H x W for 2d image and D x H x W // for 3d image @@ -439,11 +416,6 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) out << SP << "float " << OpName << "_alpha = 1.0;\n"; out << SP << "float " << OpName << "_beta = 0.0;\n"; - if (!fUseSession) { - out << SP << fType << " tensor_" << fImcol << "[" << fShapeW[0] * icstrideDil * oDepth * oHeight * oWidth - << "] = {0};\n"; - } - // Loop on batch size out << SP << "for (size_t n = 0; n < " << bsize << "; n++) {\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_GRU.hxx b/tmva/sofie/inc/TMVA/ROperator_GRU.hxx index ce6d0641f12ef..899d0b5dc4237 100644 --- a/tmva/sofie/inc/TMVA/ROperator_GRU.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_GRU.hxx @@ -178,7 +178,6 @@ template void ROperator_GRU::Initialize(RModel &model) { - fUseSession = model.UseSession(); // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE GRU Op input tensor " + fNX + " is not found in model."); @@ -360,11 +359,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " const* " << OpName << "_input = tensor_" << fNX << ";\n"; } else { - if (fUseSession) { - out << SP << fType << " * " << OpName << "_input = " << getVec("input") << ";\n"; - } else { - out << SP << fType << " " << OpName << "_input[" << seq_length * batch_size * input_size << "];\n"; - } + out << SP << fType << " * " << OpName << "_input = " << getVec("input") << ";\n"; out << SP << "for(size_t seq = 0; seq < " << seq_length << "; seq++) {\n"; out << SP << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; out << SP << SP << SP << "for(size_t i = 0; i < " << input_size << "; i++) {\n"; @@ -381,13 +376,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " *" << OpName << "_initial_hidden_state = " << " tensor_" << fNInitial_h << ";\n"; } else { - if (fUseSession) { - out << SP << fType << " * " << OpName << "_initial_hidden_state = " << getVec("initial_hidden_state") - << ";\n"; - } else { - out << SP << fType << " " << OpName << "_initial_hidden_state[" - << num_directions * batch_size * fAttrHiddenSize << "];\n"; - } + out << SP << fType << " * " << OpName << "_initial_hidden_state = " << getVec("initial_hidden_state") << ";\n"; for (size_t direction = 0; direction < num_directions; direction++) { out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; out << SP << SP << "for(size_t h = 0; h < " << fAttrHiddenSize << "; h++) {\n"; @@ -401,43 +390,21 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string } // Set the feedforward - size_t feedforward_size = seq_length * batch_size * fAttrHiddenSize; - if (fUseSession) { - out << SP << fType << " * " << OpName << "_f_update_gate = " << getVec("f_update_gate") << ";\n"; - out << SP << fType << " * " << OpName << "_f_reset_gate = " << getVec("f_reset_gate") << ";\n"; - out << SP << fType << " * " << OpName << "_f_hidden_gate = " << getVec("f_hidden_gate") << ";\n"; - } else { - out << SP << fType << " " << OpName << "_f_update_gate[" << feedforward_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_f_reset_gate[" << feedforward_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_f_hidden_gate[" << feedforward_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_f_update_gate = " << getVec("f_update_gate") << ";\n"; + out << SP << fType << " * " << OpName << "_f_reset_gate = " << getVec("f_reset_gate") << ";\n"; + out << SP << fType << " * " << OpName << "_f_hidden_gate = " << getVec("f_hidden_gate") << ";\n"; // Set the gates - size_t hidden_state_size = seq_length * num_directions * batch_size * fAttrHiddenSize; - if (fUseSession) { - out << SP << fType << " * " << OpName << "_update_gate = " << getVec("update_gate") << ";\n"; - out << SP << fType << " * " << OpName << "_reset_gate = " << getVec("reset_gate") << ";\n"; - out << SP << fType << " * " << OpName << "_hidden_gate = " << getVec("hidden_gate") << ";\n"; - } else { - out << SP << fType << " " << OpName << "_update_gate[" << hidden_state_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_reset_gate[" << hidden_state_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_hidden_gate[" << hidden_state_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_update_gate = " << getVec("update_gate") << ";\n"; + out << SP << fType << " * " << OpName << "_reset_gate = " << getVec("reset_gate") << ";\n"; + out << SP << fType << " * " << OpName << "_hidden_gate = " << getVec("hidden_gate") << ";\n"; // Set the hidden state if (fAttrLayout == 0 && !fNY.empty()) { out << SP << fType << " *" << OpName << "_hidden_state = tensor_" << fNY << ";\n"; } else { - if (fUseSession) { - out << SP << fType << " * " << OpName << "_hidden_state = " << getVec("hidden_state") << ";\n"; - } else { - out << SP << fType << " " << OpName << "_hidden_state[" << hidden_state_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_hidden_state = " << getVec("hidden_state") << ";\n"; } - if (fUseSession) { - out << SP << fType << " * " << OpName << "_feedback = " << getVec("feedback") << ";\n"; - } else { - out << SP << fType << " " << OpName << "_feedback[" << batch_size * fAttrHiddenSize << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_feedback = " << getVec("feedback") << ";\n"; out << SP << "char " << OpName << "_transA = 'N';\n"; out << SP << "char " << OpName << "_transB = 'T';\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx b/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx index ae0ee70c4eeea..3c4355a04881e 100644 --- a/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx @@ -205,7 +205,6 @@ auto ROperator_LSTM::ShapeInference(std::vector> input) - template auto ROperator_LSTM::Initialize(RModel &model) -> void { - fUseSession = model.UseSession(); // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNX + " is not found in model."); @@ -468,10 +467,7 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " const *" << OpName << "_input = tensor_" << fNX << ";\n"; } else { - if (fUseSession) - out << SP << fType << " * " << OpName << "_input = this->fVec_" << OpName << "_input;\n"; - else - out << SP << fType << " " << OpName << "_input[" << seq_length * batch_size * input_size << "] = {0};\n"; + out << SP << fType << " * " << OpName << "_input = this->fVec_" << OpName << "_input;\n"; out << SP << "for(size_t seq = 0; seq < " << seq_length << "; seq++) {\n"; out << SP << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; @@ -489,12 +485,8 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " const*" << OpName << "_initial_hidden_state = " << " tensor_" << fNInitial_h << ";\n"; } else { - if (fUseSession) - out << SP << fType << " const* " << OpName << "_initial_hidden_state = this->fVec_" << OpName - << "_initial_hidden_state;\n"; - else - out << SP << fType << " " << OpName << "_initial_hidden_state[" - << num_directions * batch_size * fAttrHiddenSize << "] = {0};\n"; + out << SP << fType << " const* " << OpName << "_initial_hidden_state = this->fVec_" << OpName + << "_initial_hidden_state;\n"; for (size_t direction = 0; direction < num_directions; direction++) { out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; @@ -513,12 +505,8 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " const*" << OpName << "_initial_cell_state = " << " tensor_" << fNInitial_c << ";\n"; } else { - if (fUseSession) - out << SP << fType << " const* " << OpName << "_initial_cell_state = this->fVec_" << OpName - << "_initial_cell_state;\n"; - else - out << SP << fType << " " << OpName << "_initial_cell_state[" - << num_directions * batch_size * fAttrHiddenSize << "] = {0};\n"; + out << SP << fType << " const* " << OpName << "_initial_cell_state = this->fVec_" << OpName + << "_initial_cell_state;\n"; for (size_t direction = 0; direction < num_directions; direction++) { out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; @@ -533,58 +521,28 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string } // Set the feedforward - size_t ff_size = seq_length * batch_size * fAttrHiddenSize; - if (fUseSession) { - out << SP << fType << " * " << OpName << "_ff_input_gate = this->fVec_" << OpName << "_ff_input_gate;\n"; - out << SP << fType << " * " << OpName << "_ff_output_gate = this->fVec_" << OpName << "_ff_output_gate;\n"; - out << SP << fType << " * " << OpName << "_ff_cell_gate = this->fVec_" << OpName << "_ff_cell_gate;\n"; - if (fAttrInputForget == 0) { - out << SP << fType << " * " << OpName << "_ff_forget_gate = this->fVec_" << OpName - << "_ff_forget_gate;\n"; - } - } else { - out << SP << fType << " " << OpName << "_ff_input_gate[" << ff_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_ff_output_gate[" << ff_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_ff_cell_gate[" << ff_size << "] = {0};\n"; - if (fAttrInputForget == 0) { - out << SP << fType << " " << OpName << "_ff_forget_gate[" << ff_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_ff_input_gate = this->fVec_" << OpName << "_ff_input_gate;\n"; + out << SP << fType << " * " << OpName << "_ff_output_gate = this->fVec_" << OpName << "_ff_output_gate;\n"; + out << SP << fType << " * " << OpName << "_ff_cell_gate = this->fVec_" << OpName << "_ff_cell_gate;\n"; + if (fAttrInputForget == 0) { + out << SP << fType << " * " << OpName << "_ff_forget_gate = this->fVec_" << OpName << "_ff_forget_gate;\n"; } // Set the gates - size_t hidden_state_size = seq_length * num_directions * batch_size * fAttrHiddenSize; - if (fUseSession) { - out << SP << fType << " * " << OpName << "_input_gate = this->fVec_" << OpName << "_input_gate;\n"; - out << SP << fType << " * " << OpName << "_output_gate = this->fVec_" << OpName << "_output_gate;\n"; - out << SP << fType << " * " << OpName << "_cell_gate = this->fVec_" << OpName << "_cell_gate;\n"; - if (fAttrInputForget == 0) { - out << SP << fType << " * " << OpName << "_forget_gate = this->fVec_" << OpName << "_forget_gate;\n"; - } - } else { - out << SP << fType << " " << OpName << "_input_gate[" << hidden_state_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_output_gate[" << hidden_state_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_cell_gate[" << hidden_state_size << "] = {0};\n"; - if (fAttrInputForget == 0) { - out << SP << fType << " " << OpName << "_forget_gate[" << hidden_state_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_input_gate = this->fVec_" << OpName << "_input_gate;\n"; + out << SP << fType << " * " << OpName << "_output_gate = this->fVec_" << OpName << "_output_gate;\n"; + out << SP << fType << " * " << OpName << "_cell_gate = this->fVec_" << OpName << "_cell_gate;\n"; + if (fAttrInputForget == 0) { + out << SP << fType << " * " << OpName << "_forget_gate = this->fVec_" << OpName << "_forget_gate;\n"; } // Set the cell state and the new cell state = h(cell state) - if (fUseSession) { - out << SP << fType << " * " << OpName << "_cell_state = this->fVec_" << OpName << "_cell_state;\n"; - out << SP << fType << " * " << OpName << "_new_cell_state = this->fVec_" << OpName << "_new_cell_state;\n"; - } else { - out << SP << fType << " " << OpName << "_cell_state[" << hidden_state_size << "] = {0};\n"; - out << SP << fType << " " << OpName << "_new_cell_state[" << hidden_state_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_cell_state = this->fVec_" << OpName << "_cell_state;\n"; + out << SP << fType << " * " << OpName << "_new_cell_state = this->fVec_" << OpName << "_new_cell_state;\n"; // Set the hidden state if (fAttrLayout == 0 && !fNY.empty()) { out << SP << fType << " *" << OpName << "_hidden_state = tensor_" << fNY << ";\n"; } else { - if (fUseSession) { - out << SP << fType << " * " << OpName << "_hidden_state = this->fVec_" << OpName << "_hidden_state;\n"; - } else { - out << SP << fType << " " << OpName << "_hidden_state[" << hidden_state_size << "] = {0};\n"; - } + out << SP << fType << " * " << OpName << "_hidden_state = this->fVec_" << OpName << "_hidden_state;\n"; } out << SP << "char " << OpName << "_transA = 'N';\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_Pool.hxx b/tmva/sofie/inc/TMVA/ROperator_Pool.hxx index 05b4f793526d8..cd3ccabd83d7e 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Pool.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Pool.hxx @@ -56,7 +56,6 @@ private: std::string fType; size_t fDim; // dimension of the MaxPool - bool fUseSession = false; public: @@ -206,8 +205,6 @@ public: void Initialize(RModel& model) override { - fUseSession = model.UseSession(); - if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE Pool op Input Tensor " + fNX + " is not found in model"); diff --git a/tmva/sofie/inc/TMVA/ROperator_RNN.hxx b/tmva/sofie/inc/TMVA/ROperator_RNN.hxx index f385a502d4077..f12009c0751e9 100644 --- a/tmva/sofie/inc/TMVA/ROperator_RNN.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_RNN.hxx @@ -176,7 +176,6 @@ auto ROperator_RNN::ShapeInference(std::vector> input) -> template auto ROperator_RNN::Initialize(RModel &model) -> void { - fUseSession = model.UseSession(); // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE RNN Op input tensor " + fNX + " is not found in model."); @@ -365,10 +364,7 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string out << SP << "float const*" << OpName << "_input = tensor_" << fNX << ";\n"; } } else { - if (fUseSession) - out << SP << fType << " * " << OpName << "_input = this->fVec_" << OpName << "_input;\n"; - else - out << SP << fType << " " << OpName << "_input[" << seq_length * batch_size * input_size << "];\n"; + out << SP << fType << " * " << OpName << "_input = this->fVec_" << OpName << "_input;\n"; out << SP << "for(size_t seq = 0; seq < " << seq_length << "; seq++) {\n"; out << SP << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; out << SP << SP << SP << "for(size_t i = 0; i < " << input_size << "; i++) {\n"; @@ -385,12 +381,8 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string if (fAttrLayout == 0) { out << SP << fType << " *" << OpName << "_initial_hidden_state = " << " tensor_" << fNInitial_h << ";\n"; } else { - if (fUseSession) - out << SP << fType << " * " << OpName << "_initial_hidden_state = this->fVec_" << OpName - << "_initial_hidden_state;\n"; - else - out << fType << " " << OpName << "_initial_hidden_state[" << num_directions * batch_size * fAttrHiddenSize - << "] = {0};\n"; + out << SP << fType << " * " << OpName << "_initial_hidden_state = this->fVec_" << OpName + << "_initial_hidden_state;\n"; for (size_t direction = 0; direction < num_directions; direction++) { out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n"; @@ -404,21 +396,13 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string } } - if (fUseSession) - out << SP << fType << " * " << OpName << "_feedforward = this->fVec_" << OpName << "_feedforward;\n"; - else - out << SP << fType << " " << OpName << "_feedforward[" << seq_length * batch_size * fAttrHiddenSize - << "] = {0};\n"; + out << SP << fType << " * " << OpName << "_feedforward = this->fVec_" << OpName << "_feedforward;\n"; // Set the hidden state if (fAttrLayout == 0 && !fNY.empty()) { out << SP << fType << " *" << OpName << "_hidden_state = tensor_" << fNY << ";\n"; } else { - if (fUseSession) - out << SP << fType << " * " << OpName << "_hidden_state = this->fVec_" << OpName << "_hidden_state;\n"; - else - out << SP << fType << " " << OpName << "_hidden_state[" - << seq_length * num_directions * batch_size * fAttrHiddenSize << "] = {0};\n"; + out << SP << fType << " * " << OpName << "_hidden_state = this->fVec_" << OpName << "_hidden_state;\n"; } out << SP << "char " << OpName << "_transA = 'N';\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_Reduce.hxx b/tmva/sofie/inc/TMVA/ROperator_Reduce.hxx index 824d11e44053e..17dd968627f7f 100755 --- a/tmva/sofie/inc/TMVA/ROperator_Reduce.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Reduce.hxx @@ -104,8 +104,6 @@ private: } void Initialize(RModel& model) override { - fUseSession = model.UseSession(); - if (!model.CheckIfTensorAlreadyExist(fNX)) { // input must be a graph input, or already initialized intermediate tensor throw std::runtime_error("TMVA SOFIE Reduce Op Input Tensor " + fNX + " is not found in model"); diff --git a/tmva/sofie/inc/TMVA/SOFIEHelpers.hxx b/tmva/sofie/inc/TMVA/SOFIEHelpers.hxx deleted file mode 100644 index def88feecf1ae..0000000000000 --- a/tmva/sofie/inc/TMVA/SOFIEHelpers.hxx +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef TMVA_SOFIE_SOFIE_HELPERS -#define TMVA_SOFIE_SOFIE_HELPERS - - -#include -#include -#include -#include - - -namespace TMVA{ -namespace Experimental{ - -///Helper class used by SOFIEFunctor to wrap the -///infer signature interface to RDataFrame -template -class SofieFunctorHelper; - -template -class SofieFunctorHelper, Session_t, T> { - /// this is the magic to define the operator() with N fixed parameter arguments - template - using AlwaysT = T; - - std::vector> fInput; - std::vector fSessions; - -public: - - SofieFunctorHelper(unsigned int nslots = 0, const std::string & filename = "") : - fInput(1) - { - // create Sessions according to given number of slots. - // if number of slots is zero create a single session - if (nslots < 1) nslots = 1; - fInput.resize(nslots); - fSessions.reserve(nslots); - for (unsigned int i = 0; i < nslots; i++) { - if (filename.empty()) - fSessions.emplace_back(); - else - fSessions.emplace_back(filename); - } - } - - double operator()(unsigned slot, AlwaysT... args) { - fInput[slot] = {args...}; - auto y = fSessions[slot].infer(fInput[slot].data()); - return y[0]; - } -}; - -/// SofieFunctor : used to wrap the infer function of the -/// generated model by SOFIE in a RDF compatible signature. -/// The number of slots is an optional parameter used to -/// create multiple SOFIE Sessions, which can be run in a parallel -/// model evaluation. One should use as number of slots the number of slots used by -/// RDataFrame. By default, in case of `nslots=0`, only a single Session will be created -/// and the Functor cannot be run in parallel. -/// Examples of using the SofieFunctor are the C++ tutorial TMVA_SOFIE_RDataFrame.C -/// and the Python tutorial TMVA_SOFIE_RDataFrame.py which makes use of the ROOT JIT -/// to compile on the fly the generated SOFIE model. -template -auto SofieFunctor(unsigned int nslots = 0, const std::string & weightsFile = "") -> SofieFunctorHelper, Session_t, float> -{ - return SofieFunctorHelper, Session_t, float>(nslots, weightsFile); -} - -}//Experimental -}//TMVA - -#endif //TMVA_SOFIE_SOFIE_HELPERS diff --git a/tmva/sofie/inc/TMVA/SOFIE_common.hxx b/tmva/sofie/inc/TMVA/SOFIE_common.hxx index fb3eaf7157521..e0fa83ba247ec 100644 --- a/tmva/sofie/inc/TMVA/SOFIE_common.hxx +++ b/tmva/sofie/inc/TMVA/SOFIE_common.hxx @@ -536,7 +536,7 @@ struct HelperFunctionsCode { }; /// Return the standalone C++ source of the inference helper functions requested -/// in `neededHelpers` (see RModel_Base::AddNeededHelperFunction), resolving +/// in `neededHelpers` (see RModel::AddNeededHelperFunction), resolving /// their inter-dependencies. Recognised keys are: "Im2col", "Im2col_3d", /// "col2im", "UnidirectionalBroadcast", "BroadcastConvBias", "Gemm_Call", /// "Relu", "Fill", "Copy", "ReadTensorFromStream", "InputTensorDims", diff --git a/tmva/sofie/src/RModel.cxx b/tmva/sofie/src/RModel.cxx index 8c4a1326270e4..8eb8b2bca1cfa 100644 --- a/tmva/sofie/src/RModel.cxx +++ b/tmva/sofie/src/RModel.cxx @@ -764,7 +764,6 @@ void RModel::InitializeSubGraph(std::shared_ptr graph) { // set the same options as parent model graph->fWeightFile = fWeightFile; graph->fUseWeightFile = fUseWeightFile; - graph->fUseSession = fUseSession; // add needed blas routines and libs std::vector blasRoutines; for (auto & e : graph->fNeededBlasRoutines) @@ -1261,11 +1260,7 @@ void RModel::GenerateOutput() } } - if (fUseSession) { - fGC += SP + "doInfer(*this, " + doInferArgs + ");\n"; - } else { - fGC += SP + "doInfer(" + doInferArgs + ");\n"; - } + fGC += SP + "doInfer(*this, " + doInferArgs + ");\n"; // If the output tensors have dynamic sizes, now is the time to set them for (std::string const &name : fOutputTensorNames) { @@ -1291,10 +1286,8 @@ void RModel::GenerateSessionCode() { std::string sessionName = !fIsSubGraph ? "Session" : "Session_" + fName; - if (fUseSession) { - // forward declare session struct - fGC += "struct " + sessionName + ";\n"; - } + // forward declare session struct + fGC += "struct " + sessionName + ";\n"; // Determine the signature of the actual inference function std::string doInferSignature = GenerateInferSignature(); @@ -1315,9 +1308,7 @@ void RModel::GenerateSessionCode() } doInferSignature.back() = ' '; - if (fUseSession) { - doInferSignature = sessionName + " const &session, " + doInferSignature; - } + doInferSignature = sessionName + " const &session, " + doInferSignature; doInferSignature = "inline void doInfer(" + doInferSignature + ")"; @@ -1325,9 +1316,7 @@ void RModel::GenerateSessionCode() fGC += doInferSignature + ";\n"; // define the Session struct - if (fUseSession) { - fGC += "struct " + sessionName + " {\n"; - } + fGC += "struct " + sessionName + " {\n"; // generate code for declaring the initialized tensors GenerateInitializedTensorInfo(); @@ -1380,98 +1369,94 @@ void RModel::GenerateSessionCode() } // Generate code for Session constructor - if (fUseSession) { - // add here specific operator code that needs to define session data members - fGC += "\n"; - for (size_t id = 0; id < fOperators.size(); id++) { - std::string opName = std::to_string(id); - fGC += fOperators[id]->GenerateSessionMembersCode(opName); - } - fGC += "\n"; - // here add initialization and reading of weight tensors - if (fUseWeightFile) { - std::string fileName = fName; - if (fWeightFile == WeightFileType::Text) { - fileName += ".dat"; - } - if (fWeightFile == WeightFileType::RootBinary) { - fileName += ".root"; - } - fGC += sessionName + "(std::string filename =\"" + fileName + "\""; - } else { - // no need to pass weight file since it is not used - // keep passing a string for compatibility - fGC += sessionName + "(std::string = \"\""; + // add here specific operator code that needs to define session data members + fGC += "\n"; + for (size_t id = 0; id < fOperators.size(); id++) { + std::string opName = std::to_string(id); + fGC += fOperators[id]->GenerateSessionMembersCode(opName); + } + fGC += "\n"; + // here add initialization and reading of weight tensors + if (fUseWeightFile) { + std::string fileName = fName; + if (fWeightFile == WeightFileType::Text) { + fileName += ".dat"; } - // add initialization of shape parameters - // assume all parameters are of type size_t - if (!fDimShapeNames.empty()) { - // need to use same order as in infer function not alphabetical one - for (auto &p : fDimShapeNames) { - fGC += ",\n"; - fGC += " size_t " + p + " = " + fShapeParams[p]; - } + if (fWeightFile == WeightFileType::RootBinary) { + fileName += ".root"; } - fGC += ") {\n"; - - // initializing dynamic parameters - if (!fDimShapeNames.empty()) { - fGC += "\n\n"; - std::sort(fDimShapeNames.begin(), fDimShapeNames.end()); - for (const auto &p : fDimShapeNames) { - fGC += " " + memberNameForDimShape(p) + " = " + p + ";\n"; - } + fGC += sessionName + "(std::string filename =\"" + fileName + "\""; + } else { + // no need to pass weight file since it is not used + // keep passing a string for compatibility + fGC += sessionName + "(std::string = \"\""; + } + // add initialization of shape parameters + // assume all parameters are of type size_t + if (!fDimShapeNames.empty()) { + // need to use same order as in infer function not alphabetical one + for (auto &p : fDimShapeNames) { + fGC += ",\n"; + fGC += " size_t " + p + " = " + fShapeParams[p]; } - // add some extra code needed for initialization of dynamic parameters - fGC += fExtraCodeForDimShapes; + } + fGC += ") {\n"; - if (fUseWeightFile) { - fGC += "\n//--- reading weights from file\n"; - ReadInitializedTensorsFromFile(); - fGC += "\n"; - // fUseWeightFile = fUseWeightFile; + // initializing dynamic parameters + if (!fDimShapeNames.empty()) { + fGC += "\n\n"; + std::sort(fDimShapeNames.begin(), fDimShapeNames.end()); + for (const auto &p : fDimShapeNames) { + fGC += " " + memberNameForDimShape(p) + " = " + p + ";\n"; } + } + // add some extra code needed for initialization of dynamic parameters + fGC += fExtraCodeForDimShapes; - // now we have passed the parameters we can allocate the dynamic tensors - GenerateDynamicTensorInfo(); + if (fUseWeightFile) { + fGC += "\n//--- reading weights from file\n"; + ReadInitializedTensorsFromFile(); + fGC += "\n"; + // fUseWeightFile = fUseWeightFile; + } - // add here initialization code for operator - for (size_t id = 0; id < fOperators.size(); id++) { - fGC += fOperators[id]->GenerateInitCode(); - } + // now we have passed the parameters we can allocate the dynamic tensors + GenerateDynamicTensorInfo(); - fGC += "}\n\n"; - - // Used to build the tangent Session objects needed to differentiate the - // generated code with Clad: the derivatives of the (constant) weights - // are zero, but a default-constructed Session holds the actual values. - fGC += "// Set all weight and constant tensors to zero. This is useful to create\n" - "// the tangent Session objects needed to differentiate the generated code\n" - "// with Clad.\n" - "void SetWeightsToZero() {\n"; - for (auto &i : fInitializedTensors) { - // IsNotWritable tensors have no emitted fTensor_ member, and integer - // tensors are structural (indices, shapes) that carry no tangent. - if (i.second.IsNotWritable() || - (i.second.type() != ETensorType::FLOAT && i.second.type() != ETensorType::DOUBLE)) - continue; - fGC += " for (auto &v : fTensor_" + i.first + ") v = 0;\n"; - } - for (auto &graph : fSubGraphs) { - fGC += " fSession_" + graph->fName + ".SetWeightsToZero();\n"; - } - fGC += "}\n\n"; + // add here initialization code for operator + for (size_t id = 0; id < fOperators.size(); id++) { + fGC += fOperators[id]->GenerateInitCode(); } + fGC += "}\n\n"; + + // Used to build the tangent Session objects needed to differentiate the + // generated code with Clad: the derivatives of the (constant) weights + // are zero, but a default-constructed Session holds the actual values. + fGC += "// Set all weight and constant tensors to zero. This is useful to create\n" + "// the tangent Session objects needed to differentiate the generated code\n" + "// with Clad.\n" + "void SetWeightsToZero() {\n"; + for (auto &i : fInitializedTensors) { + // IsNotWritable tensors have no emitted fTensor_ member, and integer + // tensors are structural (indices, shapes) that carry no tangent. + if (i.second.IsNotWritable() || + (i.second.type() != ETensorType::FLOAT && i.second.type() != ETensorType::DOUBLE)) + continue; + fGC += " for (auto &v : fTensor_" + i.first + ") v = 0;\n"; + } + for (auto &graph : fSubGraphs) { + fGC += " fSession_" + graph->fName + ".SetWeightsToZero();\n"; + } + fGC += "}\n\n"; + // generate the inference overload that returns an output struct GenerateOutput(); // end of session - if (fUseSession) { - fGC += "}; // end of Session\n\n"; + fGC += "}; // end of Session\n\n"; - GenerateRequiredInputTensorInfo(); - } + GenerateRequiredInputTensorInfo(); fGC += doInferSignature + " {\n"; fGC += "\n"; @@ -1496,14 +1481,12 @@ void RModel::GenerateSessionCode() // local variable name that we're using for the session: ReplaceAll(allOperatorCode, "this->", "session."); - if (fUseSession) { - // Collect all "tensor_*" data members that are not input or output tensors - std::vector tensorMemberNames = CollectTensorMemberNames(allOperatorCode); - for (auto const& name: tensorMemberNames) { - fGC += " auto &" + name + " = session." + name + ";\n"; - } - fGC += "\n"; + // Collect all "tensor_*" data members that are not input or output tensors + std::vector tensorMemberNames = CollectTensorMemberNames(allOperatorCode); + for (auto const& name: tensorMemberNames) { + fGC += " auto &" + name + " = session." + name + ";\n"; } + fGC += "\n"; fGC += allOperatorCode; @@ -1533,11 +1516,7 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, bo fVerbose = verbose; fBatchSize = batchSize; - // session flag is used in operator initialize - if (static_cast>(Options::kNoSession) & options) { - fUseSession = false; - fWeightFile = WeightFileType::None; - } + // weights are contained in the generated header when kNoWeightFile is used if (static_cast>(Options::kNoWeightFile) & options) { fUseWeightFile = false; fWeightFile = WeightFileType::None; @@ -1546,21 +1525,10 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, bo fUseWeightFile = true; fWeightFile = WeightFileType::RootBinary; } - if (fUseWeightFile && !fUseSession) { - throw std::runtime_error( - "TMVA-SOFIE: RModel::Generate: cannot use a separate weight file without generating a Session class"); - } // initialize the model including all operators and sub-graphs Initialize(batchSize, verbose); - // if having dynamic tensor we need to have a Session - if (!fDynamicTensorInfos.empty()) { - fUseSession = true; - if (verbose) - std::cout << "Warning: Force having a Session since model has dynamic tensors " << std::endl; - } - std::string hgname; if (!fIsSubGraph) { fGC.clear(); @@ -1995,9 +1963,110 @@ void RModel::HeadInitializedTensors(std::string name, int n_print) { } +void RModel::GenerateHeaderInfo(std::string& hgname) { + fGC += ("//Code generated automatically by TMVA for Inference of Model file [" + fFileName + "] at [" + fParseTime.substr(0, fParseTime.length()-1) +"] \n"); + // add header guards + hgname = fName; + std::transform(hgname.begin(), hgname.end(), hgname.begin(), [](unsigned char c) { + return std::toupper(c); + } ); + hgname = "ROOT_TMVA_SOFIE_" + hgname; + fGC += "\n#ifndef " + hgname + "\n"; + fGC += "#define " + hgname + "\n\n"; + // Standard library headers the generated code relies on. Listed explicitly + // now that they are no longer pulled in transitively via SOFIE_common.hxx. + for (const char *h : {"cstdint", "cstring", "string", "vector", "map", "memory", "sstream", "iostream", "iomanip", + "limits", "stdexcept", "algorithm", "cmath", "cassert"}) { + fNeededStdLib.insert(h); + } + for (auto& i: fNeededStdLib) { + fGC += "#include <" + i + ">\n"; + } + for (auto& i: fCustomOpHeaders) { + fGC += "#include \"" + i + "\"\n"; + } + // Placeholder for the #include directives needed by the embedded helper + // functions (filled in by EmitHelperFunctionsCode). + fGC += kHelperIncludesMarker; + if (fUseWeightFile) + fGC += "#include \n"; + // Include TFile when saving the weights in a binary ROOT file + if (fWeightFile == WeightFileType::RootBinary) + fGC += "#include \"TFile.h\"\n"; + + fGC += "\nnamespace TMVA_SOFIE_" + fName + "{\n"; + if (!fNeededBlasRoutines.empty()) { + fGC += ("namespace BLAS{\n"); + for (auto &routine : fNeededBlasRoutines) { + if (routine == "Gemm") { + fGC += ("\textern \"C\" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k,\n" + "\t const float * alpha, const float * A, const int * lda, const float * B, const int * ldb,\n" + "\t const float * beta, float * C, const int * ldc);\n"); + // sgemm_ now declared; the standalone Gemm_Call helper will skip its copy. + fBlasSgemmDeclared = true; + } else if (routine == "Gemv") { + fGC += ("\textern \"C\" void sgemv_(const char * trans, const int * m, const int * n, const float * alpha, const float * A,\n" + "\t const int * lda, const float * X, const int * incx, const float * beta, const float * Y, const int * incy);\n"); + } else if (routine == "Axpy") { + fGC += ("\textern \"C\" void saxpy_(const int * n, const float * alpha, const float * x,\n" + "\t const int * incx, float * y, const int * incy);\n"); + } else if (routine == "Copy") { + fGC += ("\textern \"C\" void scopy_(const int *n, const float* x, const int *incx, float* y, const int* incy);\n"); + } + } + fGC += ("}//BLAS\n"); + } + // Placeholder for the standalone definitions of the inference helper + // functions used by this model (filled in by EmitHelperFunctionsCode). It + // sits inside the generated model namespace, right before the session code. + fGC += kHelperFunctionsMarker; +} + +void RModel::EmitHelperFunctionsCode() +{ + HelperFunctionsCode code = + GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName, fBlasSgemmDeclared); + + auto replaceMarker = [this](const std::string &marker, const std::string &replacement) { + auto pos = fGC.find(marker); + if (pos != std::string::npos) { + fGC.replace(pos, marker.size(), replacement); + } + }; + + replaceMarker(kHelperIncludesMarker, code.includes); + replaceMarker(kHelperFunctionsMarker, code.definitions); + + // Clad derivatives live at file scope and reference the model's helpers, so + // insert them after the whole model namespace, just before the include guard. + if (!code.cladDefinitions.empty()) { + auto pos = fGC.rfind("#endif"); + if (pos != std::string::npos) { + fGC.insert(pos, code.cladDefinitions + "\n"); + } else { + fGC += code.cladDefinitions; + } + } +} + void RModel::OutputGenerated(std::string filename, bool append) { - RModel_Base::OutputGenerated(filename, append); + // the model can be appended only if a file name is provided + if (filename.empty()) { + // if a file is pr + filename = fName + ".hxx"; + append = false; + } + std::ofstream f; + if (append) + f.open(filename, std::ios_base::app); + else + f.open(filename); + if (!f.is_open()) { + throw std::runtime_error("tmva-sofie failed to open file for output generated inference code"); + } + f << fGC; + f.close(); // write weights in a text file if (fUseWeightFile) { diff --git a/tmva/sofie/src/RModel_Base.cxx b/tmva/sofie/src/RModel_Base.cxx deleted file mode 100644 index 82ed8fb1c931b..0000000000000 --- a/tmva/sofie/src/RModel_Base.cxx +++ /dev/null @@ -1,124 +0,0 @@ -#include -#include -#include - -#include "TMVA/RModel_Base.hxx" -#include "TMVA/SOFIE_common.hxx" - -namespace TMVA { -namespace Experimental { -namespace SOFIE { - -RModel_Base::RModel_Base(std::string name, std::string parsedtime):fFileName(name), fParseTime(parsedtime) { - fName = fFileName.substr(0, fFileName.rfind(".")); - fName = UTILITY::Clean_name(fName); -} - -void RModel_Base::GenerateHeaderInfo(std::string& hgname) { - fGC += ("//Code generated automatically by TMVA for Inference of Model file [" + fFileName + "] at [" + fParseTime.substr(0, fParseTime.length()-1) +"] \n"); - // add header guards - hgname = fName; - std::transform(hgname.begin(), hgname.end(), hgname.begin(), [](unsigned char c) { - return std::toupper(c); - } ); - hgname = "ROOT_TMVA_SOFIE_" + hgname; - fGC += "\n#ifndef " + hgname + "\n"; - fGC += "#define " + hgname + "\n\n"; - // Standard library headers the generated code relies on. Listed explicitly - // now that they are no longer pulled in transitively via SOFIE_common.hxx. - for (const char *h : {"cstdint", "cstring", "string", "vector", "map", "memory", "sstream", "iostream", "iomanip", - "limits", "stdexcept", "algorithm", "cmath", "cassert"}) { - fNeededStdLib.insert(h); - } - for (auto& i: fNeededStdLib) { - fGC += "#include <" + i + ">\n"; - } - for (auto& i: fCustomOpHeaders) { - fGC += "#include \"" + i + "\"\n"; - } - // Placeholder for the #include directives needed by the embedded helper - // functions (filled in by EmitHelperFunctionsCode). - fGC += kHelperIncludesMarker; - if (fUseWeightFile) - fGC += "#include \n"; - // Include TFile when saving the weights in a binary ROOT file - if (fWeightFile == WeightFileType::RootBinary) - fGC += "#include \"TFile.h\"\n"; - - fGC += "\nnamespace TMVA_SOFIE_" + fName + "{\n"; - if (!fNeededBlasRoutines.empty()) { - fGC += ("namespace BLAS{\n"); - for (auto &routine : fNeededBlasRoutines) { - if (routine == "Gemm") { - fGC += ("\textern \"C\" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k,\n" - "\t const float * alpha, const float * A, const int * lda, const float * B, const int * ldb,\n" - "\t const float * beta, float * C, const int * ldc);\n"); - // sgemm_ now declared; the standalone Gemm_Call helper will skip its copy. - fBlasSgemmDeclared = true; - } else if (routine == "Gemv") { - fGC += ("\textern \"C\" void sgemv_(const char * trans, const int * m, const int * n, const float * alpha, const float * A,\n" - "\t const int * lda, const float * X, const int * incx, const float * beta, const float * Y, const int * incy);\n"); - } else if (routine == "Axpy") { - fGC += ("\textern \"C\" void saxpy_(const int * n, const float * alpha, const float * x,\n" - "\t const int * incx, float * y, const int * incy);\n"); - } else if (routine == "Copy") { - fGC += ("\textern \"C\" void scopy_(const int *n, const float* x, const int *incx, float* y, const int* incy);\n"); - } - } - fGC += ("}//BLAS\n"); - } - // Placeholder for the standalone definitions of the inference helper - // functions used by this model (filled in by EmitHelperFunctionsCode). It - // sits inside the generated model namespace, right before the session code. - fGC += kHelperFunctionsMarker; -} - -void RModel_Base::EmitHelperFunctionsCode() -{ - HelperFunctionsCode code = - GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName, fBlasSgemmDeclared); - - auto replaceMarker = [this](const std::string &marker, const std::string &replacement) { - auto pos = fGC.find(marker); - if (pos != std::string::npos) { - fGC.replace(pos, marker.size(), replacement); - } - }; - - replaceMarker(kHelperIncludesMarker, code.includes); - replaceMarker(kHelperFunctionsMarker, code.definitions); - - // Clad derivatives live at file scope and reference the model's helpers, so - // insert them after the whole model namespace, just before the include guard. - if (!code.cladDefinitions.empty()) { - auto pos = fGC.rfind("#endif"); - if (pos != std::string::npos) { - fGC.insert(pos, code.cladDefinitions + "\n"); - } else { - fGC += code.cladDefinitions; - } - } -} - -void RModel_Base::OutputGenerated(std::string filename, bool append) { - // the model can be appended only if a file name is provided - if (filename.empty()) { - // if a file is pr - filename = fName + ".hxx"; - append = false; - } - std::ofstream f; - if (append) - f.open(filename, std::ios_base::app); - else - f.open(filename); - if (!f.is_open()) { - throw std::runtime_error("tmva-sofie failed to open file for output generated inference code"); - } - f << fGC; - f.close(); -} - -}//SOFIE -}//Experimental -}//TMVA diff --git a/tmva/sofie/src/SOFIE_common_helpers.cxx b/tmva/sofie/src/SOFIE_common_helpers.cxx index 74d9348e32ff8..7ff8098777e13 100644 --- a/tmva/sofie/src/SOFIE_common_helpers.cxx +++ b/tmva/sofie/src/SOFIE_common_helpers.cxx @@ -1,6 +1,6 @@ /// \file SOFIE_common_helpers.cxx /// Standalone definitions of the SOFIE inference helpers (Im2col, Gemm_Call, ...). -/// RModel records which helpers a model needs (RModel_Base::AddNeededHelperFunction) +/// RModel records which helpers a model needs (RModel::AddNeededHelperFunction) /// and dumps only those into the generated namespace, so the emitted header is /// self-contained (no TMVA/SOFIE_common.hxx include). These are dependency-free /// copies of the SOFIE_common.hxx originals: keep them in sync when those change. diff --git a/tmva/sofie/test/EmitFromONNX.cxx.in b/tmva/sofie/test/EmitFromONNX.cxx.in index 15bd82eb86e17..f296518aef4ac 100644 --- a/tmva/sofie/test/EmitFromONNX.cxx.in +++ b/tmva/sofie/test/EmitFromONNX.cxx.in @@ -5,7 +5,6 @@ // This program is automatically run when the target 'TestCustomModelsFromONNX' is built. // Usage example: $./sofiec indir/mymodel.onnx outdir/myname.hxx -#include "TMVA/RModel_Base.hxx" #include "TMVA/RModel.hxx" #include "TMVA/RModelParser_ONNX.hxx" diff --git a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.C b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.C index 433682ab858b6..015210ab442a0 100644 --- a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.C +++ b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.C @@ -18,15 +18,13 @@ /// \macro_output /// \author Lorenzo Moneta -using namespace TMVA::Experimental; - // need to add the current directory (from where we are running this macro) // to the include path for Cling R__ADD_INCLUDE_PATH($PWD) #include "HiggsModel.hxx" -#include "TMVA/SOFIEHelpers.hxx" -using namespace TMVA::Experimental; +#include +#include void TMVA_SOFIE_RDataFrame(int nthreads = 2){ @@ -38,14 +36,29 @@ void TMVA_SOFIE_RDataFrame(int nthreads = 2){ ROOT::RDataFrame df1("sig_tree", inputFile); int nslots = df1.GetNSlots(); std::cout << "Running using " << nslots << " threads" << std::endl; - auto h1 = df1.DefineSlot("DNN_Value", SofieFunctor<7, TMVA_SOFIE_HiggsModel::Session>(nslots), - {"m_jj", "m_jjj", "m_lv", "m_jlv", "m_bb", "m_wbb", "m_wwbb"}) + + // A SOFIE Session holds the model weights and the intermediate buffers and is + // not thread-safe: create one Session per RDataFrame processing slot and use + // the slot number in the DefineSlot functor to dispatch to the right one. + // The Session default constructor reads the weights from the default weight + // file (HiggsModel.dat in this case). + std::vector sessions(nslots); + + // The functor assembles the model input tensor from the RDataFrame columns + // and evaluates the model. The column order must match the ordering of the + // model input tensor. + auto evalModel = [&sessions](unsigned int slot, float m_jj, float m_jjj, float m_lv, float m_jlv, float m_bb, + float m_wbb, float m_wwbb) { + std::array input{m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb}; + auto result = sessions[slot].infer(input.data()); + return result[0]; + }; + + auto h1 = df1.DefineSlot("DNN_Value", evalModel, {"m_jj", "m_jjj", "m_lv", "m_jlv", "m_bb", "m_wbb", "m_wwbb"}) .Histo1D({"h_sig", "", 100, 0, 1}, "DNN_Value"); ROOT::RDataFrame df2("bkg_tree", inputFile); - nslots = df2.GetNSlots(); - auto h2 = df2.DefineSlot("DNN_Value", SofieFunctor<7, TMVA_SOFIE_HiggsModel::Session>(nslots), - {"m_jj", "m_jjj", "m_lv", "m_jlv", "m_bb", "m_wbb", "m_wwbb"}) + auto h2 = df2.DefineSlot("DNN_Value", evalModel, {"m_jj", "m_jjj", "m_lv", "m_jlv", "m_bb", "m_wbb", "m_wwbb"}) .Histo1D({"h_bkg", "", 100, 0, 1}, "DNN_Value"); h1->SetLineColor(kRed); diff --git a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.py b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.py index f700ce81cdf59..527127729fe6b 100644 --- a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.py +++ b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame.py @@ -31,17 +31,37 @@ model.PrintGenerated() # compile using ROOT JIT trained model -print("compiling SOFIE model and functor....") -ROOT.gInterpreter.Declare('#include "Higgs_trained_model_generated.hxx"') -ROOT.gInterpreter.Declare('auto sofie_functor = TMVA::Experimental::SofieFunctor<7,TMVA_SOFIE_'+modelName+'::Session>(0,"Higgs_trained_model_generated.dat");') +print("compiling SOFIE model and inference helper....") +ROOT.gInterpreter.Declare('#include "Higgs_trained_model_generated.hxx"\n#include \n#include ') + +# A SOFIE Session holds the model weights and the intermediate buffers and is not +# thread-safe: create one Session per RDataFrame processing slot and dispatch on +# the slot number. This tutorial runs single-threaded, so a single Session is enough. +# The weights file name is passed explicitly because the generated header was +# written under a custom name. +ROOT.gInterpreter.Declare( + 'std::vector sofie_sessions{TMVA_SOFIE_' + modelName + + '::Session("Higgs_trained_model_generated.dat")};') + +# Declare the inference function for RDataFrame: it assembles the model input +# tensor from the columns and evaluates the model. The column order must match +# the ordering of the model input tensor. +ROOT.gInterpreter.Declare(""" +double sofie_eval(unsigned int slot, float m_jj, float m_jjj, float m_lv, float m_jlv, float m_bb, float m_wbb, + float m_wwbb) +{ + std::array input{m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb}; + return sofie_sessions[slot].infer(input.data())[0]; +} +""") # run inference over input data inputFile = str(ROOT.gROOT.GetTutorialDir()) + "/machine_learning/data/Higgs_data.root" df1 = ROOT.RDataFrame("sig_tree", inputFile) -h1 = df1.Define("DNN_Value", "sofie_functor(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)").Histo1D(("h_sig", "", 100, 0, 1),"DNN_Value") +h1 = df1.Define("DNN_Value", "sofie_eval(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)").Histo1D(("h_sig", "", 100, 0, 1),"DNN_Value") df2 = ROOT.RDataFrame("bkg_tree", inputFile) -h2 = df2.Define("DNN_Value", "sofie_functor(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)").Histo1D(("h_bkg", "", 100, 0, 1),"DNN_Value") +h2 = df2.Define("DNN_Value", "sofie_eval(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)").Histo1D(("h_bkg", "", 100, 0, 1),"DNN_Value") # run over the input data once, combining both RDataFrame graphs. ROOT.RDF.RunGraphs([h1, h2]) diff --git a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame_JIT.C b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame_JIT.C index 97161ced30fb5..a01eb05bf87fe 100644 --- a/tutorials/machine_learning/TMVA_SOFIE_RDataFrame_JIT.C +++ b/tutorials/machine_learning/TMVA_SOFIE_RDataFrame_JIT.C @@ -14,22 +14,51 @@ /// \macro_output /// \author Lorenzo Moneta -using namespace TMVA::Experimental; - -/// function to compile the generated model and the declaration of the SofieFunctor -/// used by RDF. -/// Assume that the model name as in the header file -void CompileModelForRDF(const std::string & headerModelFile, unsigned int ninputs, unsigned int nslots=0) { +/// Function to compile the generated model with the ROOT JIT and to declare the +/// Session objects and the inference function used by RDataFrame. +/// A SOFIE Session holds the model weights and the intermediate buffers and is +/// not thread-safe: one Session per RDataFrame processing slot is created and +/// the slot number is used to dispatch to the right one. +/// Assume that the model name is the same as the header file name. +void CompileModelForRDF(const std::string &headerModelFile, unsigned int ninputs, unsigned int nslots = 0) +{ std::string modelName = headerModelFile.substr(0,headerModelFile.find(".hxx")); - std::string cmd = std::string("#include \"") + headerModelFile + std::string("\""); + std::string cmd = + std::string("#include \"") + headerModelFile + std::string("\"\n#include \n#include "); auto ret = gInterpreter->Declare(cmd.c_str()); if (!ret) throw std::runtime_error("Error compiling : " + cmd); std::cout << "compiled : " << cmd << std::endl; - cmd = "auto sofie_functor = TMVA::Experimental::SofieFunctor<" + std::to_string(ninputs) + ",TMVA_SOFIE_" + - modelName + "::Session>(" + std::to_string(nslots) + ");"; + // Declare one Session per processing slot. The Session default constructor + // reads the weights from the default weight file (.dat here). + if (nslots < 1) + nslots = 1; + cmd = "std::vector sofie_sessions(" + std::to_string(nslots) + ");"; + ret = gInterpreter->Declare(cmd.c_str()); + if (!ret) + throw std::runtime_error("Error compiling : " + cmd); + + // Declare the inference function for RDataFrame: it assembles the model + // input tensor from the columns and evaluates the model of the given slot. + std::string params; + std::string inputValues; + for (unsigned int i = 0; i < ninputs; i++) { + if (i > 0) { + params += ", "; + inputValues += ", "; + } + params += "float x" + std::to_string(i); + inputValues += "x" + std::to_string(i); + } + cmd = "double sofie_eval(unsigned int slot, " + params + + ") {\n" + " std::array input{" + inputValues + + "};\n" + " return sofie_sessions[slot].infer(input.data())[0];\n" + "}"; ret = gInterpreter->Declare(cmd.c_str()); if (!ret) throw std::runtime_error("Error compiling : " + cmd); @@ -61,13 +90,15 @@ void TMVA_SOFIE_RDataFrame_JIT(std::string modelName = "HiggsModel"){ std::string inputFileName = "Higgs_data.root"; std::string inputFile = std::string{gROOT->GetTutorialDir()} + "/machine_learning/data/" + inputFileName; + // The column order in the Define expressions must match the ordering of the + // model input tensor. ROOT::RDataFrame df1("sig_tree", inputFile); - auto h1 = df1.Define("DNN_Value", "sofie_functor(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)") - .Histo1D({"h_sig", "", 100, 0, 1},"DNN_Value"); + auto h1 = df1.Define("DNN_Value", "sofie_eval(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)") + .Histo1D({"h_sig", "", 100, 0, 1}, "DNN_Value"); ROOT::RDataFrame df2("bkg_tree", inputFile); - auto h2 = df2.Define("DNN_Value", "sofie_functor(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)") - .Histo1D({"h_bkg", "", 100, 0, 1},"DNN_Value"); + auto h2 = df2.Define("DNN_Value", "sofie_eval(rdfslot_,m_jj, m_jjj, m_lv, m_jlv, m_bb, m_wbb, m_wwbb)") + .Histo1D({"h_bkg", "", 100, 0, 1}, "DNN_Value"); h1->SetLineColor(kRed); h2->SetLineColor(kBlue);