diff --git a/tree/ntuple/inc/ROOT/RField/RFieldRecord.hxx b/tree/ntuple/inc/ROOT/RField/RFieldRecord.hxx index 33efe55e77090..eacb96b43868e 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldRecord.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldRecord.hxx @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -37,9 +38,11 @@ class RFieldVisitor; } // namespace Detail namespace Internal { +// TODO(gparolini): consider using a different type than RFieldDescriptor to pass around type name/version/checksum +// to CreateEmulated* functions std::unique_ptr CreateEmulatedRecordField(std::string_view fieldName, std::vector> itemFields, - std::string_view emulatedFromType); + const RFieldDescriptor &emulatedFrom); // Used by to late-model-extend fields to untyped records void AddItemToRecord(RRecordField &record, std::unique_ptr newItem); } // namespace Internal @@ -49,7 +52,7 @@ void AddItemToRecord(RRecordField &record, std::unique_ptr newItem); class RRecordField : public RFieldBase { friend std::unique_ptr Internal::CreateEmulatedRecordField(std::string_view fieldName, std::vector> itemFields, - std::string_view emulatedFromType); + const RFieldDescriptor &emulatedFrom); friend void Internal::AddItemToRecord(RRecordField &record, std::unique_ptr newItem); class RRecordDeleter : public RDeleter { @@ -67,14 +70,17 @@ class RRecordField : public RFieldBase { }; std::unordered_set fSubfieldNames; ///< Efficient detection of duplicate field names + // These are only valid if the record field is emulated (i.e. (fTraits & kTraitEmulatedField) != 0). + std::uint32_t fEmulatedTypeVersion = 0; + std::uint32_t fEmulatedTypeChecksum = 0; RRecordField(std::string_view name, const RRecordField &source); // Used by CloneImpl() - /// If `emulatedFromType` is non-empty, this field was created as a replacement for a ClassField that we lack a + /// If `emulatedFrom` is non-null, this field was created as a replacement for a ClassField that we lack a /// dictionary for and reconstructed from the on-disk information. /// Used by the public constructor and by Internal::CreateEmulatedRecordField(). RRecordField(std::string_view fieldName, std::vector> itemFields, - std::string_view emulatedFromType); + const RFieldDescriptor *emulatedFrom); bool IsPairOrTuple() const { @@ -148,6 +154,9 @@ public: void AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const final; const std::vector &GetOffsets() const { return fOffsets; } + + std::uint32_t GetTypeVersion() const final { return (fTraits & kTraitEmulatedField) ? fEmulatedTypeVersion : 0; } + std::uint32_t GetTypeChecksum() const final { return (fTraits & kTraitEmulatedField) ? fEmulatedTypeChecksum : 0; } }; //////////////////////////////////////////////////////////////////////////////// diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx index 82f685bde14e7..7c750e719300f 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx @@ -37,7 +37,7 @@ class RSoAField; namespace Internal { std::unique_ptr CreateEmulatedVectorField(std::string_view fieldName, std::unique_ptr itemField, - std::string_view emulatedFromType); + const RFieldDescriptor &emulatedFrom); } //////////////////////////////////////////////////////////////////////////////// @@ -214,7 +214,7 @@ class RVectorField : public RFieldBase { friend class RArrayAsVectorField; // to get access to the RVectorDeleter friend std::unique_ptr Internal::CreateEmulatedVectorField(std::string_view fieldName, std::unique_ptr itemField, - std::string_view emulatedFromType); + const RFieldDescriptor &emulatedFrom); class RVectorDeleter : public RDeleter { private: @@ -231,6 +231,9 @@ class RVectorField : public RFieldBase { std::size_t fItemSize; ROOT::Internal::RColumnIndex fNWritten; std::unique_ptr fItemDeleter; + // These are only valid if the record field is emulated (i.e. (fTraits & kTraitEmulatedField) != 0). + std::uint32_t fEmulatedTypeVersion = 0; + std::uint32_t fEmulatedTypeChecksum = 0; // Ensures that the std::vector pointed to by vec has at least nItems valid elements. static void ResizeVector(void *vec, std::size_t nItems, std::size_t itemSize, const RFieldBase &itemField, @@ -238,10 +241,10 @@ class RVectorField : public RFieldBase { protected: /// Creates a possibly-untyped VectorField. - /// If `emulatedFromType` is not nullopt, the field is untyped. If the string is empty, it is a "regular" + /// If `emulatedFrom` is not null, the field is untyped. If emulatedFrom->GetTypeName() is empty, it is a "regular" /// untyped vector field; otherwise, it was created as an emulated field from the given type name. RVectorField(std::string_view fieldName, std::unique_ptr itemField, - std::optional emulatedFromType); + const ROOT::RFieldDescriptor *emulatedFrom); std::unique_ptr CloneImpl(std::string_view newName) const final; diff --git a/tree/ntuple/src/RField.cxx b/tree/ntuple/src/RField.cxx index 5bf1c9ac84f97..dbdb4ad86fe14 100644 --- a/tree/ntuple/src/RField.cxx +++ b/tree/ntuple/src/RField.cxx @@ -589,30 +589,36 @@ ROOT::RRecordField::RRecordField(std::string_view fieldName, std::string_view ty std::unique_ptr ROOT::Internal::CreateEmulatedRecordField(std::string_view fieldName, std::vector> itemFields, - std::string_view emulatedFromType) + const RFieldDescriptor &emulatedFrom) { - R__ASSERT(!emulatedFromType.empty()); - return std::unique_ptr(new RRecordField(fieldName, std::move(itemFields), emulatedFromType)); + R__ASSERT(!emulatedFrom.GetTypeName().empty()); + return std::unique_ptr(new RRecordField(fieldName, std::move(itemFields), &emulatedFrom)); } std::unique_ptr ROOT::Internal::CreateEmulatedVectorField(std::string_view fieldName, std::unique_ptr itemField, - std::string_view emulatedFromType) + const RFieldDescriptor &emulatedFrom) { - R__ASSERT(!emulatedFromType.empty()); - return std::unique_ptr(new RVectorField(fieldName, std::move(itemField), emulatedFromType)); + R__ASSERT(!emulatedFrom.GetTypeName().empty()); + return std::unique_ptr(new RVectorField(fieldName, std::move(itemField), &emulatedFrom)); } ROOT::RRecordField::RRecordField(std::string_view fieldName, std::vector> itemFields, - std::string_view emulatedFromType) - : ROOT::RFieldBase(fieldName, emulatedFromType, ROOT::ENTupleStructure::kRecord, false /* isSimple */) + const ROOT::RFieldDescriptor *emulatedFrom) + : ROOT::RFieldBase(fieldName, emulatedFrom ? emulatedFrom->GetTypeName() : "", ROOT::ENTupleStructure::kRecord, + false /* isSimple */) { AttachItemFields(std::move(itemFields)); - fTraits |= !emulatedFromType.empty() * kTraitEmulatedField; + fTraits |= !!emulatedFrom * kTraitEmulatedField; + if (emulatedFrom) { + fTraits |= emulatedFrom->GetTypeChecksum().has_value() * kTraitTypeChecksum; + fEmulatedTypeVersion = emulatedFrom->GetTypeVersion(); + fEmulatedTypeChecksum = emulatedFrom->GetTypeChecksum().value_or(0); + } } ROOT::RRecordField::RRecordField(std::string_view fieldName, std::vector> itemFields) - : ROOT::RRecordField(fieldName, std::move(itemFields), "") + : ROOT::RRecordField(fieldName, std::move(itemFields), nullptr) { fTraits |= kTraitExtensible; } diff --git a/tree/ntuple/src/RFieldBase.cxx b/tree/ntuple/src/RFieldBase.cxx index 4762beeeceef8..8c328c2566ab6 100644 --- a/tree/ntuple/src/RFieldBase.cxx +++ b/tree/ntuple/src/RFieldBase.cxx @@ -541,8 +541,7 @@ ROOT::RFieldBase::Create(const std::string &fieldName, const std::string &typeNa memberFields.emplace_back(std::move(field)); } R__ASSERT(typeName == fieldDesc.GetTypeName()); - auto recordField = - Internal::CreateEmulatedRecordField(fieldName, std::move(memberFields), fieldDesc.GetTypeName()); + auto recordField = Internal::CreateEmulatedRecordField(fieldName, std::move(memberFields), fieldDesc); recordField->fTypeAlias = fieldDesc.GetTypeAlias(); return recordField; } else if (fieldDesc.GetStructure() == ENTupleStructure::kCollection) { @@ -553,8 +552,7 @@ ROOT::RFieldBase::Create(const std::string &fieldName, const std::string &typeNa const auto &itemFieldDesc = desc->GetFieldDescriptor(itemFieldId); auto itemField = Create(itemFieldDesc.GetFieldName(), itemFieldDesc.GetTypeName(), options, desc, itemFieldId).Unwrap(); - auto vecField = - ROOT::Internal::CreateEmulatedVectorField(fieldName, std::move(itemField), fieldDesc.GetTypeName()); + auto vecField = ROOT::Internal::CreateEmulatedVectorField(fieldName, std::move(itemField), fieldDesc); vecField->fTypeAlias = fieldDesc.GetTypeAlias(); return vecField; } else if (ROOT::Internal::IsCustomEnumFieldDesc(*desc, fieldDesc)) { diff --git a/tree/ntuple/src/RFieldSequenceContainer.cxx b/tree/ntuple/src/RFieldSequenceContainer.cxx index b417149f325f1..709ee762e6d06 100644 --- a/tree/ntuple/src/RFieldSequenceContainer.cxx +++ b/tree/ntuple/src/RFieldSequenceContainer.cxx @@ -529,8 +529,9 @@ void ROOT::RRVecField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const //------------------------------------------------------------------------------ ROOT::RVectorField::RVectorField(std::string_view fieldName, std::unique_ptr itemField, - std::optional emulatedFromType) - : ROOT::RFieldBase(fieldName, emulatedFromType ? *emulatedFromType : "std::vector<" + itemField->GetTypeName() + ">", + const ROOT::RFieldDescriptor *emulatedFrom) + : ROOT::RFieldBase(fieldName, + emulatedFrom ? emulatedFrom->GetTypeName() : "std::vector<" + itemField->GetTypeName() + ">", ROOT::ENTupleStructure::kCollection, false /* isSimple */), fItemSize(itemField->GetValueSize()), fNWritten(0) @@ -540,8 +541,11 @@ ROOT::RVectorField::RVectorField(std::string_view fieldName, std::unique_ptrGetAlignment()))); } - if (emulatedFromType && !emulatedFromType->empty()) + if (emulatedFrom && !emulatedFrom->GetTypeName().empty()) { fTraits |= kTraitEmulatedField; + fEmulatedTypeVersion = emulatedFrom->GetTypeVersion(); + fEmulatedTypeChecksum = emulatedFrom->GetTypeChecksum().value_or(0); + } if (!itemField->GetTypeAlias().empty()) fTypeAlias = "std::vector<" + itemField->GetTypeAlias() + ">"; @@ -552,22 +556,36 @@ ROOT::RVectorField::RVectorField(std::string_view fieldName, std::unique_ptr itemField) - : RVectorField(fieldName, std::move(itemField), {}) + : RVectorField(fieldName, std::move(itemField), nullptr) { } std::unique_ptr ROOT::RVectorField::CreateUntyped(std::string_view fieldName, std::unique_ptr itemField) { - return std::unique_ptr(new RVectorField(fieldName, itemField->Clone("_0"), "")); + ROOT::RFieldDescriptor desc; + return std::unique_ptr(new RVectorField(fieldName, itemField->Clone("_0"), &desc)); } std::unique_ptr ROOT::RVectorField::CloneImpl(std::string_view newName) const { auto newItemField = fSubfields[0]->Clone(fSubfields[0]->GetFieldName()); auto isUntyped = GetTypeName().empty() || ((fTraits & kTraitEmulatedField) != 0); - auto emulatedFromType = isUntyped ? std::make_optional(GetTypeName()) : std::nullopt; - return std::unique_ptr(new RVectorField(newName, std::move(newItemField), emulatedFromType)); + ROOT::RFieldDescriptor desc; + if (isUntyped) { + ROOT::Internal::RFieldDescriptorBuilder builder; + desc = builder.TypeVersion(GetTypeVersion()) + .TypeChecksum(GetTypeChecksum()) + .TypeName(GetTypeName()) + // we need an Id for MoveDescriptor to succeed, but we're not going to use it. + .FieldId(42) + // we also need a Structure for the same reason. + .Structure(ENTupleStructure::kCollection) + .MoveDescriptor() + .Unwrap(); + } + return std::unique_ptr( + new RVectorField(newName, std::move(newItemField), isUntyped ? &desc : nullptr)); } std::size_t ROOT::RVectorField::AppendImpl(const void *from) diff --git a/tree/ntuple/src/RNTupleMerger.cxx b/tree/ntuple/src/RNTupleMerger.cxx index e666513c8dbf6..95cae963295ea 100644 --- a/tree/ntuple/src/RNTupleMerger.cxx +++ b/tree/ntuple/src/RNTupleMerger.cxx @@ -765,7 +765,9 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM if (fieldDesc->IsProjectedField()) continue; - auto field = fieldDesc->CreateField(srcDesc); + ROOT::RCreateFieldOptions cfOpts; + cfOpts.SetEmulateUnknownTypes(true); + auto field = fieldDesc->CreateField(srcDesc, cfOpts); // Explicitly set the field representatives. This prevents UpdateSchema() from changing our column // representations via AutoAdjustColumnTypes. ROOT::RFieldBase::ColumnRepresentation_t representatives; @@ -787,7 +789,9 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM continue; ROOT::Internal::RProjectedFields::FieldMap_t fieldMap; - auto field = fieldDesc->CreateField(srcDesc); + ROOT::RCreateFieldOptions cfOpts; + cfOpts.SetEmulateUnknownTypes(true); + auto field = fieldDesc->CreateField(srcDesc, cfOpts); const auto sourceId = fieldDesc->GetProjectionSourceId(); const auto &sourceField = dstModel.GetConstField(srcDesc.GetQualifiedFieldName(sourceId)); fieldMap[field.get()] = &sourceField; diff --git a/tree/ntuple/test/CMakeLists.txt b/tree/ntuple/test/CMakeLists.txt index 74d1ac78db36a..198fd39954f12 100644 --- a/tree/ntuple/test/CMakeLists.txt +++ b/tree/ntuple/test/CMakeLists.txt @@ -44,6 +44,7 @@ if(NOT MSVC) ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_evolution_soa ntuple_evolution_soa.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_emulated ntuple_emulated.cxx LIBRARIES ROOTNTuple) + ROOT_ADD_GTEST(ntuple_merge_emulated ntuple_merge_emulated.cxx ntuple_test.cxx LIBRARIES ROOTNTuple xxHash::xxHash) endif() ROOT_ADD_GTEST(ntuple_field_name ntuple_field_name.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_join_table ntuple_join_table.cxx LIBRARIES ROOTNTuple) diff --git a/tree/ntuple/test/ntuple_emulated.cxx b/tree/ntuple/test/ntuple_emulated.cxx index 8d4b1a1f1d019..4a2cbd7416d2c 100644 --- a/tree/ntuple/test/ntuple_emulated.cxx +++ b/tree/ntuple/test/ntuple_emulated.cxx @@ -1015,88 +1015,3 @@ TEST(RNTupleEmulated, EmulatedFields_SoA) EXPECT_EQ(13, viewX(0)); EXPECT_EQ(17, viewX(1)); } - -TEST(RNTupleEmulated, MergeEmulated) -{ - FileRaii fileGuard1("test_ntuple_merge_emulated1.root"); - FileRaii fileGuard2("test_ntuple_merge_emulated2.root"); - FileRaii fileGuardOut("test_ntuple_merge_emulated_out.root"); - - EXPECT_NO_STREAMER_OR_DICTIONARY(); - - ExecInFork([&] { - fileGuard1.PreserveFile(); - fileGuard2.PreserveFile(); - - ASSERT_TRUE(gInterpreter->Declare(R"( - struct Custom { - int fInt1 = 1; - int fInt2 = 2; - ClassDefNV(Custom, 2); - }; - )")); - - auto model = RNTupleModel::Create(); - model->AddField(RFieldBase::Create("f", "Custom").Unwrap()); - - auto writer = RNTupleWriter::Recreate(model->Clone(), "ntuple", fileGuard1.GetPath()); - auto writer2 = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); - - void *ptr = writer->GetModel().GetDefaultEntry().GetPtr("f").get(); - void *ptr2 = writer2->GetModel().GetDefaultEntry().GetPtr("f").get(); - DeclarePointer("Custom", "ptr", ptr); - DeclarePointer("Custom", "ptr2", ptr2); - - ProcessLine("ptr->fInt1 = 11;"); - ProcessLine("ptr->fInt2 = 22;"); - writer->Fill(); - - ProcessLine("ptr2->fInt1 = 22;"); - ProcessLine("ptr2->fInt2 = 44;"); - writer2->Fill(); - ProcessLine("ptr2->fInt1 = 33;"); - ProcessLine("ptr2->fInt2 = 66;"); - writer2->Fill(); - - writer.reset(); - writer2.reset(); - }); - - // Gather the input sources - std::vector> sources; - sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); - sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); - std::vector sourcePtrs; - for (const auto &s : sources) { - sourcePtrs.push_back(s.get()); - } - - { - auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), RNTupleWriteOptions()); - RNTupleMerger merger{std::move(destination)}; - auto res = merger.Merge(sourcePtrs); - ASSERT_TRUE(bool(res)); - } - - { - auto reader = RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); - try { - reader->GetModel(); - FAIL() << "reading back the merged file without fEmulateUnknownTypes should fail"; - } catch (const ROOT::RException &ex) { - ASSERT_THAT(ex.GetError().GetReport(), testing::HasSubstr("unknown type")); - } - } - - auto opts = RNTupleDescriptor::RCreateModelOptions(); - opts.SetEmulateUnknownTypes(true); - auto reader = RNTupleReader::Open(opts, "ntuple", fileGuardOut.GetPath()); - EXPECT_EQ(reader->GetNEntries(), 3); - - auto vInt1 = reader->GetView("f.fInt1"); - auto vInt2 = reader->GetView("f.fInt2"); - for (auto idx : reader->GetEntryRange()) { - EXPECT_EQ(vInt1(idx), 11 * (idx + 1)); - EXPECT_EQ(vInt2(idx), 22 * (idx + 1)); - } -} diff --git a/tree/ntuple/test/ntuple_merge_emulated.cxx b/tree/ntuple/test/ntuple_merge_emulated.cxx new file mode 100644 index 0000000000000..fad334a955c28 --- /dev/null +++ b/tree/ntuple/test/ntuple_merge_emulated.cxx @@ -0,0 +1,277 @@ +#include "ntuple_test.hxx" +#include "ntuple_fork.hxx" + +static void DoMerge(std::initializer_list> inputPaths, const FileRaii &outPath, + ENTupleMergingMode mergeMode = {}) +{ + std::vector> sources; + for (const auto &path : inputPaths) + sources.push_back(RPageSource::Create("ntuple", path.get().GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) + sourcePtrs.push_back(s.get()); + + auto destination = std::make_unique("ntuple", outPath.GetPath(), RNTupleWriteOptions()); + RNTupleMerger merger{std::move(destination)}; + auto mergeOpts = RNTupleMergeOptions(); + mergeOpts.fMergingMode = mergeMode; + auto res = merger.Merge(sourcePtrs, mergeOpts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); +} + +template +static std::ostream &operator<<(std::ostream &s, const std::optional &o) +{ + if (o) + s << *o; + else + s << "(nullopt)"; + return s; +} + +static void CheckFieldDescriptorsAreEquivalent(const ROOT::RFieldDescriptor &a, const ROOT::RFieldDescriptor &b) +{ + std::stringstream errs; + +#define CompareDescField(What) \ + if (a.What() != b.What()) \ + errs << "\n - " #What "() differs: " << a.What() << " vs " << b.What() + + CompareDescField(GetFieldVersion); + CompareDescField(GetTypeVersion); + CompareDescField(GetFieldName); + CompareDescField(GetFieldDescription); + CompareDescField(GetTypeName); + CompareDescField(GetTypeAlias); + CompareDescField(GetNRepetitions); + CompareDescField(GetStructure); + CompareDescField(GetLinkIds().size); + CompareDescField(GetLogicalColumnIds().size); + CompareDescField(GetTypeChecksum); + CompareDescField(IsSoACollection); + +#undef CompareDescField + + const auto errStr = errs.str(); + if (!errStr.empty()) { + FAIL() << "field descriptors for '" << a.GetFieldName() << "' are different:" << errStr; + } +} + +static void CompareOnDiskInfo(const FileRaii &withDict, const FileRaii &withoutDict) +{ + auto opts = RNTupleDescriptor::RCreateModelOptions(); + opts.SetEmulateUnknownTypes(true); + auto reader1 = RNTupleReader::Open(opts, "ntuple", withDict.GetPath()); + auto reader2 = RNTupleReader::Open(opts, "ntuple", withoutDict.GetPath()); + const auto &desc1 = reader1->GetDescriptor(); + const auto &desc2 = reader2->GetDescriptor(); + + EXPECT_EQ(reader1->GetNEntries(), reader2->GetNEntries()); + + // Verify that all fields in desc1 are also there in desc2 and they look the same. + std::unordered_set fieldsVisited; + std::vector toVisit; + toVisit.push_back(desc1.GetFieldZeroId()); + while (!toVisit.empty()) { + auto rootId = toVisit.back(); + toVisit.pop_back(); + for (const auto &fdesc1 : desc1.GetFieldIterable(rootId)) { + const auto fqName = desc1.GetQualifiedFieldName(fdesc1.GetId()); + const auto &fdescId2 = desc2.FindFieldId(fqName); + ASSERT_NE(fdescId2, ROOT::kInvalidDescriptorId); + const auto &fdesc2 = desc2.GetFieldDescriptor(fdescId2); + CheckFieldDescriptorsAreEquivalent(fdesc1, fdesc2); + + fieldsVisited.emplace(fqName); + toVisit.emplace_back(fdesc1.GetId()); + } + } + + // Make sure we desc2 doesn't contain any extra field + toVisit.clear(); + toVisit.push_back(desc2.GetFieldZeroId()); + while (!toVisit.empty()) { + auto rootId = toVisit.back(); + toVisit.pop_back(); + for (const auto &fdesc2 : desc2.GetFieldIterable(rootId)) { + const auto fqName = desc2.GetQualifiedFieldName(fdesc2.GetId()); + EXPECT_EQ(fieldsVisited.count(fqName), 1); + toVisit.emplace_back(fdesc2.GetId()); + } + } +} + +TEST(RNTupleMergeEmulated, MergeEmulatedSimple) +{ + // Merge two files containing a simple class field with no dictionary + FileRaii fileGuard1("test_ntuple_merge_emulated1.root"); + FileRaii fileGuard2("test_ntuple_merge_emulated2.root"); + FileRaii fileGuardOut("test_ntuple_merge_emulated_out.root"); + + EXPECT_NO_STREAMER_OR_DICTIONARY(); + + ExecInFork([&] { + fileGuard1.PreserveFile(); + fileGuard2.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( + struct Custom { + int fInt1 = 1; + int fInt2 = 2; + ClassDefNV(Custom, 2); + }; + )")); + + auto model = RNTupleModel::Create(); + model->AddField(RFieldBase::Create("f", "Custom").Unwrap()); + + auto writer = RNTupleWriter::Recreate(model->Clone(), "ntuple", fileGuard1.GetPath()); + auto writer2 = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + + void *ptr = writer->GetModel().GetDefaultEntry().GetPtr("f").get(); + void *ptr2 = writer2->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("Custom", "ptr", ptr); + DeclarePointer("Custom", "ptr2", ptr2); + + ProcessLine("ptr->fInt1 = 11;"); + ProcessLine("ptr->fInt2 = 22;"); + writer->Fill(); + + ProcessLine("ptr2->fInt1 = 22;"); + ProcessLine("ptr2->fInt2 = 44;"); + writer2->Fill(); + ProcessLine("ptr2->fInt1 = 33;"); + ProcessLine("ptr2->fInt2 = 66;"); + writer2->Fill(); + + writer.reset(); + writer2.reset(); + }); + + DoMerge({fileGuard1, fileGuard2}, fileGuardOut); + + { + auto reader = RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + try { + reader->GetModel(); + FAIL() << "reading back the merged file without fEmulateUnknownTypes should fail"; + } catch (const ROOT::RException &ex) { + ASSERT_THAT(ex.GetError().GetReport(), testing::HasSubstr("unknown type")); + } + } + + auto opts = RNTupleDescriptor::RCreateModelOptions(); + opts.SetEmulateUnknownTypes(true); + auto reader = RNTupleReader::Open(opts, "ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 3); + + auto vInt1 = reader->GetView("f.fInt1"); + auto vInt2 = reader->GetView("f.fInt2"); + for (auto idx : reader->GetEntryRange()) { + EXPECT_EQ(vInt1(idx), 11 * (idx + 1)); + EXPECT_EQ(vInt2(idx), 22 * (idx + 1)); + } +} + +TEST(RNTupleMergeEmulated, MergeLateEmulated) +{ + // Union-merge two files, the first containing an empty ntuple and the second a simple class field with no dictionary + FileRaii fileGuard1("test_ntuple_merge_late_emulated1.root"); + FileRaii fileGuard2("test_ntuple_merge_late_emulated2.root"); + FileRaii fileGuardOut("test_ntuple_merge_late_emulated_out.root"); + FileRaii fileGuardOut2("test_ntuple_merge_late_emulated_out2.root"); + + EXPECT_NO_STREAMER_OR_DICTIONARY(); + + ExecInFork([&] { + fileGuard1.PreserveFile(); + fileGuard2.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( + struct Custom { + int fInt1 = 1; + ClassDefNV(Custom, 2); + }; + )")); + + { + auto writer = RNTupleWriter::Recreate(ROOT::RNTupleModel::Create(), "ntuple", fileGuard1.GetPath()); + writer->Fill(); + writer->Fill(); + writer->Fill(); + } + + auto model = RNTupleModel::Create(); + model->AddField(RFieldBase::Create("f", "Custom").Unwrap()); + auto writer2 = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + + void *ptr2 = writer2->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("Custom", "ptr2", ptr2); + + ProcessLine("ptr2->fInt1 = 11;"); + writer2->Fill(); + + ProcessLine("ptr2->fInt1 = 22;"); + writer2->Fill(); + + writer2.reset(); + + DoMerge({fileGuard1, fileGuard2}, fileGuardOut, ENTupleMergingMode::kUnion); + }); + + // Gather the input sources + DoMerge({fileGuard1, fileGuard2}, fileGuardOut2, ENTupleMergingMode::kUnion); + CompareOnDiskInfo(fileGuardOut, fileGuardOut2); +} + +TEST(RNTupleMergeEmulated, MergeLateEmulatedVector) +{ + // Union-merge two files, the first containing an empty ntuple and the second a simple class field with no dictionary + FileRaii fileGuard1("test_ntuple_merge_late_emulated_vec1.root"); + FileRaii fileGuard2("test_ntuple_merge_late_emulated_vec2.root"); + FileRaii fileGuardOut("test_ntuple_merge_late_emulated_out.root"); + FileRaii fileGuardOut2("test_ntuple_merge_late_emulated_out2.root"); + + EXPECT_NO_STREAMER_OR_DICTIONARY(); + + ExecInFork([&] { + fileGuard1.PreserveFile(); + fileGuard2.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( + struct Custom { + int fInt1 = 1; + ClassDefNV(Custom, 2); + }; + )")); + + { + auto writer = RNTupleWriter::Recreate(ROOT::RNTupleModel::Create(), "ntuple", fileGuard1.GetPath()); + writer->Fill(); + writer->Fill(); + writer->Fill(); + } + + auto model = RNTupleModel::Create(); + model->AddField(RFieldBase::Create("f", "std::vector").Unwrap()); + auto writer2 = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + + void *ptr2 = writer2->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("std::vector", "ptr2", ptr2); + + ProcessLine("ptr2->push_back(Custom{11});"); + writer2->Fill(); + + ProcessLine("ptr2->push_back(Custom{22});"); + writer2->Fill(); + + writer2.reset(); + + DoMerge({fileGuard1, fileGuard2}, fileGuardOut, ENTupleMergingMode::kUnion); + }); + + // Gather the input sources + DoMerge({fileGuard1, fileGuard2}, fileGuardOut2, ENTupleMergingMode::kUnion); + CompareOnDiskInfo(fileGuardOut, fileGuardOut2); +}