From ba6a858f7572f6ae33c0e0d128e08c10385caff0 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 5 Feb 2026 19:28:27 +0100 Subject: [PATCH 01/37] io: Clarify byte count related limits --- io/io/src/TBufferFile.cxx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index dd71715942c1d..d7987ee00e144 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -3159,7 +3159,8 @@ Version_t TBufferFile::ReadVersion(UInt_t *startpos, UInt_t *bcnt, const TClass // before reading object save start position auto full_startpos = fBufCur - fBuffer; *startpos = full_startpos <= kMaxCountPosition ? UInt_t(full_startpos) : kOverflowPosition; - fByteCountStack.push_back({(size_t)full_startpos, cl, nullptr}); + if (bcnt) + fByteCountStack.push_back({(size_t)full_startpos, cl, nullptr}); } // read byte count (older files don't have byte count) @@ -3196,7 +3197,8 @@ Version_t TBufferFile::ReadVersion(UInt_t *startpos, UInt_t *bcnt, const TClass // did not fit and thus we stored it in 'fByteCounts' instead. // Mark this case by setting startpos to kOverflowCount. *bcnt = kOverflowCount; - *startpos = kOverflowPosition; + if (startpos) + *startpos = kOverflowPosition; } } } @@ -3302,7 +3304,8 @@ Version_t TBufferFile::ReadVersionNoCheckSum(UInt_t *startpos, UInt_t *bcnt) auto full_startpos = fBufCur - fBuffer; *startpos = full_startpos < kMaxCountPosition ? UInt_t(full_startpos) : kOverflowPosition; // TODO: Extend ReadVersionNoCheckSum to take the class pointer. - fByteCountStack.push_back({(size_t)full_startpos, nullptr, nullptr}); + if (bcnt) + fByteCountStack.push_back({(size_t)full_startpos, nullptr, nullptr}); } // read byte count (older files don't have byte count) @@ -3338,7 +3341,8 @@ Version_t TBufferFile::ReadVersionNoCheckSum(UInt_t *startpos, UInt_t *bcnt) // did not fit and thus we stored it in 'fByteCounts' instead. // Mark this case by setting startpos to kOverflowCount. *bcnt = kOverflowCount; - *startpos = kOverflowPosition; + if (startpos) + *startpos = kOverflowPosition; } } } From 61cee324afc23aab38b2a6073e7f68710874f988 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 13:26:29 -0500 Subject: [PATCH 02/37] io: Improve ByteCountStack assert Support the case where the class information is passed by name rather than by TClass pointer (eg. the action ReadCollectionDouble32) --- io/io/src/TBufferFile.cxx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index d7987ee00e144..d0b8d9bd0d002 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -440,7 +440,10 @@ Long64_t TBufferFile::CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const T // The position is above 4GB but was cached using a 32 bit variable. startpos = fByteCountStack.back().locator; // See below - R__ASSERT((fByteCountStack.back().cl == nullptr || clss == fByteCountStack.back().cl) + R__ASSERT((fByteCountStack.back().cl == nullptr || + (clss == nullptr && (classname == nullptr || classname[0] == '\0')) || + (clss && clss == fByteCountStack.back().cl) || + (classname && strcmp(classname, fByteCountStack.back().cl->GetName()) == 0)) && "Class on the byte count position stack does not match the passed class"); } else { // This assert allows to reject cases that used to be valid (missing or From 687eecc2193fc0612915cc1d1036676c62c73f47 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 17:07:37 -0500 Subject: [PATCH 03/37] io: TBufferClass fix handling of large bytecount. In TBufferFile::ReadClass properly set the byte count returned to kOverflowCount when the byte count in the stream was not updated (was too large). --- io/io/src/TBufferFile.cxx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index d0b8d9bd0d002..4fe776f2905cd 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -3018,6 +3018,14 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) // return bytecount in objTag if (objTag) { *objTag = (bcnt & ~kByteCountMask); + if (*objTag == 0) { + // The byte count was stored but is zero, this means the data + // did not fit and thus we stored it in 'fByteCounts' instead. + // Mark this case by setting startpos to kOverflowCount. + *objTag = kOverflowCount; + // We do not have access to the caller's "cntpos" and can not + // update to kOverflowPosition (unlike the same code in ReadVersion). + } if (cl) fByteCountStack.back().alt = cl; } From c5d94e363ebe46dc50eb3ab9eabeb7c6bf65f26f Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 16 Feb 2026 12:58:57 -0600 Subject: [PATCH 04/37] io: Add more debugging for ByteCountStack --- io/io/src/TBufferFile.cxx | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 4fe776f2905cd..01b74a6993477 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -2661,6 +2661,8 @@ void TBufferFile::SkipObjectAny(Long64_t start, UInt_t count) /// real beginning of the object in memory. You will need to use a /// dynamic_cast later if you need to retrieve it. + + void *TBufferFile::ReadObjectAny(const TClass *clCast) { R__ASSERT(IsReading()); @@ -2668,6 +2670,20 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) // make sure fMap is initialized InitMap(); + struct CaptureAndCheck { + using ByteCountStack_t = TBufferFile::ByteCountStack_t; + size_t fCurrent; + ByteCountStack_t *fByteCountStack; + + CaptureAndCheck(ByteCountStack_t *stack) : fCurrent(stack->size()), fByteCountStack(stack) {} + ~CaptureAndCheck() { + if (fCurrent != fByteCountStack->size()) { + ::Fatal("CaptureAndCheck", "Byte count stack was not properly cleaned up. Current size is %zu but should be %zu", fByteCountStack->size(), fCurrent); + } + } + }; + CaptureAndCheck checker(&(this->fByteCountStack)); + // before reading object save start position ULong64_t startpos = static_cast(fBufCur-fBuffer); ULong64_t cntpos = startpos <= kMaxCountPosition ? startpos : kOverflowPosition; @@ -3862,6 +3878,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass else version = ReadVersion(&R__s, &R__c, cl); + size_t current = fByteCountStack.size(); Bool_t v2file = kFALSE; TFile *file = (TFile*)GetParent(); if (file && file->GetVersion() < 30000) { @@ -3880,6 +3897,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass Error("ReadClassBuffer", "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", onFileClass->GetName(), version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, onFileClass); return 0; } @@ -3904,6 +3922,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (version < -1 || version >= infocapacity) { Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %d", cl->GetName(), version, Length()); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3962,11 +3981,13 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass // When the object was written the class was version zero, so // there is no StreamerInfo to be found. // Check that the buffer position corresponds to the byte count. + --current; CheckByteCount(R__s, R__c, cl); return 0; } else { Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3978,6 +3999,9 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass ApplySequence(*(sinfo->GetReadObjectWiseActions()), (char*)pointer ); if (sinfo->TStreamerInfo::IsRecovered()) R__c=0; // 'TStreamerInfo::' avoids going via a virtual function. + if (current != fByteCountStack.size()) + Fatal("ReadClassBuffer", "We are out of sync at level %lld vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); + --current; // Check that the buffer position corresponds to the byte count. CheckByteCount(R__s, R__c, cl); @@ -4046,8 +4070,11 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter).PrintDebug(*this,obj); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); } } else { @@ -4056,7 +4083,10 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); } } From 38956e07e15413fbaf6cef58b12b1e680bdfeaa2 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:11:37 -0500 Subject: [PATCH 05/37] io: Improve ByteCountStack error message --- io/io/src/TBufferFile.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 01b74a6993477..d0e0047612ab5 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -4086,7 +4086,7 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se auto current = fByteCountStack.size(); (*iter)(*this,obj); if (current != fByteCountStack.size()) - Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); + Fatal("ApplySequence", "ByteCountStack is out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } From a4cf3bb1bd010628581ab2302111b9b481db85e4 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:03:22 -0500 Subject: [PATCH 06/37] net: Keep TMessage to 32 bits (no real enforcement yet) --- net/net/src/TMessage.cxx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/net/net/src/TMessage.cxx b/net/net/src/TMessage.cxx index f36e8b296804c..4105aadcf9f41 100644 --- a/net/net/src/TMessage.cxx +++ b/net/net/src/TMessage.cxx @@ -392,10 +392,16 @@ Int_t TMessage::Compress() fCompPos = fBufCur; bufcur = fBufComp; + if (static_cast(CompLength()) > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Compressed buffer length too large (%d)", CompLength()); + } tobuf(bufcur, (UInt_t)(CompLength() - sizeof(UInt_t))); Int_t what = fWhat | kMESS_ZIP; tobuf(bufcur, what); - tobuf(bufcur, Length()); // original uncompressed buffer length + if (static_cast(Length()) > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Original buffer length too large (%zu)", Length()); + } + tobuf(bufcur, (UInt_t)Length()); // original uncompressed buffer length return 0; } From 3e6e7ff13a95dc3c5a97db9a26633cce5f73b1e0 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 9 Feb 2026 22:09:30 +0100 Subject: [PATCH 07/37] io/meta: Ensure NewArray* takes a 64 bit integer. It is staying signed for historical reason since we do not plan to support collection larger than std::numeric_limit::max() as onfile we reserve the sign bit as a marker. --- core/cont/inc/TVirtualCollectionProxy.h | 8 ++++---- core/meta/inc/TClass.h | 8 ++++---- core/meta/src/TClass.cxx | 12 ++++++++---- io/io/inc/TEmulatedCollectionProxy.h | 12 ++++++------ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 12d6e412285d8..316f75d2e563b 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -123,19 +123,19 @@ class TVirtualCollectionProxy { } /// Construct an array of `nElements` container objects and return the base address of the array - virtual void *NewArray(Int_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } + virtual void *NewArray(Long64_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual void *NewArray(Int_t nElements, void *arena) const + virtual void *NewArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements, arena); } /// Construct an array of `nElements` container objects and return the base address of the array - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements, void *arena) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements, arena); } diff --git a/core/meta/inc/TClass.h b/core/meta/inc/TClass.h index 38f68c3b083dd..6992c56f9b7a1 100644 --- a/core/meta/inc/TClass.h +++ b/core/meta/inc/TClass.h @@ -545,12 +545,12 @@ friend class TStreamerInfo; void Move(void *arenaFrom, void *arenaTo) const; void *New(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; void *New(void *arena, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; ObjectPtr NewObject(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; ObjectPtr NewObject(void *arena, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; virtual void PostLoadCheck(); Long_t Property() const override; Int_t ReadBuffer(TBuffer &b, void *pointer, Int_t version, UInt_t start, UInt_t count); diff --git a/core/meta/src/TClass.cxx b/core/meta/src/TClass.cxx index 53efdc205b1c2..a0279149396f4 100644 --- a/core/meta/src/TClass.cxx +++ b/core/meta/src/TClass.cxx @@ -5323,8 +5323,9 @@ TClass::ObjectPtr TClass::NewObject(void *arena, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5339,8 +5340,9 @@ void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { @@ -5425,8 +5427,9 @@ TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstruct /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, arena, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5440,8 +5443,9 @@ void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) c /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { diff --git a/io/io/inc/TEmulatedCollectionProxy.h b/io/io/inc/TEmulatedCollectionProxy.h index 843807b0f13d4..0e0fa09b5a71a 100644 --- a/io/io/inc/TEmulatedCollectionProxy.h +++ b/io/io/inc/TEmulatedCollectionProxy.h @@ -157,29 +157,29 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { TClass::ObjectPtr NewObject(void *memory) const override { return {New(memory), nullptr}; } // Virtual array constructor - void *NewArray(Int_t nElements) const override + void *NewArray(Long64_t nElements) const override { void *arr = ::operator new(nElements * sizeof(Cont_t)); - for (Int_t i = 0; i < nElements; ++i) + for (Long64_t i = 0; i < nElements; ++i) WithCont(static_cast(arr) + i * sizeof(Cont_t), [](auto *c, std::size_t) { new (c) std::decay_t(); }); return arr; } // Virtual in-place array constructor - void *NewArray(Int_t nElements, void *memory) const override + void *NewArray(Long64_t nElements, void *memory) const override { - for (Int_t i = 0; i < nElements; ++i) + for (Long64_t i = 0; i < nElements; ++i) WithCont(static_cast(memory) + i * sizeof(Cont_t), [](auto *c, std::size_t) { new (c) std::decay_t(); }); return memory; } // Virtual array constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements) const override { return {NewArray(nElements), nullptr}; } + TClass::ObjectPtr NewObjectArray(Long64_t nElements) const override { return {NewArray(nElements), nullptr}; } // Virtual in-place array constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements, void *memory) const override + TClass::ObjectPtr NewObjectArray(Long64_t nElements, void *memory) const override { return {NewArray(nElements, memory), nullptr}; } From ffcad09c04f7c9ed18eb08d2614e37ab0b6c905e Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 17:07:28 -0500 Subject: [PATCH 08/37] io: TBuffer offset member to 64bits --- core/base/inc/TBuffer.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index 09488245ce33a..17481e8a1ddf2 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -45,10 +45,12 @@ class TBuffer : public TObject { protected: typedef std::vector CacheList_t; + // FIXME: decide if this is signed or unsigned. + using Size_t = std::size_t; Bool_t fMode; //Read or write mode Int_t fVersion; //Buffer format version - Int_t fBufSize; //Size of buffer + Size_t fBufSize; //Size of buffer char *fBuffer; //Buffer used to store objects char *fBufCur; //Current position in buffer char *fBufMax; //End of buffer @@ -105,9 +107,9 @@ class TBuffer : public TObject { TObject *GetParent() const; char *Buffer() const { return fBuffer; } char *GetCurrent() const { return fBufCur; } - Int_t BufferSize() const { return fBufSize; } + Size_t BufferSize() const { return fBufSize; } void DetachBuffer() { fBuffer = nullptr; } - Int_t Length() const { return (Int_t)(fBufCur - fBuffer); } + Size_t Length() const { return (Size_t)(fBufCur - fBuffer); } void Expand(Long64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize void AutoExpand(Long64_t size_needed); // expand buffer to newsize Bool_t ByteSwapBuffer(Long64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer From 4db05b1de234d835fde779711c9d3f42d84f197f Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 17:08:00 -0500 Subject: [PATCH 09/37] io: Lift size limit on TBufferFile ... --- core/base/src/TBuffer.cxx | 15 ++++++++------- io/io/src/TBufferFile.cxx | 2 +- io/io/src/TBufferIO.cxx | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 87a801be6e116..04edf66cdc6bd 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -20,7 +20,8 @@ Buffer base class used for serializing objects. #include "TProcessID.h" constexpr Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) -constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size. +constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). +constexpr ULong64_t kMaxLargeBufferSize = 0x7FFFFFFFFFFFFFFE; // largest possible size. @@ -102,8 +103,8 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize) TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) { - if (bufsize > kMaxBufferSize) - Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); + if (bufsize > kMaxLargeBufferSize) + Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%llx.", bufsize, kMaxLargeBufferSize); fBufSize = bufsize; fMode = mode; fVersion = 0; @@ -229,11 +230,11 @@ void TBuffer::Expand(Long64_t newsize, Bool_t copy) } const Int_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; - if ( newsize > kMaxBufferSize - kExtraSpace) { - if (l < kMaxBufferSize) { - newsize = kMaxBufferSize - extraspace; + if ( newsize > kMaxLargeBufferSize - kExtraSpace) { + if (l < kMaxLargeBufferSize) { + newsize = kMaxLargeBufferSize - extraspace; } else { - Fatal("Expand","Requested size (%lld) is too large (max is %d).", newsize, kMaxBufferSize); + Fatal("Expand","Requested size (%lld) is too large (max is %lld).", newsize, kMaxBufferSize); } } if ( (fMode&kWrite)!=0 ) { diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index d0e0047612ab5..fe7998eea543f 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -361,7 +361,7 @@ void TBufferFile::SetByteCount(ULong64_t cntpos, Bool_t packInVersion) assert( (cntpos == kOverflowPosition || (cntpos < kOverflowPosition && (sizeof(UInt_t) + cntpos) < static_cast(fBufCur - fBuffer))) && (fBufCur >= fBuffer) - && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() + && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() && "Byte count position is after the end of the buffer"); // We can either make this unconditional or we could split the routine diff --git a/io/io/src/TBufferIO.cxx b/io/io/src/TBufferIO.cxx index 9589eee8cad1c..4bbf86356e234 100644 --- a/io/io/src/TBufferIO.cxx +++ b/io/io/src/TBufferIO.cxx @@ -195,7 +195,7 @@ void TBufferIO::MapObject(const TObject *obj, ULong64_t offset) void TBufferIO::MapObject(const void *obj, const TClass *cl, ULong64_t offset) { - R__ASSERT(offset <= kMaxUInt); + R__ASSERT(offset <= kMaxLong64); if (IsWriting()) { if (!fMap) InitMap(); From 882f687ae76b4e37ff8f8e29ca4af418d10d87eb Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 3 Apr 2026 14:07:57 -0500 Subject: [PATCH 10/37] io: Update TBuffer API, ctor and SetBuffer 64bits: --- core/base/inc/TBuffer.h | 14 +++--- core/base/src/TBuffer.cxx | 65 ++++++++++++++------------- io/io/src/TStreamerInfoReadBuffer.cxx | 8 ++-- roottest/root/io/buffer/ownership.ref | 2 +- tree/tree/inc/TBasket.h | 12 ++--- tree/tree/inc/TBasketSQL.h | 6 +-- tree/tree/inc/TBranch.h | 2 +- tree/tree/inc/TTree.h | 2 +- tree/tree/src/TBasket.cxx | 27 ++++++----- tree/tree/src/TBasketSQL.cxx | 6 +-- tree/tree/src/TBranch.cxx | 2 +- tree/tree/src/TTree.cxx | 2 +- 12 files changed, 79 insertions(+), 69 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index 17481e8a1ddf2..d3caea120dd57 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -90,8 +90,8 @@ class TBuffer : public TObject { enum { kInitialSize = 1024, kMinimalSize = 128 }; TBuffer(EMode mode); - TBuffer(EMode mode, Long64_t bufsize); - TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + TBuffer(EMode mode, ULong64_t bufsize); + TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); virtual ~TBuffer(); Int_t GetBufferVersion() const { return fVersion; } @@ -99,10 +99,10 @@ class TBuffer : public TObject { Bool_t IsWriting() const { return (fMode & kWrite) != 0; } void SetReadMode(); void SetWriteMode(); - void SetBuffer(void *buf, Long64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + void SetBuffer(void *buf, ULong64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); ReAllocCharFun_t GetReAllocFunc() const; void SetReAllocFunc(ReAllocCharFun_t reallocfunc = nullptr); - void SetBufferOffset(Long64_t offset = 0) { fBufCur = fBuffer+offset; } + void SetBufferOffset(ULong64_t offset = 0) { fBufCur = fBuffer+offset; } void SetParent(TObject *parent); TObject *GetParent() const; char *Buffer() const { return fBuffer; } @@ -110,9 +110,9 @@ class TBuffer : public TObject { Size_t BufferSize() const { return fBufSize; } void DetachBuffer() { fBuffer = nullptr; } Size_t Length() const { return (Size_t)(fBufCur - fBuffer); } - void Expand(Long64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize - void AutoExpand(Long64_t size_needed); // expand buffer to newsize - Bool_t ByteSwapBuffer(Long64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer + void Expand(ULong64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize + void AutoExpand(ULong64_t size_needed); // expand buffer to newsize + Bool_t ByteSwapBuffer(ULong64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer virtual Bool_t CheckObject(const TObject *obj) = 0; virtual Bool_t CheckObject(const void *obj, const TClass *ptrClass) = 0; diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 04edf66cdc6bd..b740b495300dc 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -19,8 +19,8 @@ Buffer base class used for serializing objects. #include "TClass.h" #include "TProcessID.h" -constexpr Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) -constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). +constexpr UInt_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) +constexpr UInt_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). constexpr ULong64_t kMaxLargeBufferSize = 0x7FFFFFFFFFFFFFFE; // largest possible size. @@ -70,7 +70,7 @@ TBuffer::TBuffer(EMode mode) /// Create an I/O buffer object. Mode should be either TBuffer::kRead or /// TBuffer::kWrite. -TBuffer::TBuffer(EMode mode, Long64_t bufsize) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize) { if (bufsize > kMaxBufferSize) Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); @@ -101,7 +101,7 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) { if (bufsize > kMaxLargeBufferSize) Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%llx.", bufsize, kMaxLargeBufferSize); @@ -112,12 +112,20 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocC SetBit(kIsOwner); + if (buf && !adopt) + ResetBit(kIsOwner); + + SetReAllocFunc( reallocfunc ); + if (buf) { fBuffer = (char *)buf; if ( (fMode&kWrite)!=0 ) { - fBufSize -= kExtraSpace; + if (fBufSize < kExtraSpace) { + Expand( kMinimalSize ); + } else { + fBufSize -= kExtraSpace; + } } - if (!adopt) ResetBit(kIsOwner); } else { if (fBufSize < kMinimalSize) { fBufSize = kMinimalSize; @@ -126,12 +134,6 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocC } fBufCur = fBuffer; fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// @@ -155,13 +157,13 @@ TBuffer::~TBuffer() /// If the size_needed is larger than the current size, the policy /// is to expand to double the current size or the size_needed which ever is largest. -void TBuffer::AutoExpand(Long64_t size_needed) +void TBuffer::AutoExpand(ULong64_t size_needed) { if (size_needed > kMaxBufferSize) { Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%x.", size_needed, kMaxBufferSize); } if (size_needed > fBufSize) { - Long64_t doubling = 2LLU * fBufSize; + ULong64_t doubling = 2LLU * fBufSize; if (doubling > kMaxBufferSize) doubling = kMaxBufferSize; if (size_needed > doubling) { @@ -184,7 +186,7 @@ void TBuffer::AutoExpand(Long64_t size_needed) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) +void TBuffer::SetBuffer(void *buf, ULong64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) { if (newsiz > kMaxBufferSize) Fatal("SetBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", newsiz, kMaxBufferSize); @@ -196,22 +198,22 @@ void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun else ResetBit(kIsOwner); + SetReAllocFunc( reallocfunc ); + fBuffer = (char *)buf; fBufCur = fBuffer; if (newsiz > 0) { if ( (fMode&kWrite)!=0 ) { - fBufSize = newsiz - kExtraSpace; + if (newsiz >= kExtraSpace) { + fBufSize = newsiz - kExtraSpace; + } else { + Expand( kMinimalSize ); + } } else { fBufSize = newsiz; } } fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// @@ -222,22 +224,23 @@ void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun /// In order to avoid losing data, if the current length is greater than /// the requested size, we only shrink down to the current length. -void TBuffer::Expand(Long64_t newsize, Bool_t copy) +void TBuffer::Expand(ULong64_t newsize, Bool_t copy) { - Int_t l = Length(); - if ( (Long64_t(l) > newsize) && copy ) { + ULong64_t l = Length(); + if ( l > newsize && copy ) { newsize = l; } - const Int_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; + const ULong64_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; if ( newsize > kMaxLargeBufferSize - kExtraSpace) { if (l < kMaxLargeBufferSize) { newsize = kMaxLargeBufferSize - extraspace; } else { - Fatal("Expand","Requested size (%lld) is too large (max is %lld).", newsize, kMaxBufferSize); + Fatal("Expand","Requested size (%llu) is too large (max is %llu).", newsize, kMaxLargeBufferSize); } } if ( (fMode&kWrite)!=0 ) { + // FIXME-LARGE: The size can overflow and the signature of fReAllocFunc is not large enough to handle the size of the buffer. fBuffer = fReAllocFunc(fBuffer, newsize+kExtraSpace, copy ? fBufSize+kExtraSpace : 0); } else { @@ -393,13 +396,13 @@ TVirtualArray *TBuffer::PopDataCache() /// Byte-swap N primitive-elements in the buffer. /// Bulk API relies on this function. -Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) +Bool_t TBuffer::ByteSwapBuffer(ULong64_t n, EDataType type) { char *input_buf = GetCurrent(); if ((type == EDataType::kShort_t) || (type == EDataType::kUShort_t)) { #ifdef R__BYTESWAP Short_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -408,7 +411,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kFloat_t) || (type == EDataType::kInt_t) || (type == EDataType::kUInt_t)) { #ifdef R__BYTESWAP Float_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -417,7 +420,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kDouble_t) || (type == EDataType::kLong64_t) || (type == EDataType::kULong64_t)) { #ifdef R__BYTESWAP Double_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 611abf936bacb..0754648b25a38 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -89,7 +89,7 @@ TStreamerElement *TStreamerInfo::GetCurrentElement() Char_t isArray; \ b >> isArray; \ Int_t *l = (Int_t*)(arr[index]+imethod); \ - if (*l < 0 || *l > b.BufferSize()) continue; \ + if (*l < 0 || static_cast(*l) > b.BufferSize()) continue; \ name **f = (name**)(arr[index]+ioffset); \ int j; \ if (isArray) for(j=0;jfLength;j++) { \ @@ -486,7 +486,7 @@ Int_t TStreamerInfo::ReadBufferSkip(TBuffer &b, const T &arr, const TCompInfo *c DOLOOP { \ b >> isArray; \ Int_t *l = (Int_t*)(arr[k]+imethod); \ - if (*l>0 && *l < b.BufferSize()) { \ + if (*l>0 && static_cast(*l) < b.BufferSize()) { \ readbuf = new name[*l]; \ switch(newtype) { \ case TStreamerInfo::kBool: ConvCBasicPointerTo(Bool_t,ReadArrayFunc); \ @@ -798,7 +798,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%zu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -811,7 +811,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d\n", + " %s, bufpos=%zu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], ioffset); } diff --git a/roottest/root/io/buffer/ownership.ref b/roottest/root/io/buffer/ownership.ref index 5c633dbad1d39..e39e5dca5c19e 100644 --- a/roottest/root/io/buffer/ownership.ref +++ b/roottest/root/io/buffer/ownership.ref @@ -1,7 +1,7 @@ Processing runownership.C+... Fatal in : Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided. The expected error "Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided." was seen -Running custom allocator (increase from 3 to 136) +Running custom allocator (increase from 128 to 136) Running custom allocator (increase from 136 to 264) The test has completed successfully (int) 0 diff --git a/tree/tree/inc/TBasket.h b/tree/tree/inc/TBasket.h index 66a737ac2ec2c..e926496513c76 100644 --- a/tree/tree/inc/TBasket.h +++ b/tree/tree/inc/TBasket.h @@ -39,11 +39,11 @@ friend class TBranch; TBasket& operator=(const TBasket&); ///< TBasket objects are not copiable. // Internal corner cases for ReadBasketBuffers - Int_t ReadBasketBuffersUnzip(char*, Int_t, bool, TFile*); + Int_t ReadBasketBuffersUnzip(char*, ULong64_t, bool, TFile*); Int_t ReadBasketBuffersUncompressedCase(); // Helper for managing the compressed buffer. - void InitializeCompressedBuffer(Int_t len, TFile* file); + void InitializeCompressedBuffer(ULong64_t len, TFile* file); // Handles special logic around deleting / reseting the entry offset pointer. void ResetEntryOffset(); @@ -130,9 +130,9 @@ friend class TBranch; Int_t GetNevBufSize() const {return fNevBufSize;} Int_t GetLast() const {return fLast;} virtual void MoveEntries(Int_t dentries); - virtual void PrepareBasket(Long64_t /* entry */) {}; - Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - Int_t ReadBasketBytes(Long64_t pos, TFile *file); + virtual void PrepareBasket(ULong64_t /* entry */) {}; + Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + Int_t ReadBasketBytes(ULong64_t pos, TFile *file); virtual void WriteReset(); // Time spent reseting basket sizes (typically, at event cluster boundaries), in microseconds @@ -142,7 +142,7 @@ friend class TBranch; // Count of resets performed of basket size. bool GetResetAllocationCount() const { return fResetAllocation; } - Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree = nullptr); + Int_t LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree = nullptr); Long64_t CopyTo(TFile *to); void SetBranch(TBranch *branch) { fBranch = branch; } diff --git a/tree/tree/inc/TBasketSQL.h b/tree/tree/inc/TBasketSQL.h index efe9caab8261c..ba7925e8d33e7 100644 --- a/tree/tree/inc/TBasketSQL.h +++ b/tree/tree/inc/TBasketSQL.h @@ -45,9 +45,9 @@ class TBasketSQL : public TBasket TBranch *branch, TSQLResult **rs, TString *insert_query, std::vector *vc, TSQLRow **row); ~TBasketSQL() override; - void PrepareBasket(Long64_t entry) override; - virtual Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - virtual Int_t ReadBasketBytes(Long64_t pos, TFile *file); + void PrepareBasket(ULong64_t entry) override; + virtual Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + virtual Int_t ReadBasketBytes(ULong64_t pos, TFile *file); TSQLResult * GetResultSet() { return *fResultPtr;} void CreateBuffer(const char *name, TString title, std::vector * vc, TBranch *branch, TSQLResult ** rs); diff --git a/tree/tree/inc/TBranch.h b/tree/tree/inc/TBranch.h index f91d1e8380495..64be8447bdc58 100644 --- a/tree/tree/inc/TBranch.h +++ b/tree/tree/inc/TBranch.h @@ -263,7 +263,7 @@ class TBranch : public TNamed, public TAttFill { virtual bool GetMakeClass() const; TBranch *GetMother() const; TBranch *GetSubBranch(const TBranch *br) const; - TBuffer *GetTransientBuffer(Int_t size); + TBuffer *GetTransientBuffer(ULong64_t size); bool IsAutoDelete() const; bool IsFolder() const override; virtual void KeepCircular(Long64_t maxEntries); diff --git a/tree/tree/inc/TTree.h b/tree/tree/inc/TTree.h index 1899bc358781f..c59ee8debfdc5 100644 --- a/tree/tree/inc/TTree.h +++ b/tree/tree/inc/TTree.h @@ -607,7 +607,7 @@ class TTree : public TNamed, public TAttLine, public TAttFill, public TAttMarker TTreeFormula *GetSelect() { return GetPlayer()->GetSelect(); } virtual Long64_t GetSelectedRows() { return GetPlayer()->GetSelectedRows(); } virtual Int_t GetTimerInterval() const { return fTimerInterval; } - TBuffer* GetTransientBuffer(Int_t size); + TBuffer* GetTransientBuffer(ULong64_t size); virtual Long64_t GetTotBytes() const { return fTotBytes; } virtual TTree *GetTree() const { return const_cast(this); } virtual TVirtualIndex *GetTreeIndex() const { return fTreeIndex; } diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx index 8a25921edb57b..6690490cf9b1b 100644 --- a/tree/tree/src/TBasket.cxx +++ b/tree/tree/src/TBasket.cxx @@ -243,7 +243,7 @@ Int_t TBasket::GetEntryPointer(Int_t entry) /// This function is called by TTreeCloner. /// The function returns 0 in case of success, 1 in case of error. -Int_t TBasket::LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree) +Int_t TBasket::LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree) { if (fBufferRef) { // Reuse the buffer if it exist. @@ -378,7 +378,7 @@ Int_t TBasket::ReadBasketBuffersUncompressedCase() //////////////////////////////////////////////////////////////////////////////// /// We always create the TBuffer for the basket but it hold the buffer from the cache. -Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, TFile* file) +Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, ULong64_t size, bool mustFree, TFile* file) { if (fBufferRef) { fBufferRef->SetBuffer(buffer, size, mustFree); @@ -408,15 +408,19 @@ Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, T //////////////////////////////////////////////////////////////////////////////// /// Initialize a buffer for reading if it is not already initialized -static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file) +static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, ULong64_t len, TFile* file) { TBuffer* result; if (R__likely(bufferRef)) { bufferRef->SetReadMode(); - Int_t curBufferSize = bufferRef->BufferSize(); + ULong64_t curBufferSize = bufferRef->BufferSize(); if (curBufferSize < len) { // Experience shows that giving 5% "wiggle-room" decreases churn. - bufferRef->Expand(Int_t(len*1.05)); + double newSizeD = len * 1.05; + auto newSize = newSizeD < (double)std::numeric_limits::max() + ? static_cast(newSizeD) + : std::numeric_limits::max(); + bufferRef->Expand(newSize); } bufferRef->Reset(); result = bufferRef; @@ -430,7 +434,7 @@ static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t l //////////////////////////////////////////////////////////////////////////////// /// Initialize the compressed buffer; either from the TTree or create a local one. -void inline TBasket::InitializeCompressedBuffer(Int_t len, TFile* file) +void inline TBasket::InitializeCompressedBuffer(ULong64_t len, TFile* file) { bool compressedBufferExists = fCompressedBufferRef != nullptr; fCompressedBufferRef = R__InitializeReadBasketBuffer(fCompressedBufferRef, len, file); @@ -462,7 +466,7 @@ void TBasket::ResetEntryOffset() /// There is a lot of code duplication but it was necessary to assure /// the expected behavior when there is no cache. -Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) +Int_t TBasket::ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file) { if(!fBranch->GetDirectory()) { return -1; @@ -607,7 +611,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) // Note that in previous versions we didn't allocate buffers until we verified // the zip headers; this is no longer beforehand as the buffer lifetime is scoped // to the TBranch. - uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen; + uncompressedBufferLen = len > static_cast(fObjlen+fKeylen) ? len : fObjlen+fKeylen; fBufferRef = R__InitializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file); rawUncompressedBuffer = fBufferRef->Buffer(); fBuffer = rawUncompressedBuffer; @@ -699,7 +703,10 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) if (R__unlikely(!fEntryOffset)) { fEntryOffset = new Int_t[fNevBuf+1]; fEntryOffset[0] = fKeylen; - Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen); + Warning("ReadBasketBuffers", + "basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%llu, len=%llu, fNbytes=%d, fObjlen=%d, trying to " + "repair", + GetName(), fNevBuf, pos, len, fNbytes, fObjlen); return 0; } if (fIOBits & static_cast(TBasket::EIOBits::kGenerateOffsetMap)) { @@ -731,7 +738,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) /// Read first bytes of a logical record starting at position pos /// return record length (first 4 bytes of record). -Int_t TBasket::ReadBasketBytes(Long64_t pos, TFile *file) +Int_t TBasket::ReadBasketBytes(ULong64_t pos, TFile *file) { const Int_t len = 128; char buffer[len]; diff --git a/tree/tree/src/TBasketSQL.cxx b/tree/tree/src/TBasketSQL.cxx index 5c0edf3639030..6bb0ed64a33b2 100644 --- a/tree/tree/src/TBasketSQL.cxx +++ b/tree/tree/src/TBasketSQL.cxx @@ -111,7 +111,7 @@ void TBasketSQL::CreateBuffer(const char *name, TString title, //////////////////////////////////////////////////////////////////////////////// /// Prepare the basket for the next entry. -void TBasketSQL::PrepareBasket(Long64_t entry) +void TBasketSQL::PrepareBasket(ULong64_t entry) { ((TBufferSQL*)fBufferRef)->ResetOffset(); ((TTreeSQL*)fBranch->GetTree())->PrepEntry(entry); @@ -121,7 +121,7 @@ void TBasketSQL::PrepareBasket(Long64_t entry) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBytes. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) +Int_t TBasketSQL::ReadBasketBytes(ULong64_t , TFile *) { Error("ReadBasketBytes","This member function should not be called!"); return 0; @@ -130,7 +130,7 @@ Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBuffers. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBuffers(Long64_t , Int_t, TFile *) +Int_t TBasketSQL::ReadBasketBuffers(ULong64_t , ULong64_t, TFile *) { Error("ReadBasketBuffers","This member function should not be called!"); return 0; diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx index 931e5a62eea52..ddeb76f2459f5 100644 --- a/tree/tree/src/TBranch.cxx +++ b/tree/tree/src/TBranch.cxx @@ -522,7 +522,7 @@ TBranch::~TBranch() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TBranch for reading/writing baskets. -TBuffer* TBranch::GetTransientBuffer(Int_t size) +TBuffer* TBranch::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) { diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx index 0be514e18f22b..95ca1e6acad01 100644 --- a/tree/tree/src/TTree.cxx +++ b/tree/tree/src/TTree.cxx @@ -1061,7 +1061,7 @@ TTree::~TTree() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TTree for reading/writing baskets. -TBuffer* TTree::GetTransientBuffer(Int_t size) +TBuffer* TTree::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) { From b08d508b66ffe1a69649dcf7aea5a3ef4c046dbe Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 13:03:45 -0500 Subject: [PATCH 11/37] io: Read/WriteFastArray Float to Long64_t --- core/base/inc/TBuffer.h | 2 +- io/io/inc/TBufferFile.h | 6 +++--- io/io/inc/TBufferJSON.h | 2 +- io/io/src/TBufferFile.cxx | 16 ++++++++-------- io/io/src/TBufferJSON.cxx | 2 +- io/sql/inc/TBufferSQL2.h | 2 +- io/sql/src/TBufferSQL2.cxx | 2 +- io/xml/inc/TBufferXML.h | 2 +- io/xml/src/TBufferXML.cxx | 2 +- tree/tree/inc/TBufferSQL.h | 2 +- tree/tree/src/TBufferSQL.cxx | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index d3caea120dd57..d24ca721957c6 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -322,7 +322,7 @@ class TBuffer : public TObject { virtual void ReadFastArray(ULong_t *l, Int_t n) = 0; virtual void ReadFastArray(Long64_t *l, Int_t n) = 0; virtual void ReadFastArray(ULong64_t *l, Int_t n) = 0; - virtual void ReadFastArray(Float_t *f, Int_t n) = 0; + virtual void ReadFastArray(Float_t *f, Long64_t n) = 0; virtual void ReadFastArray(Double_t *d, Int_t n) = 0; virtual void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) = 0; virtual void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) = 0; diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index fe73ecf98e446..348b6d405ee7a 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -88,7 +88,7 @@ class TBufferFile : public TBufferIO { UInt_t ReserveByteCount(const TClass *cl) override; void WriteObjectClass(const void *actualObjStart, const TClass *actualClass, Bool_t cacheReuse) override; - bool ShouldNotReadCollection(Int_t lengthInBytes, Int_t nElements=1) const; + bool ShouldNotReadCollection(Long64_t lengthInBytes, Long64_t nElements=1) const; public: enum { kStreamedMemberWise = BIT(14) }; //added to version number to know if a collection has been stored member-wise @@ -193,7 +193,7 @@ class TBufferFile : public TBufferIO { void ReadFastArray(ULong_t *l, Int_t n) override; void ReadFastArray(Long64_t *l, Int_t n) override; void ReadFastArray(ULong64_t *l, Int_t n) override; - void ReadFastArray(Float_t *f, Int_t n) override; + void ReadFastArray(Float_t *f, Long64_t n) override; void ReadFastArray(Double_t *d, Int_t n) override; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) override; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) override; @@ -300,7 +300,7 @@ class TBufferFile : public TBufferIO { //---------------------- TBufferFile inlines --------------------------------------- //______________________________________________________________________________ -inline bool TBufferFile::ShouldNotReadCollection(Int_t lengthInBytes, Int_t nElements) const +inline bool TBufferFile::ShouldNotReadCollection(Long64_t lengthInBytes, Long64_t nElements) const { // Three cases here in which we should not read the collection: // 1. The collection has zero or a negative number of elements or diff --git a/io/io/inc/TBufferJSON.h b/io/io/inc/TBufferJSON.h index a9b6fbaf9da0a..4fd36806315ec 100644 --- a/io/io/inc/TBufferJSON.h +++ b/io/io/inc/TBufferJSON.h @@ -160,7 +160,7 @@ class TBufferJSON final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, const TClass *onFileClass = nullptr) final; diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index fe7998eea543f..0ed933801bf92 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -1530,9 +1530,9 @@ void TBufferFile::ReadFastArray(Long64_t *ll, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of n floats from the I/O buffer. -void TBufferFile::ReadFastArray(Float_t *f, Int_t n) +void TBufferFile::ReadFastArray(Float_t *f, Long64_t n) { - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (ShouldNotReadCollection(l, n)) return; #ifdef R__BYTESWAP @@ -1540,7 +1540,7 @@ void TBufferFile::ReadFastArray(Float_t *f, Int_t n) bswapcpy32(f, fBufCur, n); fBufCur += sizeof(Float_t)*n; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) frombuf(fBufCur, &f[i]); # endif #else @@ -2333,15 +2333,15 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) { if (n == 0) return; - constexpr Int_t dataWidth = static_cast(sizeof(Float_t)); - const Int_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; + constexpr Long64_t dataWidth = static_cast(sizeof(Float_t)); + const Long64_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; if (n < 0 || n > maxElements) { - Fatal("WriteFastArray", "Not enough space left in the buffer (1GB limit). %lld elements is greater than the max left of %d", n, maxElements); + Fatal("WriteFastArray", "Not enough space left in the buffer. %lld elements is greater than the max left of %lld", n, maxElements); return; // In case the user re-routes the error handler to not die when Fatal is called } - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (fBufCur + l > fBufMax) AutoExpand(fBufSize+l); #ifdef R__BYTESWAP @@ -2349,7 +2349,7 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) bswapcpy32(fBufCur, f, n); fBufCur += l; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) tobuf(fBufCur, f[i]); # endif #else diff --git a/io/io/src/TBufferJSON.cxx b/io/io/src/TBufferJSON.cxx index e5dc7f4e7b303..8ea61fa3026b8 100644 --- a/io/io/src/TBufferJSON.cxx +++ b/io/io/src/TBufferJSON.cxx @@ -3019,7 +3019,7 @@ void TBufferJSON::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// read array of Float_t from buffer -void TBufferJSON::ReadFastArray(Float_t *f, Int_t n) +void TBufferJSON::ReadFastArray(Float_t *f, Long64_t n) { JsonReadFastArray(f, n); } diff --git a/io/sql/inc/TBufferSQL2.h b/io/sql/inc/TBufferSQL2.h index 8db572a85b362..1088ce6f20161 100644 --- a/io/sql/inc/TBufferSQL2.h +++ b/io/sql/inc/TBufferSQL2.h @@ -198,7 +198,7 @@ class TBufferSQL2 final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/sql/src/TBufferSQL2.cxx b/io/sql/src/TBufferSQL2.cxx index 47bd08fe11151..6ec9aa8952467 100644 --- a/io/sql/src/TBufferSQL2.cxx +++ b/io/sql/src/TBufferSQL2.cxx @@ -1252,7 +1252,7 @@ void TBufferSQL2::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferSQL2::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL2::ReadFastArray(Float_t *f, Long64_t n) { SqlReadFastArray(f, n); } diff --git a/io/xml/inc/TBufferXML.h b/io/xml/inc/TBufferXML.h index b25aff793a838..db9202f2f983c 100644 --- a/io/xml/inc/TBufferXML.h +++ b/io/xml/inc/TBufferXML.h @@ -126,7 +126,7 @@ class TBufferXML final : public TBufferText, public TXMLSetup { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/xml/src/TBufferXML.cxx b/io/xml/src/TBufferXML.cxx index 74b74043462c5..ff8c52aa8cc71 100644 --- a/io/xml/src/TBufferXML.cxx +++ b/io/xml/src/TBufferXML.cxx @@ -1779,7 +1779,7 @@ void TBufferXML::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferXML::ReadFastArray(Float_t *f, Int_t n) +void TBufferXML::ReadFastArray(Float_t *f, Long64_t n) { XmlReadFastArray(f, n); } diff --git a/tree/tree/inc/TBufferSQL.h b/tree/tree/inc/TBufferSQL.h index 72908333e689c..d988488b7bd49 100644 --- a/tree/tree/inc/TBufferSQL.h +++ b/tree/tree/inc/TBufferSQL.h @@ -116,7 +116,7 @@ class TBufferSQL final : public TBufferFile { void ReadFastArray(ULong_t *, Int_t ) final; void ReadFastArray(Long64_t *, Int_t ) final; void ReadFastArray(ULong64_t *, Int_t ) final; - void ReadFastArray(Float_t *, Int_t ) final; + void ReadFastArray(Float_t *, Long64_t ) final; void ReadFastArray(Double_t *, Int_t ) final; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele=nullptr) final; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele=nullptr) final; diff --git a/tree/tree/src/TBufferSQL.cxx b/tree/tree/src/TBufferSQL.cxx index ba4d09306ef5c..eae27a605a42b 100644 --- a/tree/tree/src/TBufferSQL.cxx +++ b/tree/tree/src/TBufferSQL.cxx @@ -850,9 +850,9 @@ void TBufferSQL::ReadFastArray(ULong64_t *ull, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// ReadFastArray SQL implementation. -void TBufferSQL::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL::ReadFastArray(Float_t *f, Long64_t n) { - for(int i=0; iGetField(*fIter)); ++fIter; } From 9ccb08179c5886f9a04f5838457b4a27d9e61d1a Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 15:48:38 -0500 Subject: [PATCH 12/37] io: Relax TBuffer::AutoExpand to 64bits --- core/base/src/TBuffer.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index b740b495300dc..d193c071bd980 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -159,13 +159,13 @@ TBuffer::~TBuffer() void TBuffer::AutoExpand(ULong64_t size_needed) { - if (size_needed > kMaxBufferSize) { - Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%x.", size_needed, kMaxBufferSize); + if (size_needed > kMaxLargeBufferSize) { + Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%llx.", size_needed, kMaxLargeBufferSize); } if (size_needed > fBufSize) { ULong64_t doubling = 2LLU * fBufSize; - if (doubling > kMaxBufferSize) - doubling = kMaxBufferSize; + if (doubling > kMaxLargeBufferSize) + doubling = kMaxLargeBufferSize; if (size_needed > doubling) { Expand(size_needed); } else { From 1e85be4eb41538bbabb62e9d8119c5143cecc8d6 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:03:19 -0500 Subject: [PATCH 13/37] io: 64bits TVirtualCollectionProxy --- core/cont/inc/TVirtualCollectionProxy.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 316f75d2e563b..98d61c8b01c55 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -71,6 +71,8 @@ class TVirtualCollectionProxy { kCustomAlloc = BIT(5) ///< The collection has a custom allocator. }; + using size_type = UInt_t; + /// RAII helper class that ensures that `PushProxy()` / `PopProxy()` are called when entering / leaving a C++ context class TPushPop { public: @@ -155,7 +157,7 @@ class TVirtualCollectionProxy { } /// Return the `sizeof()` of the collection object - virtual UInt_t Sizeof() const = 0; + virtual size_type Sizeof() const = 0; /// Set the address of the container being proxied and keep track of the previous one virtual void PushProxy(void *objectstart) = 0; @@ -174,19 +176,19 @@ class TVirtualCollectionProxy { virtual EDataType GetType() const = 0; /// Return the address of the value at index `idx` - virtual void *At(UInt_t idx) = 0; + virtual void *At(size_type idx) = 0; /// Clear the container virtual void Clear(const char *opt = "") = 0; /// Return the current number of elements in the container - virtual UInt_t Size() const = 0; + virtual size_type Size() const = 0; /// Allocates space for storing at least `n` elements. This function returns a pointer to the actual object on /// which insertions should take place. For associative collections, this function returns a pointer to a temporary /// buffer known as the staging area. If the insertion happened in a staging area (i.e. the returned pointer != /// proxied object), `Commit()` should be called on the value returned by this function. - virtual void* Allocate(UInt_t n, Bool_t forceDelete) = 0; + virtual void* Allocate(size_type n, Bool_t forceDelete) = 0; /// Commits pending elements in a staging area (see Allocate() for more information). virtual void Commit(void*) = 0; @@ -196,7 +198,7 @@ class TVirtualCollectionProxy { virtual void Insert(const void *data, void *container, size_t size) = 0; /// Return the address of the value at index `idx` - char *operator[](UInt_t idx) const { return (char *)(const_cast(this))->At(idx); } + char *operator[](size_type idx) const { return (char *)(const_cast(this))->At(idx); } // Functions related to member-wise actions virtual TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *oldClass, Int_t version) = 0; From d194ae759213c617af40295eb899cad955865ee5 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:03:52 -0500 Subject: [PATCH 14/37] io: Read TStreamerInfo version to 11, 64bits collection size in old code --- io/io/src/TStreamerInfoReadBuffer.cxx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 0754648b25a38..63c521b993b1e 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -1303,7 +1303,17 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, TVirtualCollectionProxy::TPushPop helper( newProxy, obj ); Int_t nobjects; b >> nobjects; - void* env = newProxy->Allocate(nobjects,true); + ULong64_t nobjects64; + if (std::signbit(nobjects) && vers >= 11) { + nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; + UInt_t nobjectsLow; + b >> nobjectsLow; + nobjects64 |= nobjectsLow; + } else { + nobjects64 = nobjects; + } + R__ASSERT(nobjects64 == nobjects && "Allocate is not yet 64 bit enabled"); + void* env = newProxy->Allocate(nobjects64, true); if (!nobjects && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. From 30ebdcc99151396a003f3db2b7138d3b116efe66 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 16:07:34 -0500 Subject: [PATCH 15/37] io: (for) TStreamerInfo version to 11, 64bits collection size --- io/io/src/TStreamerInfoActions.cxx | 12 ++--- io/io/src/TStreamerInfoCollectionUtils.h | 67 ++++++++++++++++++++++++ io/io/src/TStreamerInfoReadBuffer.cxx | 17 ++---- 3 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 io/io/src/TStreamerInfoCollectionUtils.h diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index 08468706c7f67..d0b5b5f90fe90 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -19,6 +19,7 @@ #include "TVirtualArray.h" #include "TBufferFile.h" #include "TBufferText.h" +#include "TStreamerInfoCollectionUtils.h" #include "TMemberStreamer.h" #include "TClassEdit.h" #include "TVirtualCollectionIterators.h" @@ -2227,21 +2228,20 @@ namespace TStreamerInfoActions TConfigSTL *config = (TConfigSTL*)conf; UInt_t start, count; - /* Version_t vers = */ buf.ReadVersion(&start, &count, config->fOldClass); + Version_t vers = buf.ReadVersion(&start, &count, config->fOldClass); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues; - buf.ReadInt(nvalues); - vec->resize(nvalues); + ULong64_t nvalues64 = TStreamerInfoUtils::ReadCollectionSize(buf, vers); + vec->resize(nvalues64); #ifdef R__VISUAL_CPLUSPLUS - if (nvalues <= 0) { + if (nvalues64 == 0) { buf.CheckByteCount(start,count,config->fTypeName); return 0; } #endif T *begin = vec->data(); - buf.ReadFastArray(begin, nvalues); + buf.ReadFastArray(begin, nvalues64); buf.CheckByteCount(start,count,config->fTypeName); return 0; diff --git a/io/io/src/TStreamerInfoCollectionUtils.h b/io/io/src/TStreamerInfoCollectionUtils.h new file mode 100644 index 0000000000000..be6cad82c80d6 --- /dev/null +++ b/io/io/src/TStreamerInfoCollectionUtils.h @@ -0,0 +1,67 @@ +// @(#)root/io:$Id$ +// Author: Philippe Canal 04/2026 + +/************************************************************************* + * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TStreamerInfoCollectionUtils +#define ROOT_TStreamerInfoCollectionUtils + +#include "TBuffer.h" +#include "RtypesCore.h" +#include // std::signbit +#include // std::numeric_limits + +namespace TStreamerInfoUtils { + +/// Read the number-of-objects count from the buffer using the encoding introduced +/// in TStreamerInfo version 11: when the high bit of the 32-bit word is set the +/// remaining 31 bits form the high part of a 64-bit count, and the following +/// UInt_t carries the low 32 bits. +/// +/// \param b The input buffer (operator>> must be available for Int_t/UInt_t). +/// \param vers The TStreamerInfo version that was used when the data was written. +/// \return The number of objects encoded in the stream. +inline ULong64_t ReadCollectionSize(TBuffer &b, Version_t vers) +{ + Int_t nobjects; + b >> nobjects; + ULong64_t nobjects64; + if (std::signbit(nobjects) && vers >= 11) { + nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; + UInt_t nobjectsLow; + b >> nobjectsLow; + nobjects64 |= nobjectsLow; + } else { + nobjects64 = nobjects; + } + return nobjects64; +} + +/// Write the number-of-objects count to the buffer using the encoding introduced +/// in TStreamerInfo version 11: counts that fit in a non-negative Int_t are written +/// as a single Int_t; larger counts set the high bit of the first Int_t (carrying +/// bits 32–62 of the count) and follow with a UInt_t carrying the low 32 bits. +/// +/// \param b The output buffer (WriteInt/WriteUInt must be available). +/// \param n The number of objects to encode. +inline void WriteCollectionSize(TBuffer &b, ULong64_t n) +{ + if (n <= static_cast(std::numeric_limits::max())) { + b.WriteInt(static_cast(n)); + } else { + // Encode high 31 bits with the sign bit set, then the low 32 bits. + Int_t high = static_cast((n >> 32) | 0x80000000u); + b.WriteInt(high); + b.WriteUInt(static_cast(n & 0xffffffffu)); + } +} + +} // namespace TStreamerInfoUtils + +#endif // ROOT_TStreamerInfoCollectionUtils diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 63c521b993b1e..4042c0a99a80a 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -13,6 +13,7 @@ #include "TFile.h" #include "TClass.h" #include "TBufferFile.h" +#include "TStreamerInfoCollectionUtils.h" #include "TClonesArray.h" #include "TError.h" #include "TRef.h" @@ -1301,20 +1302,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, for(; obj> nobjects; - ULong64_t nobjects64; - if (std::signbit(nobjects) && vers >= 11) { - nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; - UInt_t nobjectsLow; - b >> nobjectsLow; - nobjects64 |= nobjectsLow; - } else { - nobjects64 = nobjects; - } - R__ASSERT(nobjects64 == nobjects && "Allocate is not yet 64 bit enabled"); + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + R__ASSERT(nobjects64 <= (ULong64_t)std::numeric_limits::max() && "Allocate is not yet 64 bit enabled"); void* env = newProxy->Allocate(nobjects64, true); - if (!nobjects && (vers>=7)) { + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { From 18c6617f66b46991c64d9d71b5f8aa79aa517834 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 18:08:26 -0500 Subject: [PATCH 16/37] io: Extent TVirtualCollection to 64bits. NOTE: this might be backward incompatible --- core/cont/inc/TVirtualCollectionProxy.h | 2 +- io/io/inc/TEmulatedCollectionProxy.h | 10 +++++----- io/io/inc/TEmulatedMapProxy.h | 4 ++-- io/io/inc/TGenCollectionProxy.h | 10 +++++----- io/io/src/TEmulatedCollectionProxy.cxx | 8 ++++---- io/io/src/TEmulatedMapProxy.cxx | 4 ++-- io/io/src/TGenCollectionProxy.cxx | 20 ++++++++++---------- tree/ntuple/test/CustomStruct.hxx | 8 ++++---- tree/ntuple/test/SimpleCollectionProxy.hxx | 8 ++++---- tree/ntuple/test/ntuple_emulated.cxx | 8 ++++---- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 98d61c8b01c55..ec3d3e7a76cce 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -71,7 +71,7 @@ class TVirtualCollectionProxy { kCustomAlloc = BIT(5) ///< The collection has a custom allocator. }; - using size_type = UInt_t; + using size_type = ULong64_t; /// RAII helper class that ensures that `PushProxy()` / `PopProxy()` are called when entering / leaving a C++ context class TPushPop { diff --git a/io/io/inc/TEmulatedCollectionProxy.h b/io/io/inc/TEmulatedCollectionProxy.h index 0e0fa09b5a71a..f90a2e3e5ed47 100644 --- a/io/io/inc/TEmulatedCollectionProxy.h +++ b/io/io/inc/TEmulatedCollectionProxy.h @@ -191,22 +191,22 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { void DeleteArray(void* p, Bool_t dtorOnly = kFALSE) const override; // TVirtualCollectionProxy overload: Return the sizeof the collection object. - UInt_t Sizeof() const override { return sizeof(Cont_t); } + size_type Sizeof() const override { return sizeof(Cont_t); } // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container void Clear(const char *opt = "") override; // Resize the container - void Resize(UInt_t n, Bool_t force_delete) override; + void Resize(size_type n, Bool_t force_delete) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Block commit of containees void Commit(void* env) override; diff --git a/io/io/inc/TEmulatedMapProxy.h b/io/io/inc/TEmulatedMapProxy.h index 48e990d85e86d..232a7b1bbd27f 100644 --- a/io/io/inc/TEmulatedMapProxy.h +++ b/io/io/inc/TEmulatedMapProxy.h @@ -38,10 +38,10 @@ class TEmulatedMapProxy : public TEmulatedCollectionProxy { ~TEmulatedMapProxy() override; // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Read portion of the streamer void ReadBuffer(TBuffer &buff, void *pObj) override; diff --git a/io/io/inc/TGenCollectionProxy.h b/io/io/inc/TGenCollectionProxy.h index 162b88c33a031..d70749e75396f 100644 --- a/io/io/inc/TGenCollectionProxy.h +++ b/io/io/inc/TGenCollectionProxy.h @@ -384,7 +384,7 @@ class TGenCollectionProxy ULong_t GetIncrement() const override; // Return the sizeof the collection object. - UInt_t Sizeof() const override; + size_type Sizeof() const override; // Push new proxy environment. void PushProxy(void *objstart) override; @@ -402,19 +402,19 @@ class TGenCollectionProxy EDataType GetType() const override; // Return the address of the value at index 'idx'. - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container. void Clear(const char *opt = "") override; // Resize the container. - virtual void Resize(UInt_t n, Bool_t force_delete); + virtual void Resize(size_type n, Bool_t force_delete); // Return the current size of the container. - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees. - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Insert data into the container where data is a C-style array of the actual type contained in the collection // of the given size. For associative container (map, etc.), the data type is the pair. diff --git a/io/io/src/TEmulatedCollectionProxy.cxx b/io/io/src/TEmulatedCollectionProxy.cxx index 883dab626fd3a..17eea7631e9f3 100644 --- a/io/io/src/TEmulatedCollectionProxy.cxx +++ b/io/io/src/TEmulatedCollectionProxy.cxx @@ -248,7 +248,7 @@ Bool_t TEmulatedCollectionProxy::IsValid() const return (0 != fCreateEnv.call); } -UInt_t TEmulatedCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedCollectionProxy::Size() const { // Return the current size of the container @@ -495,7 +495,7 @@ void TEmulatedCollectionProxy::Expand(UInt_t nCurr, UInt_t left, Bool_t force) } } -void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) +void TEmulatedCollectionProxy::Resize(TVirtualCollectionProxy::size_type left, Bool_t force) { // Resize the container @@ -516,7 +516,7 @@ void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) Fatal("TEmulatedCollectionProxy","Resize> Logic error - no proxy object set."); } -void* TEmulatedCollectionProxy::At(UInt_t idx) +void* TEmulatedCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx' if ( fEnv && fEnv->fObject ) { @@ -531,7 +531,7 @@ void* TEmulatedCollectionProxy::At(UInt_t idx) return 0; } -void* TEmulatedCollectionProxy::Allocate(UInt_t n, Bool_t forceDelete) +void* TEmulatedCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t forceDelete) { // Allocate the necessary space. diff --git a/io/io/src/TEmulatedMapProxy.cxx b/io/io/src/TEmulatedMapProxy.cxx index d1ec2d546bd55..6fe1932ba18de 100644 --- a/io/io/src/TEmulatedMapProxy.cxx +++ b/io/io/src/TEmulatedMapProxy.cxx @@ -66,7 +66,7 @@ TVirtualCollectionProxy* TEmulatedMapProxy::Generate() const return new TEmulatedMapProxy(*this); } -void* TEmulatedMapProxy::At(UInt_t idx) +void* TEmulatedMapProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx'. if ( fEnv && fEnv->fObject ) { @@ -77,7 +77,7 @@ void* TEmulatedMapProxy::At(UInt_t idx) return 0; } -UInt_t TEmulatedMapProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedMapProxy::Size() const { // Return the current size of the container. if ( fEnv && fEnv->fObject ) { diff --git a/io/io/src/TGenCollectionProxy.cxx b/io/io/src/TGenCollectionProxy.cxx index eb974f5dd6e69..1612e89be82c7 100644 --- a/io/io/src/TGenCollectionProxy.cxx +++ b/io/io/src/TGenCollectionProxy.cxx @@ -47,7 +47,7 @@ class TGenVectorProxy : public TGenCollectionProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { fEnv->fIdx = idx; @@ -97,7 +97,7 @@ class TGenVectorBoolProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -143,7 +143,7 @@ class TGenBitsetProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -198,7 +198,7 @@ class TGenListProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { switch( idx ) { @@ -240,7 +240,7 @@ class TGenSetProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { if ( fEnv->fUseTemp ) { @@ -1015,7 +1015,7 @@ ULong_t TGenCollectionProxy::GetIncrement() const { //////////////////////////////////////////////////////////////////////////////// /// Return the sizeof the collection object. -UInt_t TGenCollectionProxy::Sizeof() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Sizeof() const { return fClass->Size(); } @@ -1065,7 +1065,7 @@ EDataType TGenCollectionProxy::GetType() const //////////////////////////////////////////////////////////////////////////////// /// Return the address of the value at index 'idx' -void* TGenCollectionProxy::At(UInt_t idx) +void* TGenCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { if ( fEnv && fEnv->fObject ) { switch (fSTL_type) { @@ -1154,7 +1154,7 @@ void TGenCollectionProxy::Clear(const char* opt) //////////////////////////////////////////////////////////////////////////////// /// Return the current size of the container -UInt_t TGenCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Size() const { if ( fEnv && fEnv->fObject ) { if (fEnv->fUseTemp) { @@ -1170,7 +1170,7 @@ UInt_t TGenCollectionProxy::Size() const //////////////////////////////////////////////////////////////////////////////// /// Resize the container -void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) +void TGenCollectionProxy::Resize(TVirtualCollectionProxy::size_type n, Bool_t force) { if ( fEnv && fEnv->fObject ) { if ( force && fPointers ) { @@ -1193,7 +1193,7 @@ void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) /// For associative collection, this returns a TStaging object that /// need to be deleted manually __or__ returned by calling Commit(TStaging*) -void* TGenCollectionProxy::Allocate(UInt_t n, Bool_t /* forceDelete */ ) +void* TGenCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t /* forceDelete */ ) { if ( fEnv && fEnv->fObject ) { switch ( fSTL_type ) { diff --git a/tree/ntuple/test/CustomStruct.hxx b/tree/ntuple/test/CustomStruct.hxx index 0cde0ccb3e091..e5b53bd1b1db8 100644 --- a/tree/ntuple/test/CustomStruct.hxx +++ b/tree/ntuple/test/CustomStruct.hxx @@ -362,16 +362,16 @@ public: TVirtualCollectionProxy *Generate() const final { return new CyclicCollectionProxy(); } Int_t GetCollectionType() const final { return 0; } ULong_t GetIncrement() const final { return 0; } - UInt_t Sizeof() const final { return 0; } + size_type Sizeof() const final { return 0; } bool HasPointers() const final { return false; } TClass *GetValueClass() const final; EDataType GetType() const final { return EDataType::kOther_t; } void PushProxy(void *) final {} void PopProxy() final {} - void *At(UInt_t) final { return nullptr; } + void *At(size_type) final { return nullptr; } void Clear(const char * = "") final {} - UInt_t Size() const final { return 0; } - void *Allocate(UInt_t, bool) final { return nullptr; } + size_type Size() const final { return 0; } + void *Allocate(size_type, bool) final { return nullptr; } void Commit(void *) final {} void Insert(const void *, void *, size_t) final {} TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *, Int_t) final { return nullptr; } diff --git a/tree/ntuple/test/SimpleCollectionProxy.hxx b/tree/ntuple/test/SimpleCollectionProxy.hxx index 6d152f76c8482..bbf2ca6c4cdd1 100644 --- a/tree/ntuple/test/SimpleCollectionProxy.hxx +++ b/tree/ntuple/test/SimpleCollectionProxy.hxx @@ -55,7 +55,7 @@ public: } Int_t GetCollectionType() const override { return CollectionKind; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -80,10 +80,10 @@ public: void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; diff --git a/tree/ntuple/test/ntuple_emulated.cxx b/tree/ntuple/test/ntuple_emulated.cxx index f8858a296c972..c432ef287e27f 100644 --- a/tree/ntuple/test/ntuple_emulated.cxx +++ b/tree/ntuple/test/ntuple_emulated.cxx @@ -551,7 +551,7 @@ TEST(RNTupleEmulated, CollectionProxy) } Int_t GetCollectionType() const override { return ROOT::kSTLvector; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -574,10 +574,10 @@ TEST(RNTupleEmulated, CollectionProxy) void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; From 45754a82b7c361a3173a813eedd0d4c84ecf5eae Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 10 Apr 2026 15:48:57 -0500 Subject: [PATCH 17/37] io: TStreamerInfo version to 11, 64bits collection size --- io/io/inc/TStreamerInfo.h | 2 +- io/io/src/TStreamerInfoActions.cxx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/io/io/inc/TStreamerInfo.h b/io/io/inc/TStreamerInfo.h index ffe008fd308b0..07850cbe5a795 100644 --- a/io/io/inc/TStreamerInfo.h +++ b/io/io/inc/TStreamerInfo.h @@ -253,7 +253,7 @@ class TStreamerInfo : public TVirtualStreamerInfo { Int_t WriteBufferAux (TBuffer &b, const T &arr, TCompInfo *const*const compinfo, Int_t first, Int_t last, Int_t narr,Int_t eoffset,Int_t mode); //WARNING this class version must be the same as TVirtualStreamerInfo - ClassDefOverride(TStreamerInfo, 10) // Streamer information for one class version + ClassDefOverride(TStreamerInfo, 11) // Streamer information for one class version }; diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index d0b5b5f90fe90..f159ca3dd925c 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -2256,8 +2256,8 @@ namespace TStreamerInfoActions UInt_t start = buf.WriteVersion(config->fInfo->IsA(), kTRUE); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues = vec->size(); - buf.WriteInt(nvalues); + auto nvalues = vec->size(); + TStreamerInfoUtils::WriteCollectionSize(buf, nvalues); T *begin = vec->data(); buf.WriteFastArray(begin, nvalues); From 38e1a06c176c4ba94b634158cc39368ac64da279 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 8 Apr 2026 10:31:20 -0500 Subject: [PATCH 18/37] io: Address warning after signature changes --- io/io/src/TBufferFile.cxx | 16 ++++++++-------- io/io/src/TBufferText.cxx | 12 ++++++------ io/io/src/TStreamerInfoActions.cxx | 6 +++--- io/io/src/TStreamerInfoWriteBuffer.cxx | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index 0ed933801bf92..9e5c455308afa 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -3785,7 +3785,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); CheckByteCount(start, count, onFileClass); return 0; @@ -3801,7 +3801,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length() ); CheckByteCount(start, count, cl); return 0; @@ -3832,7 +3832,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio CheckByteCount(start, count, cl); return 0; } else { - Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); CheckByteCount(start, count, cl); return 0; @@ -3895,7 +3895,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); --current; CheckByteCount(R__s, R__c, onFileClass); @@ -3920,7 +3920,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass Int_t infocapacity = infos->Capacity(); if (infocapacity) { if (version < -1 || version >= infocapacity) { - Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); --current; CheckByteCount(R__s, R__c, cl); @@ -3985,7 +3985,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass CheckByteCount(R__s, R__c, cl); return 0; } else { - Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); --current; CheckByteCount(R__s, R__c, cl); @@ -4000,7 +4000,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (sinfo->TStreamerInfo::IsRecovered()) R__c=0; // 'TStreamerInfo::' avoids going via a virtual function. if (current != fByteCountStack.size()) - Fatal("ReadClassBuffer", "We are out of sync at level %lld vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); + Fatal("ReadClassBuffer", "We are out of sync at level %zu vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); --current; // Check that the buffer position corresponds to the byte count. CheckByteCount(R__s, R__c, cl); @@ -4074,7 +4074,7 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se (*iter).PrintDebug(*this,obj); (*iter)(*this,obj); if (current != fByteCountStack.size()) - Fatal("ApplySequence", "We are out of sync starting at level %lld vs %zu", current, fByteCountStack.size()); + Fatal("ApplySequence", "We are out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } else { diff --git a/io/io/src/TBufferText.cxx b/io/io/src/TBufferText.cxx index 0afac88799087..0906b9479d13b 100644 --- a/io/io/src/TBufferText.cxx +++ b/io/io/src/TBufferText.cxx @@ -250,7 +250,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(start, count, onFileClass); return 0; @@ -266,7 +266,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(start, count, cl); return 0; @@ -295,7 +295,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(start, count, cl); return 0; @@ -353,7 +353,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, onFileClass); return 0; @@ -378,7 +378,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (infocapacity) { if (version < -1 || version >= infocapacity) { Error("ReadClassBuffer", - "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(R__s, R__c, cl); return 0; @@ -441,7 +441,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, cl); return 0; diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index f159ca3dd925c..2c4b011885716 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -120,7 +120,7 @@ namespace TStreamerInfoActions aElement->GetSequenceType(sequenceType); printf("StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d (%s)\n", + " %s, bufpos=%lu, arr=%p, offset=%d (%s)\n", info->GetClass()->GetName(), aElement->GetName(), fElemId, fCompInfo->fType, aElement->ClassName(), buf.Length(), addr, fOffset, sequenceType.Data()); } @@ -1268,7 +1268,7 @@ namespace TStreamerInfoActions { if (gDebug > 1) { TStreamerInfo *info = (TStreamerInfo*)fInfo; - printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%d, arr=%p, offset=%d, onfileObject=%p\n", + printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%lu, arr=%p, offset=%d, onfileObject=%p\n", info->GetClass()->GetName(), fOnfileObject ? "Push" : "Pop", buffer.Length(), object, fOffset, fOnfileObject); } @@ -1344,7 +1344,7 @@ namespace TStreamerInfoActions TStreamerInfo *info = (TStreamerInfo*)fInfo; TStreamerElement *aElement = fCompInfo->fElem; fprintf(stdout,"StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", info->GetClass()->GetName(),aElement->GetName(),fElemId,fCompInfo->fType, aElement->ClassName(),b.Length(),addr, 0,b.PeekDataCache() ? b.PeekDataCache()->GetObjectAt(0) : 0); } diff --git a/io/io/src/TStreamerInfoWriteBuffer.cxx b/io/io/src/TStreamerInfoWriteBuffer.cxx index 7a62f21ae346b..1b09138fb45ce 100644 --- a/io/io/src/TStreamerInfoWriteBuffer.cxx +++ b/io/io/src/TStreamerInfoWriteBuffer.cxx @@ -151,7 +151,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -161,7 +161,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, not a write rule, skipping.\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, not a write rule, skipping.\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], eoffset); } @@ -175,7 +175,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d, %s, " - "bufpos=%d, arr=%p, offset=%d\n", + "bufpos=%lu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType,aElement->ClassName(), b.Length(),arr[0],ioffset); } From 0c9fdb4d08c0a3dcabadde0f5b857af070060ba1 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Wed, 11 Feb 2026 13:53:16 +0100 Subject: [PATCH 19/37] io: Test long range Read/WriteObjectAny --- io/io/test/testByteCount.cxx | 103 ++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/io/io/test/testByteCount.cxx b/io/io/test/testByteCount.cxx index 07a6966786e56..a4a464f366ea3 100644 --- a/io/io/test/testByteCount.cxx +++ b/io/io/test/testByteCount.cxx @@ -5,7 +5,7 @@ #include -int testByteCount() +int unittestByteCount() { int errors = 0; unsigned int expectedByteCounts = 1; @@ -124,3 +124,104 @@ int testByteCount() std::cerr << "The end.\n"; return errors; } + +struct LargeByteCountsFixture { + std::size_t fSize{0}; + void resize(size_t size) { + fSize = size; + } + void Streamer(TBuffer &b) { + // Bare minimum to trigger the large byte count mechanism, + // we don't care about the content of the data. + if (b.IsReading()) { + b >> fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } else { + b << fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } + } +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class LargeByteCountsFixture-; +#endif + +int readAndCheck(const std::string &msg, TBufferFile &b, size_t expected_size) +{ + // Same as b >> ptr; + auto obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeByteCountsFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back the object\n"; + return 1; + } else { + auto* readFixture = static_cast(obj); + if (readFixture->fSize != expected_size ) { + std::cerr << msg << ": The size of the data vectors do not match the original ones\n"; + delete readFixture; + return 1; + } + delete readFixture; + } + return 0; +} + + +char *DoNothingAllocator(char* input, size_t, size_t) +{ + // We 'could' check that the requested memory in under what we + // preallocated. + return input; +} + +int testReadWriteObjectAny() +{ + int errors = 0; + + // TBufferFile currently reject size larger than 2GB. + // SetBufferOffset does not check against the size, + // so we can provide and use a larger buffer. + std::vector databuffer{}; + databuffer.reserve(8 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 8 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); + + LargeByteCountsFixture fixture; + fixture.resize(100); // Small object, should be written with the regular byte count mechanism. + + auto startPos = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + + errors += readAndCheck("Small object written in regular section", b, 100); + + // Large object, should be written with the large byte count mechanism + b.SetWriteMode(); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in regular section", b, 1024 * 1024 * 256); + + // Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in long range section", b, 1024 * 1024 * 256); + + return errors; +} + + +int testByteCount() +{ + int res = unittestByteCount(); + res += testReadWriteObjectAny(); + return res; +} \ No newline at end of file From e1d57cd9ce1c74a368c5e2833db29e8f8fe19d9a Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 7 Apr 2026 16:53:46 -0500 Subject: [PATCH 20/37] io: Add test for ByteCount for 4GB+ objects --- io/io/test/testByteCount.cxx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/io/io/test/testByteCount.cxx b/io/io/test/testByteCount.cxx index a4a464f366ea3..8c9e71a6c97e0 100644 --- a/io/io/test/testByteCount.cxx +++ b/io/io/test/testByteCount.cxx @@ -182,8 +182,8 @@ int testReadWriteObjectAny() // SetBufferOffset does not check against the size, // so we can provide and use a larger buffer. std::vector databuffer{}; - databuffer.reserve(8 * 1024 * 1024 * 1024ll); - TBufferFile b(TBuffer::kWrite, 8 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); + databuffer.reserve(9 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 9 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); LargeByteCountsFixture fixture; fixture.resize(100); // Small object, should be written with the regular byte count mechanism. @@ -215,6 +215,17 @@ int testReadWriteObjectAny() b.SetBufferOffset(startPos); errors += readAndCheck("Large object written in long range section", b, 1024 * 1024 * 256); + // Very Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 200); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 1024 + 8ll); // a bit more than 4GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Very Large object written in long range section", b, 1024 * 1024 * 1024 + 8); + return errors; } From bc1b03d3639788f20df8609460b7f24604fc360a Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 9 Apr 2026 17:45:53 -0500 Subject: [PATCH 21/37] io: Old StreamerInfo read/write 64 bit collections --- io/io/src/TStreamerInfoReadBuffer.cxx | 10 +++++----- io/io/src/TStreamerInfoWriteBuffer.cxx | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 4042c0a99a80a..65b23432aa4d6 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -1192,10 +1192,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, cont = contp[j]; } TVirtualCollectionProxy::TPushPop helper( newProxy, cont ); - Int_t nobjects; - b >> nobjects; - env = newProxy->Allocate(nobjects,true); - if (!nobjects && (vers>=7)) { + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + + env = newProxy->Allocate(nobjects64,true); + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { @@ -1303,7 +1303,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, for(; obj::max() && "Allocate is not yet 64 bit enabled"); + void* env = newProxy->Allocate(nobjects64, true); if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, diff --git a/io/io/src/TStreamerInfoWriteBuffer.cxx b/io/io/src/TStreamerInfoWriteBuffer.cxx index 1b09138fb45ce..9c37917f8b1fe 100644 --- a/io/io/src/TStreamerInfoWriteBuffer.cxx +++ b/io/io/src/TStreamerInfoWriteBuffer.cxx @@ -17,6 +17,7 @@ #include "TStreamer.h" #include "TStreamerElement.h" #include "TStreamerInfo.h" +#include "TStreamerInfoCollectionUtils.h" #include "TVirtualCollectionProxy.h" #include "TRefTable.h" #include "TFile.h" @@ -523,9 +524,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(int j=0;jfLength;++j) { char *cont = contp[j]; TVirtualCollectionProxy::TPushPop helper( proxy, cont ); - Int_t nobjects = cont ? proxy->Size() : 0; - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = cont ? proxy->Size() : 0; + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; @@ -587,9 +588,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(Int_t j=0; jSize(); - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = proxy->Size(); + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; From c4bc06262fb5bf4115d3d68a35308bdb922401ed Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 23 Apr 2026 13:53:39 -0500 Subject: [PATCH 22/37] io: Add missing CheckByteCount in TConvertClonesArrayToProxy --- io/io/src/TContainerConverters.cxx | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/io/io/src/TContainerConverters.cxx b/io/io/src/TContainerConverters.cxx index 6958ad082e2fc..b74fe8b30f4b6 100644 --- a/io/io/src/TContainerConverters.cxx +++ b/io/io/src/TContainerConverters.cxx @@ -72,9 +72,8 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz Int_t nobjects, dummy; char nch; TString s; - char classv[256]; - void *env; - UInt_t start, bytecount; + char classv[256]; + void *env; R__ASSERT(b.IsReading()); @@ -95,6 +94,9 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz if (size==0) size=1; for(Int_t k=0; kCommit(env); b.CheckByteCount(start, bytecount,TClonesArray::Class()); + if (needAlloc) { + // Balance/check for the ReadClass byte count. + b.CheckByteCount(cntpos, tag, TClonesArray::Class()); + } } } From 97713d640bf44e50d1ba5f7ab4d33c9c62247551 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 24 Apr 2026 15:03:05 -0500 Subject: [PATCH 23/37] io: Adjust file size due to TStreamerInfo version change. The change in TStreamerInfo class version changes the compressed size of the TStreamerInfo record in the file. --- roottest/root/io/filemerger/CMakeLists.txt | 2 +- .../tree/fastcloning/references/execCheckClusterRange.ref32 | 4 ++-- .../references/execCheckClusterRange_zlib_win64.ref | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roottest/root/io/filemerger/CMakeLists.txt b/roottest/root/io/filemerger/CMakeLists.txt index 2c4d05d148051..4ab605b9104b3 100644 --- a/roottest/root/io/filemerger/CMakeLists.txt +++ b/roottest/root/io/filemerger/CMakeLists.txt @@ -346,7 +346,7 @@ if(ZLIB_NG) else() ROOTTEST_ADD_TEST(simple-zlib-compr-level1 MACRO testSimpleFile.C - MACROARG "\"hsimple101.root\",25000,101,415025,25" + MACROARG "\"hsimple101.root\",25000,101,415004,25" FIXTURES_REQUIRED root-io-filemerger-hsimple101-fixture) ROOTTEST_ADD_TEST(simple-zlib-compr-level6 diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 b/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 index d6cb093e57d19..8a1bbe9488ff7 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 @@ -141,7 +141,7 @@ Cluster #47 starts at 1740 and ends at 1769 Cluster #48 starts at 1770 and ends at 1799 ****************************************************************************** *Tree :t1 : * -*Entries : 1800 : Total = 29346 bytes File Size = 17990 * +*Entries : 1800 : Total = 29346 bytes File Size = 17985 * * : : Tree compression factor = 1.59 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20848 * +*Entries : 2400 : Total = 33566 bytes File Size = 20841 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref index 00f9d698c83b4..8a08d9b2bca9a 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref @@ -141,7 +141,7 @@ Cluster #47 starts at 1740 and ends at 1769 Cluster #48 starts at 1770 and ends at 1799 ****************************************************************************** *Tree :t1 : * -*Entries : 1800 : Total = 29346 bytes File Size = 17989 * +*Entries : 1800 : Total = 29346 bytes File Size = 17988 * * : : Tree compression factor = 1.59 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20845 * +*Entries : 2400 : Total = 33566 bytes File Size = 20847 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters From 488c47c21b16606722e779c657515279e5b511c9 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 24 Apr 2026 18:12:44 -0500 Subject: [PATCH 24/37] io: Adjust file size due to TStreamerInfo version change. The change in TStreamerInfo class version changes the compressed size of the TStreamerInfo record in the file. In this test, the file is updated/added-to, consequently the keys/baskets that are written after the first TStreamerInfo record are moved slightly and this changes the compressed size of the TTree object on file. --- .../tree/fastcloning/references/execCheckClusterRange_zlib.ref | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref index 2e65ecc45e778..2ac6f68b65ff9 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20846 * +*Entries : 2400 : Total = 33566 bytes File Size = 20845 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters From 958394c55c93034d2968f2dc9a9e3200d96e2df5 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 1 May 2026 18:46:25 -0500 Subject: [PATCH 25/37] io: std::signbit(int) not avail on Windows --- io/io/src/TStreamerInfoCollectionUtils.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/io/io/src/TStreamerInfoCollectionUtils.h b/io/io/src/TStreamerInfoCollectionUtils.h index be6cad82c80d6..aefae12142a26 100644 --- a/io/io/src/TStreamerInfoCollectionUtils.h +++ b/io/io/src/TStreamerInfoCollectionUtils.h @@ -14,7 +14,6 @@ #include "TBuffer.h" #include "RtypesCore.h" -#include // std::signbit #include // std::numeric_limits namespace TStreamerInfoUtils { @@ -32,7 +31,9 @@ inline ULong64_t ReadCollectionSize(TBuffer &b, Version_t vers) Int_t nobjects; b >> nobjects; ULong64_t nobjects64; - if (std::signbit(nobjects) && vers >= 11) { + // std::signbit is not available (yet?) on Windows. + // Instead, we check if nobjects is negative to determine if the high bit is set. + if (nobjects < 0 && vers >= 11) { nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; UInt_t nobjectsLow; b >> nobjectsLow; From 339376924d751fdf59b7869d8b6e68fb64c363ce Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 4 May 2026 11:58:37 -0500 Subject: [PATCH 26/37] io: Allow to map long range TObjects This is use in particular for TClass to register the upcoming object type. --- io/io/src/TBufferIO.cxx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/io/io/src/TBufferIO.cxx b/io/io/src/TBufferIO.cxx index 4bbf86356e234..bafb7072d62f6 100644 --- a/io/io/src/TBufferIO.cxx +++ b/io/io/src/TBufferIO.cxx @@ -161,7 +161,9 @@ void TBufferIO::InitMap() void TBufferIO::MapObject(const TObject *obj, ULong64_t offset) { - R__ASSERT(offset <= kMaxUInt); + // This is too high, the real limit is kMaxLongRange which is + // currently declared inside TBufferFile.cxx. + R__ASSERT(offset <= kMaxULong64); if (IsWriting()) { if (!fMap) InitMap(); From be6369d1d80457f4250abbd0160f4e4014d10339 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 6 Apr 2026 11:57:14 -0500 Subject: [PATCH 27/37] io: Test very large collections --- io/io/test/CMakeLists.txt | 8 + io/io/test/testLargeCollection.cxx | 442 +++++++++++++++++++++++++++++ io/io/test/testLargeCollection.ref | 8 + 3 files changed, 458 insertions(+) create mode 100644 io/io/test/testLargeCollection.cxx create mode 100644 io/io/test/testLargeCollection.ref diff --git a/io/io/test/CMakeLists.txt b/io/io/test/CMakeLists.txt index 0647127f26203..16af1c15a3be0 100644 --- a/io/io/test/CMakeLists.txt +++ b/io/io/test/CMakeLists.txt @@ -64,4 +64,12 @@ if(NOT _32BIT) MACRO testByteCount.cxx+ OUTREF testByteCount.ref FIXTURES_REQUIRED io-io-tobj-fixture) + + ROOTTEST_COMPILE_MACRO(testLargeCollection.cxx + FIXTURES_SETUP io-io-testcoll-fixture) + + ROOTTEST_ADD_TEST(testLargeCollections + MACRO testLargeCollection.cxx+ + OUTREF testLargeCollection.ref + FIXTURES_REQUIRED io-io-testcoll-fixture) endif() diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx new file mode 100644 index 0000000000000..95a8c4488e6ed --- /dev/null +++ b/io/io/test/testLargeCollection.cxx @@ -0,0 +1,442 @@ +/* Test large collections + + Need to test: + - direct store in a TBufferFile + - store as part of an object + - those 2 things both for numerical type and structs. + - at least one nested test. + - (maybe not) as part of a split TTree. +*/ + +#include "TBufferFile.h" +#include "TClass.h" + +#include +#include +#include + +// ----------------------------------------------------------------------- +// Helper: a non-trivial struct so we exercise the object-array path +// ----------------------------------------------------------------------- +struct DataPoint { + float x{0}, y{0}, z{0}; + bool operator==(const DataPoint &o) const { return x == o.x && y == o.y && z == o.z; } +}; + +// ----------------------------------------------------------------------- +// Helper: an object that owns a large numerical vector and a large struct +// vector, with a minimal hand-written Streamer so we control the layout. +// ----------------------------------------------------------------------- +struct LargeCollectionFixture { + std::vector fFloats; + std::vector fPoints; + +#if 0 + void Streamer(TBuffer &b) + { + if (b.IsReading()) { + Long64_t nf = 0, np = 0; + b >> nf; + fFloats.resize(nf); + b.ReadFastArray(fFloats.data(), nf); + b >> np; + fPoints.resize(np); + b.ReadFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } else { + Long64_t nf = fFloats.size(), np = fPoints.size(); + b << nf; + b.WriteFastArray(fFloats.data(), nf); + b << np; + b.WriteFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } + } +#endif +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class DataPoint + ; +#pragma link C++ class LargeCollectionFixture + ; +#endif + +// ----------------------------------------------------------------------- +// A do-nothing reallocator: we pre-allocate the buffer ourselves. +// ----------------------------------------------------------------------- +static char *DoNothingAllocator(char *input, size_t, size_t) +{ + return input; +} + +// ----------------------------------------------------------------------- +// 1. Direct store of a numerical array in a TBufferFile +// ----------------------------------------------------------------------- +int testDirectNumerical() +{ + int errors = 0; + + // Use a pre-allocated 6 GB region so TBuffer never has to realloc. + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + // --- small array (stays in the <2 GB region) --- + const Long64_t smallN = 1000; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = float(i) * 0.1f; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), smallN); + + // --- large array (crosses the 2 GB boundary) --- + // 512 M floats = 2 GB of payload + const Long64_t largeN = 512 * 1024 * 1024ll; + std::vector orig_large(largeN); + for (Long64_t i = 0; i < largeN; ++i) + orig_large[i] = float(i % 65536); + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + b.WriteFastArray(orig_large.data(), largeN); + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectNumerical: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (got != orig_small) { + std::cerr << "testDirectNumerical: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectNumerical: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (got != orig_large) { + std::cerr << "testDirectNumerical: large array content mismatch\n"; + int nprinted = 0; + for (Long64_t i = 0; i < n && nprinted < 10; ++i) { + if (got[i] != orig_large[i]) { + std::cerr << " [" << i << "] expected " << orig_large[i] << " got " << got[i] << '\n'; + ++nprinted; + } + } + ++errors; + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2. Direct store of a struct array in a TBufferFile +// ----------------------------------------------------------------------- +int testDirectStruct() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *pointClass = TClass::GetClass(typeid(DataPoint)); + + // --- small struct array --- + const Long64_t smallN = 500; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), pointClass, smallN); + + // --- large struct array (crosses 2 GB) --- + // ~170 M Points * 12 bytes = ~2 GB + const Long64_t largeN = 170 * 1024 * 1024ll; + std::vector orig_large(largeN); + for (Long64_t i = 0; i < largeN; ++i) + orig_large[i] = {float(i % 1000), float((i + 1) % 1000), float((i + 2) % 1000)}; + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + b.WriteFastArray(orig_large.data(), pointClass, largeN); + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectStruct: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), pointClass, n); + if (got != orig_small) { + std::cerr << "testDirectStruct: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectStruct: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), pointClass, n); + if (got != orig_large) { + std::cerr << "testDirectStruct: large array content mismatch\n"; + ++errors; + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2b. Direct store of std::vector via StreamObject +// ----------------------------------------------------------------------- +int testDirectVector() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *floatVecClass = TClass::GetClass("vector"); + auto *pointVecClass = TClass::GetClass("vector"); + + // --- small std::vector --- + std::vector orig_small_f(1000); + for (int i = 0; i < 1000; ++i) + orig_small_f[i] = float(i) * 0.5f; + + auto startSmallF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_f, floatVecClass); + + // --- large std::vector (512 M entries = 2 GB) --- + std::vector orig_large_f(512 * 1024 * 1024ll); + for (size_t i = 0; i < orig_large_f.size(); ++i) + orig_large_f[i] = float(i % 65536); + + auto startLargeF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_large_f, floatVecClass); + + // --- small std::vector --- + std::vector orig_small_p(500); + for (int i = 0; i < 500; ++i) + orig_small_p[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmallP = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_p, pointVecClass); + + // --- read back small float vector --- + b.SetReadMode(); + b.SetBufferOffset(startSmallF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (got != orig_small_f) { + std::cerr << "testDirectVector: small float vector content mismatch\n"; + ++errors; + } + } + + // --- read back large float vector --- + b.SetBufferOffset(startLargeF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (got != orig_large_f) { + std::cerr << "testDirectVector: large float vector content mismatch\n"; + ++errors; + } + } + + // --- read back small DataPoint vector --- + b.SetBufferOffset(startSmallP); + { + std::vector got; + b.StreamObject(&got, pointVecClass); + if (got != orig_small_p) { + std::cerr << "testDirectVector: small DataPoint vector content mismatch\n"; + ++errors; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 3. Store as part of an object (numerical + struct members) +// ----------------------------------------------------------------------- +static int readAndCheckFixture(const std::string &msg, TBufferFile &b, Long64_t startPos, size_t expectedFloats, + size_t expectedPoints) +{ + b.SetReadMode(); + b.SetBufferOffset(startPos); + auto *obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back object\n"; + return 1; + } + auto *f = static_cast(obj); + int errors = 0; + if (f->fFloats.size() != expectedFloats) { + std::cerr << msg << ": fFloats size mismatch: got " << f->fFloats.size() << " expected " << expectedFloats + << '\n'; + ++errors; + } + if (f->fPoints.size() != expectedPoints) { + std::cerr << msg << ": fPoints size mismatch: got " << f->fPoints.size() << " expected " << expectedPoints + << '\n'; + ++errors; + } + delete f; + return errors; +} + +int testAsPartOfObject() +{ + int errors = 0; + + std::vector raw; + raw.reserve(10 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 10 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + LargeCollectionFixture fixture; + + // --- small object (well within 2 GB region) --- + fixture.fFloats.assign(1000, 1.0f); + fixture.fPoints.assign(500, {1.f, 2.f, 3.f}); + auto startSmall = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("small object", b, startSmall, 1000, 500); + + // --- large object (floats cross 2 GB, written in regular section) --- + b.SetWriteMode(); + fixture.fFloats.assign(2 * 1024 * 1024 * 1024ll, 2.0f); // 2 G of floats (8GB) + fixture.fPoints.assign(100, {4.f, 5.f, 6.f}); + auto startLarge = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in regular section", b, startLarge, 2 * 1024 * 1024 * 1024ll, 100); + + // --- large object written past the 4 GB mark --- + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + fixture.fFloats.assign(256 * 1024 * 1024ll, 3.0f); // 1 GB of floats + fixture.fPoints.assign(200, {7.f, 8.f, 9.f}); + auto startFar = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in long-range section", b, startFar, 256 * 1024 * 1024ll, 200); + + return errors; +} + +// ----------------------------------------------------------------------- +// 4. Nested: a vector of LargeCollectionFixture objects +// ----------------------------------------------------------------------- +int testNested() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *fixtureClass = TClass::GetClass(typeid(LargeCollectionFixture)); + const Long64_t nObjects = 3; + std::vector objs(nObjects); + // Give each fixture a different size so we can verify round-trip. + objs[0].fFloats.assign(100, 1.0f); + objs[0].fPoints.assign(10, {1, 1, 1}); + objs[1].fFloats.assign(200, 2.0f); + objs[1].fPoints.assign(20, {2, 2, 2}); + objs[2].fFloats.assign(300, 3.0f); + objs[2].fPoints.assign(30, {3, 3, 3}); + + auto startPos = b.GetCurrent() - b.Buffer(); + b << nObjects; + b.WriteFastArray(objs.data(), fixtureClass, nObjects); + + b.SetReadMode(); + b.SetBufferOffset(startPos); + { + Long64_t n = 0; + b >> n; + if (n != nObjects) { + std::cerr << "testNested: object count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), fixtureClass, n); + for (Long64_t i = 0; i < n; ++i) { + if (got[i].fFloats != objs[i].fFloats || got[i].fPoints != objs[i].fPoints) { + std::cerr << "testNested: content mismatch at index " << i << '\n'; + ++errors; + } + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// Entry point +// ----------------------------------------------------------------------- +int testLargeCollection() +{ + int errors = 0; + + std::cerr << "testDirectNumerical ...\n"; + errors += testDirectNumerical(); + + std::cerr << "testDirectStruct ...\n"; + errors += testDirectStruct(); + + std::cerr << "testDirectVector ...\n"; + errors += testDirectVector(); + + std::cerr << "testAsPartOfObject ...\n"; + errors += testAsPartOfObject(); + + std::cerr << "testNested ...\n"; + errors += testNested(); + + std::cerr << "Done. errors=" << errors << '\n'; + return errors; +} diff --git a/io/io/test/testLargeCollection.ref b/io/io/test/testLargeCollection.ref new file mode 100644 index 0000000000000..54a710a0334ea --- /dev/null +++ b/io/io/test/testLargeCollection.ref @@ -0,0 +1,8 @@ +Processing io/io/test/testLargeCollection.C... +testDirectNumerical ... +testDirectStruct ... +testDirectVector ... +testAsPartOfObject ... +testNested ... +Done. errors=0 +(int) 0 From abfb40a5c524b665114631fa3bd04b75cf21624c Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 27 Apr 2026 07:40:39 -0500 Subject: [PATCH 28/37] io: increase timeout for testLargeCollections --- io/io/test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/io/io/test/CMakeLists.txt b/io/io/test/CMakeLists.txt index 16af1c15a3be0..05b3672d07746 100644 --- a/io/io/test/CMakeLists.txt +++ b/io/io/test/CMakeLists.txt @@ -69,6 +69,7 @@ if(NOT _32BIT) FIXTURES_SETUP io-io-testcoll-fixture) ROOTTEST_ADD_TEST(testLargeCollections + TIMEOUT 1200 MACRO testLargeCollection.cxx+ OUTREF testLargeCollection.ref FIXTURES_REQUIRED io-io-testcoll-fixture) From 11f57815cbf46ff30ebb9dbb4d37b058531884d8 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Thu, 30 Apr 2026 13:47:53 -0500 Subject: [PATCH 29/37] Speed up testLargeCollection --- io/io/test/testLargeCollection.cxx | 194 ++++++++++++++++++++++------- 1 file changed, 150 insertions(+), 44 deletions(-) diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx index 95a8c4488e6ed..09457dab73fc7 100644 --- a/io/io/test/testLargeCollection.cxx +++ b/io/io/test/testLargeCollection.cxx @@ -11,9 +11,120 @@ #include "TBufferFile.h" #include "TClass.h" +#include +#include #include #include #include +#include + +// Timing helper — writes to a dedicated file, not captured by the ctest driver. +static std::ofstream &timingLog() +{ + static std::ofstream f("/tmp/testLargeCollection_timing.txt"); + return f; +} +static double now_sec() +{ + using namespace std::chrono; + return duration(steady_clock::now().time_since_epoch()).count(); +} + +// Returns the process peak RSS in MB. +// ru_maxrss is bytes on macOS, kilobytes on Linux. +static double peak_rss_mb() +{ + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); +#if defined(__APPLE__) + return ru.ru_maxrss / (1024.0 * 1024.0); +#else + return ru.ru_maxrss / 1024.0; +#endif +} +#define TIME_SUBTEST(timing, label, call) \ + do { \ + std::cerr << (label) << " ...\n"; \ + double _t0 = (timing) ? now_sec() : 0.0; \ + errors += (call); \ + if (timing) { \ + timingLog() << (label) << " done in " << (now_sec() - _t0) \ + << " s peak RSS: " << peak_rss_mb() << " MB\n"; \ + timingLog().flush(); \ + } \ + } while (0) +#define TLOG(timing, msg) do { if (timing) { timingLog() << msg << "\n"; timingLog().flush(); } } while(0) + +// ----------------------------------------------------------------------- +// Spot-check a large array: verify the first/last kSpot elements and +// kSpot elements centred on the 2 GB boundary, rather than comparing +// the entire array element-by-element. +// ----------------------------------------------------------------------- +template +static bool spotCheck(const std::vector &got, const std::vector &expected, const char *tag) +{ + if (got.size() != expected.size()) { + std::cerr << tag << ": size mismatch " << got.size() << " vs " << expected.size() << '\n'; + return false; + } + constexpr Long64_t kSpot = 1024; + const Long64_t n = (Long64_t)got.size(); + const Long64_t boundaryN = (Long64_t)(2LL * 1024 * 1024 * 1024 / sizeof(T)); + bool ok = true; + + auto check = [&](Long64_t i) { + if (i < 0 || i >= n) + return; + if (!(got[i] == expected[i])) { + if (ok) + std::cerr << tag << ": mismatch at index " << i << '\n'; + ok = false; + } + }; + + for (Long64_t i = 0; i < kSpot; ++i) + check(i); // beginning + for (Long64_t i = boundaryN - kSpot; i < boundaryN + kSpot; ++i) + check(i); // around 2 GB boundary + for (Long64_t i = n - kSpot; i < n; ++i) + check(i); // end + + return ok; +} + +// Variant of spotCheck that computes expected values on-the-fly via a +// generator, so the caller can free the source array before read-back. +template +static bool spotCheckFn(const std::vector &got, Long64_t expectedSize, Gen expectedAt, const char *tag) +{ + if ((Long64_t)got.size() != expectedSize) { + std::cerr << tag << ": size mismatch " << got.size() << " vs " << expectedSize << '\n'; + return false; + } + constexpr Long64_t kSpot = 1024; + const Long64_t n = expectedSize; + const Long64_t boundaryN = (Long64_t)(2LL * 1024 * 1024 * 1024 / sizeof(T)); + bool ok = true; + + auto check = [&](Long64_t i) { + if (i < 0 || i >= n) + return; + if (!(got[i] == expectedAt(i))) { + if (ok) + std::cerr << tag << ": mismatch at index " << i << '\n'; + ok = false; + } + }; + + for (Long64_t i = 0; i < kSpot; ++i) + check(i); + for (Long64_t i = boundaryN - kSpot; i < boundaryN + kSpot; ++i) + check(i); + for (Long64_t i = n - kSpot; i < n; ++i) + check(i); + + return ok; +} // ----------------------------------------------------------------------- // Helper: a non-trivial struct so we exercise the object-array path @@ -68,8 +179,9 @@ static char *DoNothingAllocator(char *input, size_t, size_t) // ----------------------------------------------------------------------- // 1. Direct store of a numerical array in a TBufferFile +// Returns the large float vector so testDirectVector can reuse it. // ----------------------------------------------------------------------- -int testDirectNumerical() +int testDirectNumerical(std::vector &orig_large_out) { int errors = 0; @@ -91,14 +203,14 @@ int testDirectNumerical() // --- large array (crosses the 2 GB boundary) --- // 512 M floats = 2 GB of payload - const Long64_t largeN = 512 * 1024 * 1024ll; - std::vector orig_large(largeN); + const Long64_t largeN = 512 * 1024 * 1024ll; + orig_large_out.resize(largeN); for (Long64_t i = 0; i < largeN; ++i) - orig_large[i] = float(i % 65536); + orig_large_out[i] = float(i & 0xFFFF); // 65536 = 2^16; & is faster than % auto startLarge = b.GetCurrent() - b.Buffer(); b << largeN; - b.WriteFastArray(orig_large.data(), largeN); + b.WriteFastArray(orig_large_out.data(), largeN); // --- read back small --- b.SetReadMode(); @@ -130,17 +242,8 @@ int testDirectNumerical() } else { std::vector got(n); b.ReadFastArray(got.data(), n); - if (got != orig_large) { - std::cerr << "testDirectNumerical: large array content mismatch\n"; - int nprinted = 0; - for (Long64_t i = 0; i < n && nprinted < 10; ++i) { - if (got[i] != orig_large[i]) { - std::cerr << " [" << i << "] expected " << orig_large[i] << " got " << got[i] << '\n'; - ++nprinted; - } - } + if (!spotCheck(got, orig_large_out, "testDirectNumerical large")) ++errors; - } } } @@ -150,7 +253,7 @@ int testDirectNumerical() // ----------------------------------------------------------------------- // 2. Direct store of a struct array in a TBufferFile // ----------------------------------------------------------------------- -int testDirectStruct() +int testDirectStruct(bool timing = false) { int errors = 0; @@ -175,12 +278,18 @@ int testDirectStruct() // ~170 M Points * 12 bytes = ~2 GB const Long64_t largeN = 170 * 1024 * 1024ll; std::vector orig_large(largeN); + // Use bitwise AND (power-of-2 period) instead of % 1000 to avoid + // 510 M integer divisions in this fill loop. + { double t0 = now_sec(); for (Long64_t i = 0; i < largeN; ++i) - orig_large[i] = {float(i % 1000), float((i + 1) % 1000), float((i + 2) % 1000)}; + orig_large[i] = {float(i & 0x3FF), float((i + 1) & 0x3FF), float((i + 2) & 0x3FF)}; + TLOG(timing, " testDirectStruct fill: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } auto startLarge = b.GetCurrent() - b.Buffer(); b << largeN; + { double t0 = now_sec(); b.WriteFastArray(orig_large.data(), pointClass, largeN); + TLOG(timing, " testDirectStruct write: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } // --- read back small --- b.SetReadMode(); @@ -211,11 +320,15 @@ int testDirectStruct() ++errors; } else { std::vector got(n); + { double t0 = now_sec(); b.ReadFastArray(got.data(), pointClass, n); - if (got != orig_large) { - std::cerr << "testDirectStruct: large array content mismatch\n"; + TLOG(timing, " testDirectStruct read: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } + if (!spotCheckFn(got, largeN, + [](Long64_t i) { + return DataPoint{float(i & 0x3FF), float((i + 1) & 0x3FF), float((i + 2) & 0x3FF)}; + }, + "testDirectStruct large")) ++errors; - } } } @@ -224,8 +337,10 @@ int testDirectStruct() // ----------------------------------------------------------------------- // 2b. Direct store of std::vector via StreamObject +// Takes the already-built large float vector from testDirectNumerical +// to avoid re-allocating and re-filling 2 GB of data. // ----------------------------------------------------------------------- -int testDirectVector() +int testDirectVector(std::vector &orig_large_f) { int errors = 0; @@ -246,9 +361,7 @@ int testDirectVector() b.StreamObject(&orig_small_f, floatVecClass); // --- large std::vector (512 M entries = 2 GB) --- - std::vector orig_large_f(512 * 1024 * 1024ll); - for (size_t i = 0; i < orig_large_f.size(); ++i) - orig_large_f[i] = float(i % 65536); + // orig_large_f is passed in from testDirectNumerical — no re-allocation needed. auto startLargeF = b.GetCurrent() - b.Buffer(); b.StreamObject(&orig_large_f, floatVecClass); @@ -278,10 +391,10 @@ int testDirectVector() { std::vector got; b.StreamObject(&got, floatVecClass); - if (got != orig_large_f) { - std::cerr << "testDirectVector: large float vector content mismatch\n"; + if (!spotCheckFn(got, 512 * 1024 * 1024ll, + [](Long64_t i) { return float(i & 0xFFFF); }, + "testDirectVector large float")) ++errors; - } } // --- read back small DataPoint vector --- @@ -332,8 +445,8 @@ int testAsPartOfObject() int errors = 0; std::vector raw; - raw.reserve(10 * 1024 * 1024 * 1024ll); - TBufferFile b(TBuffer::kWrite, 10 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, DoNothingAllocator); LargeCollectionFixture fixture; @@ -418,24 +531,17 @@ int testNested() // ----------------------------------------------------------------------- // Entry point // ----------------------------------------------------------------------- -int testLargeCollection() +int testLargeCollection(bool timing = true) { int errors = 0; - std::cerr << "testDirectNumerical ...\n"; - errors += testDirectNumerical(); - - std::cerr << "testDirectStruct ...\n"; - errors += testDirectStruct(); - - std::cerr << "testDirectVector ...\n"; - errors += testDirectVector(); - - std::cerr << "testAsPartOfObject ...\n"; - errors += testAsPartOfObject(); - - std::cerr << "testNested ...\n"; - errors += testNested(); + std::vector sharedLargeFloats; + TIME_SUBTEST(timing, "testDirectNumerical", testDirectNumerical(sharedLargeFloats)); + TIME_SUBTEST(timing, "testDirectStruct", testDirectStruct(timing)); + TIME_SUBTEST(timing, "testDirectVector", testDirectVector(sharedLargeFloats)); + { std::vector{}.swap(sharedLargeFloats); } // release 2 GB before the large-object test + TIME_SUBTEST(timing, "testAsPartOfObject", testAsPartOfObject()); + TIME_SUBTEST(timing, "testNested", testNested()); std::cerr << "Done. errors=" << errors << '\n'; return errors; From b2bd3a374dcdd5a9f16c9839f0c90767549c12a0 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Fri, 1 May 2026 18:38:11 -0500 Subject: [PATCH 30/37] io: Extend large collection test --- io/io/test/testLargeCollection.cxx | 104 +++++++++++++++++++++++++++-- io/io/test/testLargeCollection.ref | 1 + 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx index 09457dab73fc7..66d9ec2f02514 100644 --- a/io/io/test/testLargeCollection.cxx +++ b/io/io/test/testLargeCollection.cxx @@ -444,12 +444,12 @@ int testAsPartOfObject() { int errors = 0; - std::vector raw; - raw.reserve(6 * 1024 * 1024 * 1024ll); - TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, - DoNothingAllocator); + // Let TBufferFile own and manage its own buffer so it can auto-expand freely. + // Start small; it will grow as needed (up to ~8+ GB for the 2G-float large object). + TBufferFile b(TBuffer::kWrite, 1024); - LargeCollectionFixture fixture; + { + LargeCollectionFixture fixture; // --- small object (well within 2 GB region) --- fixture.fFloats.assign(1000, 1.0f); @@ -457,6 +457,8 @@ int testAsPartOfObject() auto startSmall = b.GetCurrent() - b.Buffer(); b.WriteObject(&fixture, false /* cacheReuse */); errors += readAndCheckFixture("small object", b, startSmall, 1000, 500); + } + LargeCollectionFixture fixture; // --- large object (floats cross 2 GB, written in regular section) --- b.SetWriteMode(); @@ -528,10 +530,99 @@ int testNested() return errors; } +// ----------------------------------------------------------------------- +// 5. Minimal reproducer for two related >4 GB TBufferFile bugs: +// +// Bug A (write-side): WriteObject on an object whose serialised content +// crosses the 4 GB mark in the buffer. SetByteCount fires an assert +// because cntpos < 4 GB but fBufCur - fBuffer > 4 GB after streaming. +// Minimal trigger: start writing just below 4 GB, payload > remaining space. +// +// Bug B (read-side): ReadObjectAny on an object whose buffer start +// position > kMaxUInt. MapObject fires "offset <= kMaxUInt" assert. +// Minimal trigger: seek to > 4 GB, WriteObject tiny fixture, ReadObjectAny. +// ----------------------------------------------------------------------- +int testMapObjectLargeOffset() +{ + int errors = 0; + + // Each sub-test uses its own fresh TBufferFile so the class map is clean, + // ensuring every read-back sees a genuine "new class" entry in ReadClass + // and exercises the MapObject path at the relevant buffer offset. + + // --- Bug A: object START is just below 4 GB but the payload (10 M floats = 40 MB) + // pushes fBufCur past kMaxUInt. SetByteCount / WriteObjectClass must handle + // a byte-count position that was recorded below 4 GB while the current + // position is above it. + { + const Long64_t kBufSize = (4LL * 1024 + 128) * 1024 * 1024; // 4.125 GB + std::vector raw(kBufSize, 0); + TBufferFile b(TBuffer::kWrite, kBufSize, raw.data(), false /* don't adopt */, DoNothingAllocator); + + const Long64_t kStartA = 4LL * 1024 * 1024 * 1024 - 256; + b.SetBufferOffset(kStartA); + LargeCollectionFixture fixture; + fixture.fFloats.assign(10 * 1024 * 1024, 1.0f); // 40 MB — crosses the 4 GB boundary + fixture.fPoints.assign(3, {1.f, 2.f, 3.f}); + b.WriteObject(&fixture, false /* cacheReuse */); + + b.SetReadMode(); + b.SetBufferOffset(kStartA); + auto *objA = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!objA) { + std::cerr << "testMapObjectLargeOffset BugA: ReadObjectAny returned null\n"; + ++errors; + } else { + auto *f = static_cast(objA); + if (f->fFloats.size() != 10u * 1024 * 1024 || f->fPoints.size() != 3) { + std::cerr << "testMapObjectLargeOffset BugA: size mismatch\n"; + ++errors; + } + delete f; + } + } + + // --- Bug B: object written and read at a start position entirely above kMaxUInt. + // ReadObjectAny -> ReadClass -> MapObject(cl, startpos+kMapOffset) where + // startpos+kMapOffset > kMaxUInt, hitting R__ASSERT(offset <= kMaxUInt) + // in the TObject* overload of TBufferIO::MapObject. + // A fresh TBufferFile is required so that ReadClass sees a "new class" + // (not already cached) and calls MapObject with the >4 GB offset. + { + const Long64_t kBufSize = (4LL * 1024 + 128) * 1024 * 1024; // 4.125 GB + std::vector raw(kBufSize, 0); + TBufferFile b(TBuffer::kWrite, kBufSize, raw.data(), false /* don't adopt */, DoNothingAllocator); + + const Long64_t kStartB = 4LL * 1024 * 1024 * 1024 + 100; // just above kMaxUInt + b.SetBufferOffset(kStartB); + LargeCollectionFixture fixture; + fixture.fFloats.assign(10, 2.0f); + fixture.fPoints.assign(2, {2.f, 3.f, 4.f}); + b.WriteObject(&fixture, false /* cacheReuse */); + + b.SetReadMode(); + b.SetBufferOffset(kStartB); + auto *objB = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!objB) { + std::cerr << "testMapObjectLargeOffset BugB: ReadObjectAny returned null\n"; + ++errors; + } else { + auto *f = static_cast(objB); + if (f->fFloats.size() != 10 || f->fPoints.size() != 2) { + std::cerr << "testMapObjectLargeOffset BugB: size mismatch\n"; + ++errors; + } + delete f; + } + } + + return errors; +} + // ----------------------------------------------------------------------- // Entry point // ----------------------------------------------------------------------- -int testLargeCollection(bool timing = true) +int testLargeCollection(bool timing = false) { int errors = 0; @@ -541,6 +632,7 @@ int testLargeCollection(bool timing = true) TIME_SUBTEST(timing, "testDirectVector", testDirectVector(sharedLargeFloats)); { std::vector{}.swap(sharedLargeFloats); } // release 2 GB before the large-object test TIME_SUBTEST(timing, "testAsPartOfObject", testAsPartOfObject()); + TIME_SUBTEST(timing, "testMapObjectLargeOffset", testMapObjectLargeOffset()); TIME_SUBTEST(timing, "testNested", testNested()); std::cerr << "Done. errors=" << errors << '\n'; diff --git a/io/io/test/testLargeCollection.ref b/io/io/test/testLargeCollection.ref index 54a710a0334ea..f817fb669e889 100644 --- a/io/io/test/testLargeCollection.ref +++ b/io/io/test/testLargeCollection.ref @@ -3,6 +3,7 @@ testDirectNumerical ... testDirectStruct ... testDirectVector ... testAsPartOfObject ... +testMapObjectLargeOffset ... testNested ... Done. errors=0 (int) 0 From 936a3057acc5ac4743114a265c48f02ac1049ae0 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 4 May 2026 13:11:47 -0500 Subject: [PATCH 31/37] io: add memory info to LargeCollection test --- io/io/test/testLargeCollection.cxx | 69 +++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx index 66d9ec2f02514..880c84bac1402 100644 --- a/io/io/test/testLargeCollection.cxx +++ b/io/io/test/testLargeCollection.cxx @@ -17,6 +17,13 @@ #include #include #include +#if defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#elif defined(_WIN32) +#include +#endif // Timing helper — writes to a dedicated file, not captured by the ctest driver. static std::ofstream &timingLog() @@ -42,6 +49,59 @@ static double peak_rss_mb() return ru.ru_maxrss / 1024.0; #endif } +// Returns the available (free) physical memory in MB. +static double available_mem_mb() +{ +#if defined(__APPLE__) + uint64_t memsize = 0; + size_t len = sizeof(memsize); + // vm.pagesize * vm_stat free pages gives available memory + int pagesize = 0; + size_t plen = sizeof(pagesize); + sysctlbyname("hw.pagesize", &pagesize, &plen, nullptr, 0); + // Use hw.memsize for total; approximate available via sysctl vm.page_free_count + uint64_t pagefree = 0; + size_t pflen = sizeof(pagefree); + if (sysctlbyname("vm.page_free_count", &pagefree, &pflen, nullptr, 0) == 0 && pagesize > 0) + return (pagefree * (double)pagesize) / (1024.0 * 1024.0); + // fallback: total physical memory + sysctlbyname("hw.memsize", &memsize, &len, nullptr, 0); + return memsize / (1024.0 * 1024.0); +#elif defined(__linux__) + struct sysinfo si; + if (sysinfo(&si) == 0) + return (si.freeram * (double)si.mem_unit) / (1024.0 * 1024.0); + return -1.0; +#else + return -1.0; +#endif +} + +// Returns the available (free) swap space in MB. +static double available_swap_mb() +{ +#if defined(__APPLE__) + struct xsw_usage swapinfo; + size_t len = sizeof(swapinfo); + if (sysctlbyname("vm.swapusage", &swapinfo, &len, nullptr, 0) == 0) + return swapinfo.xsu_avail / (1024.0 * 1024.0); + return -1.0; +#elif defined(__linux__) + struct sysinfo si; + if (sysinfo(&si) == 0) + return (si.freeswap * (double)si.mem_unit) / (1024.0 * 1024.0); + return -1.0; +#elif defined(_WIN32) + MEMORYSTATUSEX ms; + ms.dwLength = sizeof(ms); + if (GlobalMemoryStatusEx(&ms)) + return ms.ullAvailPageFile / (1024.0 * 1024.0); + return -1.0; +#else + return -1.0; +#endif +} + #define TIME_SUBTEST(timing, label, call) \ do { \ std::cerr << (label) << " ...\n"; \ @@ -622,10 +682,17 @@ int testMapObjectLargeOffset() // ----------------------------------------------------------------------- // Entry point // ----------------------------------------------------------------------- -int testLargeCollection(bool timing = false) +int testLargeCollection(bool timing = false, bool memoryCheck = false) { int errors = 0; + if (memoryCheck) { + std::cerr << "Available memory at start: " << available_mem_mb() << " MB\n"; + double swap = available_swap_mb(); + if (swap >= 0.0) + std::cerr << "Available swap at start: " << swap << " MB\n"; + } + std::vector sharedLargeFloats; TIME_SUBTEST(timing, "testDirectNumerical", testDirectNumerical(sharedLargeFloats)); TIME_SUBTEST(timing, "testDirectStruct", testDirectStruct(timing)); From 2ae8a7a78762d428b8038ca60020bb0e06758e20 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 5 May 2026 10:17:43 -0500 Subject: [PATCH 32/37] io: Skip large collection test when not enough memory --- cmake/modules/RootTestDriver.cmake | 7 ++++- io/io/test/CMakeLists.txt | 45 +++++++++++++++++++++++++++++- io/io/test/testLargeCollection.cxx | 20 +++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/cmake/modules/RootTestDriver.cmake b/cmake/modules/RootTestDriver.cmake index 6b65c50c62aae..79ff70ea2e1a4 100644 --- a/cmake/modules/RootTestDriver.cmake +++ b/cmake/modules/RootTestDriver.cmake @@ -134,7 +134,12 @@ endif() #---Execute pre-command----------------------------------------------------------------------------- if(PRE) execute_process(COMMAND ${_pre} ${_cwd} RESULT_VARIABLE _rc) - if(_rc) + if(_rc EQUAL 77) + # Exit code 77 from a pre-command means "skip this test". + # Print a sentinel recognised by SKIP_REGULAR_EXPRESSION on the test. + message(STATUS "ROOTTEST_SKIP: pre-command requested skip -- skipping test") + return() + elseif(_rc) message(FATAL_ERROR "pre-command error code : ${_rc}") endif() endif() diff --git a/io/io/test/CMakeLists.txt b/io/io/test/CMakeLists.txt index 05b3672d07746..7d652d968813d 100644 --- a/io/io/test/CMakeLists.txt +++ b/io/io/test/CMakeLists.txt @@ -68,9 +68,52 @@ if(NOT _32BIT) ROOTTEST_COMPILE_MACRO(testLargeCollection.cxx FIXTURES_SETUP io-io-testcoll-fixture) + # Memory-check script: exits 0 when enough RAM+swap is available, 77 (skip) otherwise. + # RootTestDriver.cmake propagates exit code 77 from PRECMD to the test's own exit code, + # and SKIP_RETURN_CODE 77 on the main test turns that into a proper CTest "Skipped". + set(_memcheck_script "${CMAKE_CURRENT_BINARY_DIR}/check_mem_for_large_coll.sh") + file(WRITE ${_memcheck_script} + "#!/bin/sh\n" + "# Exit 77 (CTest SKIP via SKIP_RETURN_CODE) if total RAM+swap < 32 GB.\n" + "# We use TOTAL (not free) because the OS reclaims cached pages on demand.\n" + "THRESHOLD_KB=33554432\n" + "if [ -r /proc/meminfo ]; then\n" + " # Linux: use MemTotal + SwapTotal (kernel will reclaim caches as needed)\n" + " RAM_KB=\$(awk '/^MemTotal:/{print \$2}' /proc/meminfo)\n" + " SWAP_KB=\$(awk '/^SwapTotal:/{print \$2}' /proc/meminfo)\n" + " RAM_KB=\${RAM_KB:-0}\n" + " SWAP_KB=\${SWAP_KB:-0}\n" + " TOTAL_KB=\$(( RAM_KB + SWAP_KB ))\n" + "elif [ \"\$(uname)\" = \"Darwin\" ]; then\n" + " # macOS: total physical RAM via sysctl hw.memsize (bytes -> kB)\n" + " RAM_BYTES=\$(sysctl -n hw.memsize 2>/dev/null)\n" + " RAM_KB=\$(( \${RAM_BYTES:-0} / 1024 ))\n" + " # sysctl output: 'total = NNN.NNM used = ... free = ...'\n" + " SWAP_TOTAL_MB=\$(sysctl -n vm.swapusage 2>/dev/null | sed 's/total = \\([0-9.]*\\)M.*/\\1/')\n" + " if [ -n \"\$SWAP_TOTAL_MB\" ]; then\n" + " SWAP_KB=\$(awk \"BEGIN{printf \\\"%d\\\", \$SWAP_TOTAL_MB * 1024}\")\n" + " else\n" + " SWAP_KB=0\n" + " fi\n" + " TOTAL_KB=\$(( RAM_KB + SWAP_KB ))\n" + "else\n" + " echo \"testLargeCollections memory check: unsupported OS, assuming enough memory\"\n" + " exit 0\n" + "fi\n" + "echo \"testLargeCollections memory check: total \${TOTAL_KB} kB (RAM \${RAM_KB} + swap \${SWAP_KB}), need \${THRESHOLD_KB} kB\"\n" + "if [ \"\$TOTAL_KB\" -lt \"\$THRESHOLD_KB\" ]; then\n" + " echo \"Insufficient memory -- skipping testLargeCollections\"\n" + " exit 77\n" + "fi\n" + "exit 0\n" + ) + execute_process(COMMAND chmod +x ${_memcheck_script}) + ROOTTEST_ADD_TEST(testLargeCollections TIMEOUT 1200 MACRO testLargeCollection.cxx+ OUTREF testLargeCollection.ref - FIXTURES_REQUIRED io-io-testcoll-fixture) + FIXTURES_REQUIRED io-io-testcoll-fixture + PRECMD sh ${_memcheck_script} + PROPERTIES "SKIP_REGULAR_EXPRESSION;ROOTTEST_SKIP") endif() diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx index 880c84bac1402..6e73dc9e6d839 100644 --- a/io/io/test/testLargeCollection.cxx +++ b/io/io/test/testLargeCollection.cxx @@ -16,19 +16,32 @@ #include #include #include +#if !defined(_WIN32) #include +#endif #if defined(__APPLE__) #include #elif defined(__linux__) #include #elif defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include +#include #endif // Timing helper — writes to a dedicated file, not captured by the ctest driver. static std::ofstream &timingLog() { +#if defined(_WIN32) + static std::ofstream f("testLargeCollection_timing.txt"); +#else static std::ofstream f("/tmp/testLargeCollection_timing.txt"); +#endif return f; } static double now_sec() @@ -41,6 +54,12 @@ static double now_sec() // ru_maxrss is bytes on macOS, kilobytes on Linux. static double peak_rss_mb() { +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) + return pmc.PeakWorkingSetSize / (1024.0 * 1024.0); + return -1.0; +#else struct rusage ru; getrusage(RUSAGE_SELF, &ru); #if defined(__APPLE__) @@ -48,6 +67,7 @@ static double peak_rss_mb() #else return ru.ru_maxrss / 1024.0; #endif +#endif } // Returns the available (free) physical memory in MB. static double available_mem_mb() From 4071fccfc19055c8b0a04470e4d867b8ba3753e4 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 28 Apr 2026 16:01:20 -0500 Subject: [PATCH 33/37] tree: Add protection against incorrect leaflist SetBranchAddress --- roottest/root/tree/addresses/CMakeLists.txt | 1 + roottest/root/tree/addresses/execCheckErrors.C | 9 +++++---- roottest/root/tree/addresses/execCheckErrors.ref | 11 ++++++----- tree/tree/src/TTree.cxx | 7 +++++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/roottest/root/tree/addresses/CMakeLists.txt b/roottest/root/tree/addresses/CMakeLists.txt index cf46966388037..b2c55000415f3 100644 --- a/roottest/root/tree/addresses/CMakeLists.txt +++ b/roottest/root/tree/addresses/CMakeLists.txt @@ -238,6 +238,7 @@ ROOTTEST_ADD_TEST(BranchObjSplit0 ROOTTEST_ADD_TEST(CheckErrors MACRO execCheckErrors.C + COPY_TO_BUILDDIR memleak.root OUTREF execCheckErrors.ref) ROOTTEST_ADD_TEST(CMSobj diff --git a/roottest/root/tree/addresses/execCheckErrors.C b/roottest/root/tree/addresses/execCheckErrors.C index a87eb8537712d..9f2cf10d45544 100644 --- a/roottest/root/tree/addresses/execCheckErrors.C +++ b/roottest/root/tree/addresses/execCheckErrors.C @@ -63,11 +63,12 @@ void execCheckErrors(bool what = 0) std::cout << "Float_t ... " << r << std::endl; r = tree->SetBranchAddress("px", pxp); std::cout << "Float_t ... " << r << std::endl; - r = tree->SetBranchAddress("px", &pxp); - std::cout << "Float_t ... " << r << std::endl; - // Int_t ... (should fail) std::cout << std::endl << "ALL should FAIL ... " << std::endl; + // Point to pointer ... (should fail) + r = tree->SetBranchAddress("px", &pxp); + std::cout << "Float_t pointer ... " << r << std::endl; + // Int_t ... (should fail) p = ((TBranch *)-1); r = tree->SetBranchAddress("px", &ix, &p); std::cout << "Int_t ... " << r << std::endl; @@ -83,7 +84,7 @@ void execCheckErrors(bool what = 0) r = tree->SetBranchAddress("px", ixp); std::cout << "Int_t ... " << r << std::endl; r = tree->SetBranchAddress("px", &ixp); - std::cout << "Int_t ... " << r << std::endl; + std::cout << "Int_t pointer ... " << r << std::endl; // TString ... (should fail) std::cout << std::endl << "ALL should FAIL ... " << std::endl; diff --git a/roottest/root/tree/addresses/execCheckErrors.ref b/roottest/root/tree/addresses/execCheckErrors.ref index 5769366af1d0f..4171c8b76e2ce 100644 --- a/roottest/root/tree/addresses/execCheckErrors.ref +++ b/roottest/root/tree/addresses/execCheckErrors.ref @@ -5,9 +5,10 @@ Float_t ... 0 p set to the branch address Float_t ... 0 Float_t ... 0 -Float_t ... 0 ALL should FAIL ... +Error in : The branch "px" is a leaflist branch (data type "Float_t") and requires a direct data address, not a pointer-to-pointer to a "Float_t". Please remove the '&' in the SetBranchAddress call. +Float_t pointer ... -2 Error in : The pointer type given "Int_t" (3) does not correspond to the type needed "Float_t" (5) by the branch: px Int_t ... -2 Error in : The pointer type given "Int_t" (3) does not correspond to the type needed "Float_t" (5) by the branch: px @@ -15,8 +16,8 @@ p set to zero Int_t ... -2 Error in : The pointer type given "Int_t" (3) does not correspond to the type needed "Float_t" (5) by the branch: px Int_t ... -2 -Error in : The pointer type given "Int_t" (3) does not correspond to the type needed "Float_t" (5) by the branch: px -Int_t ... -2 +Error in : The branch "px" is a leaflist branch (data type "Float_t") and requires a direct data address, not a pointer-to-pointer to a "Int_t". Please remove the '&' in the SetBranchAddress call. +Int_t pointer ... -2 ALL should FAIL ... Error in : The pointer type given "TString" does not correspond to the type needed "Float_t" (5) by the branch: px @@ -24,7 +25,7 @@ TString ... -2 Error in : The pointer type given "TString" does not correspond to the type needed "Float_t" (5) by the branch: px p set to zero TString ... -2 -Error in : The pointer type given "TString" does not correspond to the type needed "Float_t" (5) by the branch: px +Error in : The branch "px" is a leaflist branch (data type "Float_t") and requires a direct data address, not a pointer-to-pointer to a "TString". Please remove the '&' in the SetBranchAddress call. TString ... -2 ALL should FAIL ... @@ -33,7 +34,7 @@ TObjString ... -2 Error in : The pointer type given "TObjString" does not correspond to the type needed "Float_t" (5) by the branch: px p set to zero TObjString ... -2 -Error in : The pointer type given "TObjString" does not correspond to the type needed "Float_t" (5) by the branch: px +Error in : The branch "px" is a leaflist branch (data type "Float_t") and requires a direct data address, not a pointer-to-pointer to a "TObjString". Please remove the '&' in the SetBranchAddress call. TObjString ... -2 ALL should FAIL ... diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx index 95ca1e6acad01..2a2e3c3a5bf3c 100644 --- a/tree/tree/src/TTree.cxx +++ b/tree/tree/src/TTree.cxx @@ -2913,6 +2913,13 @@ Int_t TTree::CheckBranchAddressType(TBranch* branch, TClass* ptrClass, EDataType return kInternalError; } bool isBranchElement = branch->InheritsFrom( TBranchElement::Class() ); + if (!isBranchElement && !expectedClass && isptr) { + Error("SetBranchAddress", + "The branch \"%s\" is a leaflist branch (data type \"%s\") and requires a direct data address, " + "not a pointer-to-pointer to a \"%s\". Please remove the '&' in the SetBranchAddress call.", + branch->GetName(), TDataType::GetTypeName(expectedType), ptrClass ? ptrClass->GetName() : TDataType::GetTypeName(datatype)); + return kMismatch; + } if (expectedClass && datatype == kOther_t && ptrClass == nullptr) { if (isBranchElement) { TBranchElement* bEl = (TBranchElement*)branch; From f380d772a70a182cd30f0c681902d9dabe8ccfa0 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 4 May 2026 12:06:56 -0500 Subject: [PATCH 34/37] REVERTME: focus on alma10 arm64 --- .github/workflows/root-ci.yml | 44 ++++------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/.github/workflows/root-ci.yml b/.github/workflows/root-ci.yml index c3dbb65dfac62..b0d6dbd76268a 100644 --- a/.github/workflows/root-ci.yml +++ b/.github/workflows/root-ci.yml @@ -86,15 +86,8 @@ concurrency: jobs: build-macos: - # For any event that is not a PR, the CI will always run. In PRs, the CI - # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. - if: | - (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || - (github.event_name == 'pull_request' && !( - contains(github.event.pull_request.title, '[skip-ci]') || - contains(github.event.pull_request.title, '[skip ci]') || - contains(github.event.pull_request.labels.*.name, 'skip ci') - )) + # Temporarily disabled - only alma10 arm64 is active + if: false permissions: contents: read @@ -240,15 +233,8 @@ jobs: build-windows: - # For any event that is not a PR, the CI will always run. In PRs, the CI - # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. - if: | - (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || - (github.event_name == 'pull_request' && !( - contains(github.event.pull_request.title, '[skip-ci]') || - contains(github.event.pull_request.title, '[skip ci]') || - contains(github.event.pull_request.labels.*.name, 'skip ci') - )) + # Temporarily disabled - only alma10 arm64 is active + if: false permissions: contents: read @@ -394,27 +380,7 @@ jobs: # Common configs: {Release,Debug,RelWithDebInfo) # Build options: https://root.cern/install/build_from_source/#all-build-options include: - - image: fedora43 - - image: fedora44 - - image: alma8 - - image: alma9 - overrides: ["CMAKE_BUILD_TYPE=Debug"] - - image: alma10 - - image: ubuntu22 - python_venv: "/py-venv-3.12/ROOT-CI" - overrides: ["imt=Off", "CMAKE_BUILD_TYPE=Debug"] - - image: ubuntu2404 - overrides: ["CMAKE_BUILD_TYPE=Debug"] - - image: ubuntu2604 - - image: ubuntu2610 - - image: debian13 - overrides: ["dev=ON", "CMAKE_CXX_FLAGS=-Wsuggest-override"] - # Special builds - - image: alma9 - platform_config: alma9-modules_off - is_special: true - property: modules_off - overrides: ["CMAKE_CXX_STANDARD=20"] + # Temporarily only alma10 arm64 is active - image: alma10 is_special: true property: arm64 From c25609a0558c004b78caec26d578572c13ddd1aa Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 4 May 2026 12:09:14 -0500 Subject: [PATCH 35/37] REVERTME: focus on LargeCollection --- .../root-ci-config/buildconfig/alma10-arm64-largetest.txt | 4 ++++ .github/workflows/root-ci.yml | 1 + 2 files changed, 5 insertions(+) create mode 100644 .github/workflows/root-ci-config/buildconfig/alma10-arm64-largetest.txt diff --git a/.github/workflows/root-ci-config/buildconfig/alma10-arm64-largetest.txt b/.github/workflows/root-ci-config/buildconfig/alma10-arm64-largetest.txt new file mode 100644 index 0000000000000..b5862bdfbd932 --- /dev/null +++ b/.github/workflows/root-ci-config/buildconfig/alma10-arm64-largetest.txt @@ -0,0 +1,4 @@ +builtin_vdt=ON +pythia8=ON +r=OFF +ROOT_CTEST_CUSTOM_FLAGS=-R LargeCollection diff --git a/.github/workflows/root-ci.yml b/.github/workflows/root-ci.yml index b0d6dbd76268a..8f44dc1bbaea0 100644 --- a/.github/workflows/root-ci.yml +++ b/.github/workflows/root-ci.yml @@ -382,6 +382,7 @@ jobs: include: # Temporarily only alma10 arm64 is active - image: alma10 + platform_config: alma10-arm64-largetest is_special: true property: arm64 overrides: ["CMAKE_BUILD_TYPE=RelWithDebInfo", "builtin_zlib=ON", "builtin_zstd=ON", "CMAKE_CXX_STANDARD=20"] From 919071a80289c5f2048a0c9d94198cb5333f1122 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Tue, 5 May 2026 11:02:34 -0500 Subject: [PATCH 36/37] Revert "REVERTME: focus on alma10 arm64" This reverts commit 159967c4fbbc6aa57fb556480148fce8b449869c. --- .github/workflows/root-ci.yml | 42 ++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/root-ci.yml b/.github/workflows/root-ci.yml index 8f44dc1bbaea0..6b2fd1243a3b7 100644 --- a/.github/workflows/root-ci.yml +++ b/.github/workflows/root-ci.yml @@ -86,8 +86,15 @@ concurrency: jobs: build-macos: - # Temporarily disabled - only alma10 arm64 is active - if: false + # For any event that is not a PR, the CI will always run. In PRs, the CI + # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. + if: | + (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || + (github.event_name == 'pull_request' && !( + contains(github.event.pull_request.title, '[skip-ci]') || + contains(github.event.pull_request.title, '[skip ci]') || + contains(github.event.pull_request.labels.*.name, 'skip ci') + )) permissions: contents: read @@ -233,8 +240,15 @@ jobs: build-windows: - # Temporarily disabled - only alma10 arm64 is active - if: false + # For any event that is not a PR, the CI will always run. In PRs, the CI + # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. + if: | + (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || + (github.event_name == 'pull_request' && !( + contains(github.event.pull_request.title, '[skip-ci]') || + contains(github.event.pull_request.title, '[skip ci]') || + contains(github.event.pull_request.labels.*.name, 'skip ci') + )) permissions: contents: read @@ -380,7 +394,25 @@ jobs: # Common configs: {Release,Debug,RelWithDebInfo) # Build options: https://root.cern/install/build_from_source/#all-build-options include: - # Temporarily only alma10 arm64 is active + - image: fedora43 + - image: fedora44 + - image: alma8 + - image: alma9 + overrides: ["CMAKE_BUILD_TYPE=Debug"] + - image: alma10 + - image: ubuntu22 + overrides: ["imt=Off", "CMAKE_BUILD_TYPE=Debug"] + - image: ubuntu2404 + overrides: ["CMAKE_BUILD_TYPE=Debug"] + - image: ubuntu2604 + - image: debian13 + overrides: ["dev=ON", "CMAKE_CXX_FLAGS=-Wsuggest-override"] + # Special builds + - image: alma9 + platform_config: alma9-modules_off + is_special: true + property: modules_off + overrides: ["CMAKE_CXX_STANDARD=20"] - image: alma10 platform_config: alma10-arm64-largetest is_special: true From 46afec7089a265cab52b168209f9dad73d633856 Mon Sep 17 00:00:00 2001 From: Philippe Canal Date: Mon, 2 Jun 2025 16:56:46 -0500 Subject: [PATCH 37/37] NOT READY: [tree] Fix handling of poorly cluster file in the retain previous cluster's basket code. Without this all the baskets of a poorly clustered branch are retaining until the deletion of the TTree object --- tree/tree/src/TBranch.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx index ddeb76f2459f5..3f1b00de95601 100644 --- a/tree/tree/src/TBranch.cxx +++ b/tree/tree/src/TBranch.cxx @@ -1991,7 +1991,7 @@ TBasket *TBranch::GetFreshCluster(TBuffer* user_buffer) // basket, just iterate backwards until the correct basket is reached. This should // be fast as long as the number of baskets per cluster is small Int_t basketToUnload = fReadBasket; - while (fBasketEntry[basketToUnload] != entryToUnload) { + while (fBasketEntry[basketToUnload] > entryToUnload) { basketToUnload--; if (basketToUnload < 0) { return CreateOrReuseBasket();