From fec241407468d293242fe09eab597d43fb9d19be Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 3 Aug 2026 19:44:35 -0400 Subject: [PATCH 1/3] add patch and test --- src/Parquet.Test/ParquetReaderOnTestFilesTest.cs | 15 ++++++++++++++- .../data/all_null_column_pyarrow_v25.parquet | Bin 0 -> 2099 bytes src/Parquet/File/DataColumnReader.cs | 13 +++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 src/Parquet.Test/data/all_null_column_pyarrow_v25.parquet diff --git a/src/Parquet.Test/ParquetReaderOnTestFilesTest.cs b/src/Parquet.Test/ParquetReaderOnTestFilesTest.cs index 74d2bdd0..7dcb4d2d 100644 --- a/src/Parquet.Test/ParquetReaderOnTestFilesTest.cs +++ b/src/Parquet.Test/ParquetReaderOnTestFilesTest.cs @@ -356,7 +356,6 @@ public async Task PyArrow23() { Assert.Equal(39_366, rs.Data.Count); } - [Fact] public async Task UnshreddedVariant() { using Stream s = OpenTestFile("variant_unshredded.parquet"); @@ -364,4 +363,18 @@ public async Task UnshreddedVariant() { Assert.NotNull(r.Schema); } + [Fact] + public async Task AllNullColumnPyArrowV25() { + using Stream s = OpenTestFile("all_null_column_pyarrow_v25.parquet"); + await using ParquetReader r = await ParquetReader.CreateAsync(s); + using ParquetRowGroupReader groupReader = r.OpenRowGroupReader(0); + DataField[] fs = r.Schema.GetDataFields(); + double?[] data = await ReadNullableValuesAsync(groupReader, fs[0]); + Assert.Equal(46, data.Length); + Assert.All(data, d => Assert.Null(d)); + + data = await ReadNullableValuesAsync(groupReader, fs[1]); + Assert.Equal(46, data.Length); + Assert.All(data, d => Assert.Equal(0, d)); + } } diff --git a/src/Parquet.Test/data/all_null_column_pyarrow_v25.parquet b/src/Parquet.Test/data/all_null_column_pyarrow_v25.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6d4ddabbc20e00d291b2a9853101b8ae5f5cf895 GIT binary patch literal 2099 zcmb_eOK;;;6u!w+nQ8{1&Pa|D=%TV90;x0PS;%Nu+_;V7&~}=1Y9}F3`IW@2?F2uP zx>cq52_v!LC$M18h80VG13!c%=UiK9GPFCa*7u(GcfNa%+Y{O?aT^@r(ueRO&k4!E z`Z1T?<%ZlAm*FzodqYCV(?{>pu)y{axNmXULoUtSKYdKdXXMZK|NNbTz~|g2cY(Q0 zh+xlh_byAmxJUc!8Mr;&h)bkWLoAYiT_d@SuTM-zdxcJ+hctQpWhP(1_Q+~J4NIJwMBbq&?e{l0O-TdrYVo$FIO zbX=c*we$TAe+fF+@t*3AO@4QWcU8v(g&W#7NPeJt0fGHnB>TumM0IuD{5*5gCh(x*lYA#`TZ(%=Uk;&RMDHPk=_-y7a^!Pj!f z7CI4*B@r{I}wr z{(mTl)T=?@Sy~t{+HU{@uY0B%Og%=EZE2qBMaa@DQax|FKuB}b^R20i+H&n>@nw<6 zxCGFOu>!U-e@k*7(QfyR@+zK`+;5v-;d3?Kr<8W!7oYDat=o1>sm47HuORsb%y4wLAT_o~c7oXE-YMotYT5Of=5%dSl&`UlQt>U#X$FH+h!txkOyQ5`40&pl^p&T+E z@5Am4e68Fp9$$1G!tQ8dePZnQi}6{MY(s2b_IusQl9yJ|ms%@43hhuUA$JerTC?k7 z6YfLvx>xDebl?!q)EYQf^M0>WJ{BZA4>;ceKvc+K0-UmM`kDf{N#7H34tH-GyJ7LfqxhN1y&PNt^fc4 literal 0 HcmV?d00001 diff --git a/src/Parquet/File/DataColumnReader.cs b/src/Parquet/File/DataColumnReader.cs index c1dd270e..031c75c2 100644 --- a/src/Parquet/File/DataColumnReader.cs +++ b/src/Parquet/File/DataColumnReader.cs @@ -66,11 +66,11 @@ public async ValueTask ReadAsync(ReadingColumn rc, CancellationToken cance if(_stats?.NullCount != null) definedValuesCount -= (int)_stats.NullCount.Value; - //using var pc = new PackedColumn(_dataField, totalValuesInChunk, definedValuesCount); long fileOffset = GetFileOffset(); _inputStream.Seek(fileOffset, SeekOrigin.Begin); - while(rc.ValuesRead < totalValuesInChunk) { + bool allNullColumnProcessed = false; + while(rc.ValuesRead < totalValuesInChunk && !allNullColumnProcessed) { PageHeader ph = PageHeader.Read(new ThriftCompactProtocolReader(_inputStream)); switch(ph.Type) { @@ -79,6 +79,7 @@ public async ValueTask ReadAsync(ReadingColumn rc, CancellationToken cance break; case PageType.DATA_PAGE: await ReadDataPageV1Async(ph, rc, cancellationToken); + allNullColumnProcessed = definedValuesCount == 0; break; case PageType.DATA_PAGE_V2: await ReadDataPageV2Async(ph, rc, totalValuesInChunk, cancellationToken); @@ -122,12 +123,16 @@ private long GetFileOffset() => private async ValueTask ReadDataPageV1Async(PageHeader ph, ReadingColumn rc, CancellationToken cancellationToken) where T : struct { using IMemoryOwner bytes = await ReadPageDataAsync(ph); + int allValueCount = (int)_thriftColumnChunk.MetaData!.NumValues; if(ph.DataPageHeader == null) { - throw new ParquetException($"column '{_dataField.Path}' is missing data page header, file is corrupt"); + if (allValueCount != (int?)_stats?.NullCount) { + throw new ParquetException($"column '{_dataField.Path}' is missing data page header, file is corrupt"); + } + rc.MarkValuesRead(allValueCount); + return; } int dataUsed = 0; - int allValueCount = (int)_thriftColumnChunk.MetaData!.NumValues; int pageValueCount = ph.DataPageHeader.NumValues; if(_dataField.MaxRepetitionLevel > 0) { From 98d9a9eeb47a0fef27c6d4b82d93bee1d9be711a Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 3 Aug 2026 19:46:08 -0400 Subject: [PATCH 2/3] comment --- src/Parquet/File/DataColumnReader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parquet/File/DataColumnReader.cs b/src/Parquet/File/DataColumnReader.cs index 031c75c2..c4617041 100644 --- a/src/Parquet/File/DataColumnReader.cs +++ b/src/Parquet/File/DataColumnReader.cs @@ -128,6 +128,7 @@ private async ValueTask ReadDataPageV1Async(PageHeader ph, ReadingColumn r if (allValueCount != (int?)_stats?.NullCount) { throw new ParquetException($"column '{_dataField.Path}' is missing data page header, file is corrupt"); } + // all columns are meant to be null; mark as read and return. rc.MarkValuesRead(allValueCount); return; } From 3ad68049df5982b5af1c2704c2e5424a2806fed7 Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 3 Aug 2026 19:46:21 -0400 Subject: [PATCH 3/3] fix comment --- src/Parquet/File/DataColumnReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parquet/File/DataColumnReader.cs b/src/Parquet/File/DataColumnReader.cs index c4617041..dd45d94d 100644 --- a/src/Parquet/File/DataColumnReader.cs +++ b/src/Parquet/File/DataColumnReader.cs @@ -128,7 +128,7 @@ private async ValueTask ReadDataPageV1Async(PageHeader ph, ReadingColumn r if (allValueCount != (int?)_stats?.NullCount) { throw new ParquetException($"column '{_dataField.Path}' is missing data page header, file is corrupt"); } - // all columns are meant to be null; mark as read and return. + // all values are meant to be null; mark as read and return. rc.MarkValuesRead(allValueCount); return; }