Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/Parquet.Data.Analysis/Extensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ private static async Task<DataFrameColumn> CreateColumnAsync(
if(t == typeof(float)) return new SingleDataFrameColumn(name, await ReadAsync<float>(rgr, field, rowCount, ct));
if(t == typeof(double)) return new DoubleDataFrameColumn(name, await ReadAsync<double>(rgr, field, rowCount, ct));
if(t == typeof(decimal)) return new DecimalDataFrameColumn(name, await ReadAsync<decimal>(rgr, field, rowCount, ct));
if(t == typeof(DateOnly)) return new PrimitiveDataFrameColumn<DateOnly>(name, await ReadAsync<DateOnly>(rgr, field, rowCount, ct));
if(t == typeof(DateTime)) return new DateTimeDataFrameColumn(name, await ReadAsync<DateTime>(rgr, field, rowCount, ct));
if(t == typeof(TimeSpan)) return new PrimitiveDataFrameColumn<TimeSpan>(name, await ReadAsync<TimeSpan>(rgr, field, rowCount, ct));
if(t == typeof(string)) return new StringDataFrameColumn(name, await ReadStringsAsync(rgr, field, rowCount, ct));
Expand All @@ -103,6 +104,7 @@ private static async Task AppendToColumnAsync(
if(t == typeof(float)) { Append((PrimitiveDataFrameColumn<float>)col, await ReadAsync<float>(rgr, field, rowCount, ct)); return; }
if(t == typeof(double)) { Append((PrimitiveDataFrameColumn<double>)col, await ReadAsync<double>(rgr, field, rowCount, ct)); return; }
if(t == typeof(decimal)) { Append((PrimitiveDataFrameColumn<decimal>)col, await ReadAsync<decimal>(rgr, field, rowCount, ct)); return; }
if(t == typeof(DateOnly)) { Append((PrimitiveDataFrameColumn<DateOnly>)col, await ReadAsync<DateOnly>(rgr, field, rowCount, ct)); return; }
if(t == typeof(DateTime)) { Append((PrimitiveDataFrameColumn<DateTime>)col, await ReadAsync<DateTime>(rgr, field, rowCount, ct)); return; }
if(t == typeof(TimeSpan)) { Append((PrimitiveDataFrameColumn<TimeSpan>)col, await ReadAsync<TimeSpan>(rgr, field, rowCount, ct)); return; }
if(t == typeof(string)) { Append((StringDataFrameColumn)col, await ReadStringsAsync(rgr, field, rowCount, ct)); return; }
Expand Down Expand Up @@ -153,6 +155,7 @@ private static async Task WriteColumnAsync(DataField field, DataFrameColumn col,
if(t == typeof(float)) { await rgw.WriteAsync<float>(field, ((PrimitiveDataFrameColumn<float>)col).ToArray().AsMemory()); return; }
if(t == typeof(double)) { await rgw.WriteAsync<double>(field, ((PrimitiveDataFrameColumn<double>)col).ToArray().AsMemory()); return; }
if(t == typeof(decimal)) { await rgw.WriteAsync<decimal>(field, ((PrimitiveDataFrameColumn<decimal>)col).ToArray().AsMemory()); return; }
if(t == typeof(DateOnly)) { await rgw.WriteAsync<DateOnly>(field, ((PrimitiveDataFrameColumn<DateOnly>)col).ToArray().AsMemory()); return; }
if(t == typeof(DateTime)) { await rgw.WriteAsync<DateTime>(field, ((PrimitiveDataFrameColumn<DateTime>)col).ToArray().AsMemory()); return; }
if(t == typeof(TimeSpan)) { await rgw.WriteAsync<TimeSpan>(field, ((PrimitiveDataFrameColumn<TimeSpan>)col).ToArray().AsMemory()); return; }
if(t == typeof(string)) {
Expand Down
12 changes: 6 additions & 6 deletions src/Parquet.Test/ParquetReaderOnTestFilesTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,16 @@ public async Task Datetypes_all(string parquetFile) {
await using ParquetReader r = await ParquetReader.CreateAsync(s);
using ParquetRowGroupReader rgr = r.OpenRowGroupReader(0);

var col0 = new DateTime?[rgr.RowGroup.NumRows];
var col0 = new DateOnly?[rgr.RowGroup.NumRows];
var col1 = new DateTime?[rgr.RowGroup.NumRows];
await rgr.ReadAsync<DateTime>(r.Schema.DataFields[0], col0);
await rgr.ReadAsync<DateOnly>(r.Schema.DataFields[0], col0);
await rgr.ReadAsync<DateTime>(r.Schema.DataFields[1], col1);

DateTime? o0 = col0[0];
DateTime? o1 = col0[1];
DateOnly? o0 = col0[0];
DateOnly? o1 = col0[1];

Assert.Equal(new DateTime(2017, 1, 1), o0);
Assert.Equal(new DateTime(2017, 2, 1), o1);
Assert.Equal(new DateOnly(2017, 1, 1), o0);
Assert.Equal(new DateOnly(2017, 2, 1), o1);
}

[Theory]
Expand Down
1 change: 0 additions & 1 deletion src/Parquet.Test/StatisticsTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public async Task Issue735() {

var parquetOptions = new ParquetOptions {
UseBigDecimal = false,
UseDateOnlyTypeForDates = true,
UseTimeOnlyTypeForTimeMillis = true,
MaximumSmallPoolFreeBytes = 32 * 1024 * 1024,
MaximumLargePoolFreeBytes = 64 * 1024 * 1024,
Expand Down
3 changes: 1 addition & 2 deletions src/Parquet.Test/Types/EndToEndTypeTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -405,8 +405,7 @@ public async Task Type_date_writes_and_reads() {
public async Task Type_dateonly_writes_and_reads() {
var field = new DataField<DateOnly>("dateOnly");
DateOnly expected = DateOnly.FromDateTime(DateTime.UtcNow);
var options = new ParquetOptions { UseDateOnlyTypeForDates = true };
DateOnly actual = await WriteReadSingleAsync(field, expected, options);
DateOnly actual = await WriteReadSingleAsync(field, expected);
Assert.Equal(expected, actual);
}

Expand Down
2 changes: 1 addition & 1 deletion src/Parquet/Encodings/SchemaEncoder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ static bool TryBuildStruct(List<SchemaElement> schema,

// date
if(se.LogicalType?.DATE != null || se.ConvertedType == ConvertedType.DATE) {
return options.UseDateOnlyTypeForDates ? typeof(DateOnly) : typeof(DateTime);
return typeof(DateOnly);
}

// time
Expand Down
8 changes: 1 addition & 7 deletions src/Parquet/ParquetOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,7 @@ public class ParquetOptions {
/// When true, big integers are always treated as dates when reading files.
/// </summary>
public bool TreatBigIntegersAsDates { get; set; } = true;

/// <summary>
/// When set to true, parquet dates will be deserialized as <see cref="DateOnly"/>, otherwise as
/// <see cref="DateTime"/> with missing time part.
/// </summary>
public bool UseDateOnlyTypeForDates { get; set; } = false;


/// <summary>
/// When set to true, parquet times with millisecond precision will be deserialized as <see cref="TimeOnly"/>,
/// otherwise as <see cref="TimeSpan"/> with missing time part.
Expand Down
Loading