Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ zstd = "0.13.3"
rdkafka = { version = "0.36.0", features = ["tokio"] }

[patch.crates-io]
# datafusion: branch=v49.0.0-blaze
# datafusion: branch=v49.0.0-auron
datafusion = { path = "./dev/vendors/packages/datafusion/datafusion/core"}
datafusion-common = { path = "./dev/vendors/packages/datafusion/datafusion/common"}
datafusion-expr = { path = "./dev/vendors/packages/datafusion/datafusion/expr"}
Expand All @@ -214,7 +214,7 @@ datafusion-physical-expr = { path = "./dev/vendors/packages/datafusion/datafusio
datafusion-spark = { path = "./dev/vendors/packages/datafusion/datafusion/spark"}
orc-rust = { path = "./dev/vendors/packages/orc-rust"}

# arrow: branch=v55.2.0-blaze
# arrow: branch=v55.2.0-auron
arrow = { path = "./dev/vendors/packages/arrow-rs/arrow"}
arrow-arith = { path = "./dev/vendors/packages/arrow-rs/arrow-arith"}
arrow-array = { path = "./dev/vendors/packages/arrow-rs/arrow-array"}
Expand All @@ -229,5 +229,5 @@ arrow-select = { path = "./dev/vendors/packages/arrow-rs/arrow-select"}
arrow-string = { path = "./dev/vendors/packages/arrow-rs/arrow-string"}
parquet = { path = "./dev/vendors/packages/arrow-rs/parquet"}

# serde_json: branch=v1.0.96-blaze
# serde_json: branch=v1.0.96-auron
serde_json = { path = "./dev/vendors/packages/serde_json" }
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From ed4ace347da4acfe0c3453e1b34ede067de894e2 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 20:54:37 +0800
Subject: [PATCH] blaze: fix NaN comparison semantic
Subject: [PATCH] auron: fix NaN comparison semantic

---
arrow-array/src/arithmetic.rs | 19 ++++++++++++++++++-
Expand All @@ -16,7 +16,7 @@ index b5f4a106f..c4e88b91a 100644
#[inline]
fn compare(self, rhs: Self) -> Ordering {
- <$t>::total_cmp(&self, &rhs)
+ // blaze:
+ // auron:
+ // nan-safe version according to semantics where NaN == NaN and
+ // NaN is greater than any non-NaN double.
+ if self.is_nan() && rhs.is_nan() {
Expand All @@ -35,7 +35,7 @@ index b5f4a106f..c4e88b91a 100644
// Equivalent to `self.total_cmp(&rhs).is_eq()`
// but LLVM isn't able to realise this is bitwise equality
// https://rust.godbolt.org/z/347nWGxoW
+ // blaze:
+ // auron:
+ // nan-safe version according to semantics where NaN == NaN and
+ // NaN is greater than any non-NaN double.
+ if self.is_nan() && rhs.is_nan() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
From 83e2918853b9519b669a3143293943df9626fd6b Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 21:00:43 +0800
Subject: [PATCH] blaze: add get_dictionary
Subject: [PATCH] auron: add get_dictionary

---
parquet/src/arrow/array_reader/mod.rs | 2 +-
parquet/src/arrow/async_reader/mod.rs | 12 +-
parquet/src/arrow/mod.rs | 4 +-
parquet/src/blaze.rs | 274 ++++++++++++++++++++++++++
parquet/src/auron.rs | 274 ++++++++++++++++++++++++++
parquet/src/column/reader.rs | 2 +-
parquet/src/lib.rs | 4 +
6 files changed, 293 insertions(+), 5 deletions(-)
create mode 100644 parquet/src/blaze.rs
create mode 100644 parquet/src/auron.rs

diff --git a/parquet/src/arrow/array_reader/mod.rs b/parquet/src/arrow/array_reader/mod.rs
index 94d61c9ea..06b06c9de 100644
Expand All @@ -36,9 +36,9 @@ index 611d6999e..248de5be3 100644
) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>>;
+
+ /// Sync version of get_bytes
+ /// this is only used by blaze, for reading dictionary values for row group pruning
+ /// this is only used by auron, for reading dictionary values for row group pruning
+ fn get_bytes_sync(&mut self, range: Range<u64>) -> Result<Bytes> {
+ unimplemented!("blaze only")
+ unimplemented!("auron only")
+ }
}

Expand Down Expand Up @@ -71,11 +71,11 @@ index e33d6a05a..4afcb9464 100644
mod decoder;

#[cfg(feature = "async")]
diff --git a/parquet/src/blaze.rs b/parquet/src/blaze.rs
diff --git a/parquet/src/auron.rs b/parquet/src/auron.rs
new file mode 100644
index 000000000..88ab30d1e
--- /dev/null
+++ b/parquet/src/blaze.rs
+++ b/parquet/src/auron.rs
@@ -0,0 +1,274 @@
+use std::io::Read;
+use std::sync::{Arc, Mutex};
Expand Down Expand Up @@ -285,7 +285,7 @@ index 000000000..88ab30d1e
+ use crate::arrow::arrow_reader::ArrowReaderOptions;
+ use crate::arrow::async_reader::AsyncFileReader;
+ use crate::arrow::ParquetRecordBatchStreamBuilder;
+ use crate::blaze::get_dictionary_for_pruning;
+ use crate::auron::get_dictionary_for_pruning;
+ use crate::errors::ParquetError;
+ use crate::file::footer::{decode_footer, decode_metadata};
+ use crate::file::FOOTER_SIZE;
Expand Down Expand Up @@ -325,7 +325,7 @@ index 000000000..88ab30d1e
+ #[tokio::test]
+ async fn test_get_dictionary() -> crate::errors::Result<()> {
+ let input = AsyncLocalFileReader(
+ File::open("/Volumes/Workspace/blaze-init/arrow-rs/with-dict-2.parquet")?
+ File::open("/Volumes/Workspace/auron-init/arrow-rs/with-dict-2.parquet")?
+ );
+ let mut builder = ParquetRecordBatchStreamBuilder::new_with_options(
+ input,
Expand Down Expand Up @@ -375,7 +375,7 @@ index f814ddeb0..b69d85ba8 100644
+
+#[cfg(feature = "arrow")]
+#[cfg(feature = "async")]
+pub mod blaze;
+pub mod auron;
--
2.50.1

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 0ed0f11556f78301050d71d4c40b77d46ecba52a Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 21:14:08 +0800
Subject: [PATCH] blaze: skip parquet utf-8 validating
Subject: [PATCH] auron: skip parquet utf-8 validating

---
arrow-array/src/types.rs | 40 +++++++++++------------
Expand Down Expand Up @@ -114,7 +114,7 @@ index 5051dce12..802df6ed0 100644
/// [`Self::try_push`] can perform this validation check on insertion
pub fn check_valid_utf8(&self, start_offset: usize) -> Result<()> {
- check_valid_utf8(&self.values.as_slice()[start_offset..])
+ // blaze: ignore utf8 validation
+ // auron: ignore utf8 validation
+ // check_valid_utf8(&self.values.as_slice()[start_offset..])
+ Ok(())
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From ac1cb670bf1d82289c9a7aacca46a438905a8fcd Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 21:21:14 +0800
Subject: [PATCH] blaze: map_field's name fixed to 'entries'
Subject: [PATCH] auron: map_field's name fixed to 'entries'

---
parquet/src/arrow/async_reader/metadata.rs | 6 ++++++
Expand All @@ -18,9 +18,9 @@ index e0f7bdbbe..9a985e885 100644
fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>>;
+
+ /// Sync version of get_bytes
+ /// this is only used by blaze, for reading dictionary values for row group pruning
+ /// this is only used by auron, for reading dictionary values for row group pruning
+ fn get_bytes_sync(&mut self, range: Range<usize>) -> Result<Bytes> {
+ unimplemented!("blaze only")
+ unimplemented!("auron only")
+ }
}

Expand All @@ -34,7 +34,7 @@ index 16d46bd85..af70b8684 100644

let map_field = Field::new_struct(
- map_key_value.name(),
+ "entries", // blaze: map_field's name fixed to 'entries'
+ "entries", // auron: map_field's name fixed to 'entries'
[key_field, value_field],
false, // The inner map field is always non-nullable (#1697)
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 308b3c77cd28c654a84b38df46871cf0a88444f3 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 21:24:51 +0800
Subject: [PATCH] blaze: fix 'Statistics null count is negative (-1)' error
Subject: [PATCH] auron: fix 'Statistics null count is negative (-1)' error

---
parquet/src/file/statistics.rs | 3 ++-
Expand All @@ -17,7 +17,7 @@ index b7522a76f..b7849e4f9 100644
// see https://github.com/apache/arrow-rs/pull/6216/files
- let null_count = stats.null_count.unwrap_or(0);
+ let null_count = stats.null_count.unwrap_or(0)
+ .max(0); // blaze - fix 'Statistics null count is negative (-1)' error
+ .max(0); // auron - fix 'Statistics null count is negative (-1)' error

if null_count < 0 {
return Err(ParquetError::General(format!(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 0f9132f81a3a6d69ed2b893a5bc5d7820f819a4d Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Mon, 16 Sep 2024 00:16:57 +0800
Subject: [PATCH] blaze: expose fields of AsyncReader/ArrowReaderBuilder to
Subject: [PATCH] auron: expose fields of AsyncReader/ArrowReaderBuilder to
public

---
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 417f165b72b99a25cc8f910fde9b43c1d5e06587 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Tue, 19 Nov 2024 10:52:22 +0800
Subject: [PATCH] blaze: use bufreader to read page header, avoid reading in
Subject: [PATCH] auron: use bufreader to read page header, avoid reading in
fragments

---
Expand All @@ -16,10 +16,10 @@ index 9a985e885..2c994b9f0 100644
@@ -87,7 +87,7 @@ pub trait MetadataSuffixFetch: MetadataFetch {

/// Sync version of get_bytes
/// this is only used by blaze, for reading dictionary values for row group pruning
/// this is only used by auron, for reading dictionary values for row group pruning
- fn get_bytes_sync(&mut self, range: Range<usize>) -> Result<Bytes> {
+ fn get_bytes_sync(&mut self, range: Range<u64>) -> Result<Bytes> {
unimplemented!("blaze only")
unimplemented!("auron only")
}
}
diff --git a/parquet/src/bin/parquet-layout.rs b/parquet/src/bin/parquet-layout.rs
Expand All @@ -31,7 +31,7 @@ index 46a231a7d..26e22e1b2 100644
}

- let input = reader.get_read(offset)?;
+ // blaze: use BufReader to avoid reading in fragments
+ // auron: use BufReader to avoid reading in fragments
+ let input = std::io::BufReader::new(reader.get_read(offset)?);
let mut tracked = TrackedRead(input, 0);
let mut prot = TCompactInputProtocol::new(&mut tracked);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 5de02520ca6c9f8f092b4b364906dc0507002cbb Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sat, 21 Dec 2024 02:16:53 +0800
Subject: [PATCH] blaze: parquet supports processing old-style
Subject: [PATCH] auron: parquet supports processing old-style
array<struct<..>> represented as `REPEATED group field_nme {..}`

---
Expand All @@ -20,7 +20,7 @@ index af70b8684..bf497fb77 100644
- "incompatible arrow schema, expected struct got {}",
- d
- ))
+ // blaze: match old-style array<struct<..>> represented as
+ // auron: match old-style array<struct<..>> represented as
+ // `REPEATED group field_nme {..}`
+ let mut old_style_fields = None;
+ if let DataType::List(f) = d {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 0015 Mon Sep 17 00:00:00 2001
From: blaze <blaze@example.com>
From: auron <auron@example.com>
Date: Mon, 27 Jul 2026 02:26:00 +0800
Subject: [PATCH] blaze: update arrow dependency to vendored v55.2.0-blaze
Subject: [PATCH] auron: update arrow dependency to vendored v55.2.0-auron

---
Cargo.toml | 16 ++++++++++++++++
Expand All @@ -17,7 +17,7 @@ index 11cd3c637..b2fbf0df3 100644
unused_qualifications = "deny"
+
+[patch.crates-io]
+# arrow: vendored from v55.2.0-blaze
+# arrow: vendored from v55.2.0-auron
+arrow = { path = "../arrow-rs/arrow" }
+arrow-arith = { path = "../arrow-rs/arrow-arith" }
+arrow-array = { path = "../arrow-rs/arrow-array" }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From a3f528a1ea0d3e64992dbda9d764774fbc6185b0 Mon Sep 17 00:00:00 2001
From: zhangli20 <richselian@gmail.com>
Date: Wed, 13 Aug 2025 11:34:56 +0800
Subject: [PATCH] blaze: exclude testing submodules
Subject: [PATCH] auron: exclude testing submodules

---
.gitmodules | 20 ++++++++++----------
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From feeca02d670231014a2144293331e311494e31c6 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 18:43:15 +0800
Subject: [PATCH] blaze: supports parquet scan with adaptive batch size
Subject: [PATCH] auron: supports parquet scan with adaptive batch size

---
datafusion/datasource-parquet/src/opener.rs | 28 +++++++++++++++++++++
Expand All @@ -15,7 +15,7 @@ index 7c208d142..ffeaaf63c 100644
builder = builder.with_limit(limit)
}

+ // blaze:
+ // auron:
+ // target batch size is usually 1MB ~ 16MB as there are lightweight compressions
+ // like RLE/DICT encodings
+ let adaptive_mem_size = 1048576;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From c9fda37e2cfd77d13698f59f3ff594dd17b38d6a Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 22:59:40 +0800
Subject: [PATCH] blaze: handle special cases when pruning float column with
Subject: [PATCH] auron: handle special cases when pruning float column with
NaN values

---
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From a887b7659f10ec62ed9102aa9a3e9fd5d2e9ea0c Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Sun, 15 Sep 2024 22:02:24 +0800
Subject: [PATCH] blaze: supports parquet dictionary filtering
Subject: [PATCH] auron: supports parquet dictionary filtering

---
datafusion-examples/examples/pruning.rs | 4 +
Expand Down Expand Up @@ -118,7 +118,7 @@ index 51d50d780..982eed293 100644

assert_eq!(groups.len(), self.access_plan.len());
// Indexes of row groups still to scan
+ // blaze: for less reading of dictionary pages, we still prune in separated passes
+ // auron: for less reading of dictionary pages, we still prune in separated passes
let row_group_indexes = self.access_plan.row_group_indexes();
- let row_group_metadatas = row_group_indexes
- .iter()
Expand Down Expand Up @@ -213,7 +213,7 @@ index 51d50d780..982eed293 100644
- fn metadata_iter(&'a self) -> impl Iterator<Item = &'a RowGroupMetaData> + 'a {
- self.row_group_metadatas.iter().copied()
+ fn metadata_iter(&self) -> impl Iterator<Item = &'a RowGroupMetaData> + 'a {
+ // blaze: bypass lifetime checker
+ // auron: bypass lifetime checker
+ let unchecked_self: &'a Self = unsafe {
+ std::mem::transmute(self)
+ };
Expand All @@ -225,7 +225,7 @@ index 51d50d780..982eed293 100644
+ &self,
column: &'b Column,
) -> Result<StatisticsConverter<'a>> {
+ // blaze: bypass lifetime checker
+ // auron: bypass lifetime checker
+ let unchecked_self: &'a Self = unsafe {
+ std::mem::transmute(self)
+ };
Expand All @@ -235,7 +235,7 @@ index 51d50d780..982eed293 100644
- self.parquet_schema,
+ unchecked_self.arrow_schema,
+ unsafe {
+ // blaze: bypass lifetime checker
+ // auron: bypass lifetime checker
+ std::mem::transmute(unchecked_self.builder.borrow_mut().parquet_schema())
+ }
)?)
Expand Down Expand Up @@ -271,7 +271,7 @@ index 51d50d780..982eed293 100644
+ let mut values = vec![];
+ for row_group_metadata in self.metadata_iter() {
+ let dict_values =
+ parquet::blaze::get_dictionary_for_pruning(
+ parquet::auron::get_dictionary_for_pruning(
+ &mut self.builder.borrow_mut().input.0,
+ row_group_metadata,
+ col_idx,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From 534651739be8f95b6dedf34b06fe7c355611af70 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Mon, 16 Sep 2024 10:03:29 +0800
Subject: [PATCH] blaze: allow nullable parquet scan partition key
Subject: [PATCH] auron: allow nullable parquet scan partition key

---
datafusion/datasource/src/file_scan_config.rs | 7 ++++---
Expand All @@ -21,7 +21,7 @@ index 959a2dd33..2606c5387 100644
+ self.table_partition_cols[partition_idx]
+ .as_ref()
+ .clone()
+ .with_nullable(true) // blaze: allow nullable parquet scan partition key
+ .with_nullable(true) // auron: allow nullable parquet scan partition key
}
})
.collect();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From b781bce65fef561bb3e4006c874d8f88291b637d Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Mon, 16 Sep 2024 10:04:54 +0800
Subject: [PATCH] blaze: make scatter() public
Subject: [PATCH] auron: make scatter() public

---
datafusion/physical-expr-common/src/physical_expr.rs | 2 +-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
From da84929cc253eeb4b14b20f6f141145da2a14d05 Mon Sep 17 00:00:00 2001
From: zhangli20 <zhangli20@kuaishou.com>
Date: Mon, 16 Sep 2024 11:15:48 +0800
Subject: [PATCH] blaze: use deprecated min/max values only when min == max
Subject: [PATCH] auron: use deprecated min/max values only when min == max

---
.../src/row_group_filter.rs | 40 +++++++++++++++++--
Expand Down
Loading
Loading