From f6eb96fd813fc3f29dab54262c8d68b2633d303c Mon Sep 17 00:00:00 2001 From: yongman Date: Wed, 19 Aug 2026 15:42:24 +0800 Subject: [PATCH 01/10] columnar: support late materialization Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- contrib/tiflash-columnar-hub/Cargo.lock | 111 +----- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 24 ++ .../hub-runtime/src/columnar_impls.rs | 161 ++++++++- .../hub-runtime/src/interfaces.rs | 43 +++ dbms/src/Flash/Coprocessor/DAGUtils.h | 1 + dbms/src/Interpreters/Settings.h | 1 + .../Storages/StorageDisaggregatedColumnar.cpp | 320 +++++++++++++++++- .../Storages/StorageDisaggregatedColumnar.h | 15 + 9 files changed, 574 insertions(+), 104 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 219b17ddf78..1e882735ec3 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 219b17ddf78dae2b9ae41afeca38dd33f08b9c0c +Subproject commit 1e882735ec30d96a7cfdb4ff6c7bb8040287db02 diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 0ad81c9eae3..8e4740ab765 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -1042,27 +1042,6 @@ dependencies = [ "which", ] -[[package]] -name = "bindgen" -version = "0.65.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex 1.3.0", - "syn 2.0.117", -] - [[package]] name = "bitfield" version = "0.13.2" @@ -1206,16 +1185,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "case_macros" version = "0.1.0" @@ -2893,7 +2862,7 @@ version = "0.10.3+1.44.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23adc509a3c4dea990e0ab8d2add4a65389ee69c288b7851d75dd1df7a6d6c6" dependencies = [ - "bindgen 0.59.2", + "bindgen", "cc", "cmake", "libc", @@ -3749,7 +3718,6 @@ dependencies = [ "rand 0.8.6", "recovery", "regex", - "rocksdb", "rusoto_core", "rusoto_credential", "rusoto_mock", @@ -3886,39 +3854,6 @@ dependencies = [ "libc", ] -[[package]] -name = "librocksdb_sys" -version = "0.1.0" -source = "git+https://github.com/tikv/rust-rocksdb.git#c9aefbe69f89ade0bda6a51b614a46eee184515a" -dependencies = [ - "bindgen 0.65.1", - "bzip2-sys", - "cc", - "cmake", - "libc", - "libtitan_sys", - "libz-sys", - "lz4-sys", - "openssl-sys", - "snappy-sys", - "zstd-sys", -] - -[[package]] -name = "libtitan_sys" -version = "0.0.1" -source = "git+https://github.com/tikv/rust-rocksdb.git#c9aefbe69f89ade0bda6a51b614a46eee184515a" -dependencies = [ - "bzip2-sys", - "cc", - "cmake", - "libc", - "libz-sys", - "lz4-sys", - "snappy-sys", - "zstd-sys", -] - [[package]] name = "libz-sys" version = "1.1.28" @@ -4285,15 +4220,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "mountinfo" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6341a1c48d5425a185c43f75480227dc738cf85ecc6a4bc8ca847406d44f1c17" -dependencies = [ - "regex", -] - [[package]] name = "multimap" version = "0.8.3" @@ -5747,15 +5673,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" -[[package]] -name = "rocksdb" -version = "0.3.0" -source = "git+https://github.com/tikv/rust-rocksdb.git#c9aefbe69f89ade0bda6a51b614a46eee184515a" -dependencies = [ - "libc", - "librocksdb_sys", -] - [[package]] name = "roxmltree" version = "0.14.1" @@ -6481,16 +6398,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "snappy-sys" -version = "0.1.0" -source = "git+https://github.com/tikv/rust-snappy.git?branch=static-link#8c12738bad811397600455d6982aff754ea2ac44" -dependencies = [ - "cmake", - "libc", - "pkg-config", -] - [[package]] name = "socket2" version = "0.4.10" @@ -7315,7 +7222,6 @@ dependencies = [ "libc", "log", "log_wrappers", - "mountinfo", "nix 0.24.3", "num-traits", "num_cpus", @@ -7756,6 +7662,17 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "rand 0.7.3", + "static_assertions", +] + [[package]] name = "txn_types" version = "0.1.0" @@ -8608,6 +8525,7 @@ dependencies = [ "sqlx-mysql", "stable_deref_trait", "standback", + "static_assertions", "strum 0.27.2", "syn 1.0.109", "syn 2.0.117", @@ -8621,9 +8539,12 @@ dependencies = [ "tower 0.4.13", "tracing", "tracing-core", + "twox-hash", "typenum", "url", "uuid 1.23.1", + "zstd", + "zstd-safe", "zstd-sys", ] diff --git a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 4eff355463a..72959feb15f 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -261,6 +261,30 @@ struct CloudStorageEngineInterfaces { ColumnarScanStats (*fn_columnar_scan_stats)(ColumnarReaderPtr); }; +// Optional extension for the two-phase columnar reader protocol. This is +// intentionally discovered through an independently exported, weak symbol so +// that extending it cannot change RaftStoreProxyFFIHelper's ABI. +struct ColumnarLateMaterializationInterfaces { + uint32_t version; + uint32_t size; + uint64_t (*fn_read_early_block)(ColumnarReaderPtr, uint64_t, BaseBuffView, + uint64_t *, int64_t *); + RustStrWithView (*fn_read_early_column)(ColumnarReaderPtr, uint64_t, + int64_t); + uint64_t (*fn_materialize_selected)(ColumnarReaderPtr, uint64_t, uint8_t, + BaseBuffView); + RustStrWithView (*fn_read_late_column)(ColumnarReaderPtr, uint64_t, + int64_t); + uint8_t (*fn_finish_materialized_block)(ColumnarReaderPtr, uint64_t); + uint8_t (*fn_discard_late_materialization_batch)(ColumnarReaderPtr, + uint64_t); +}; + +#if defined(__GNUC__) || defined(__clang__) +extern "C" const ColumnarLateMaterializationInterfaces * + tiflash_columnar_get_late_materialization_interfaces() __attribute__((weak)); +#endif + enum class MsgPBType : uint32_t { ReadIndexResponse = 0, ServerInfoResponse, diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs index 20661db32ce..7a848289bab 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs @@ -14,14 +14,15 @@ use std::convert::TryInto; -use kvengine::{CloudColumnarReaders, TableCtx}; +use kvengine::{CloudColumnarReaders, LateMaterializationSelection, TableCtx}; use protobuf::{parse_from_bytes, Message}; use crate::{ build_from_string, build_from_vec_string, interfaces_ffi::{ - BaseBuffView, ColumnarReaderErrorType, ColumnarReaderPtr, ColumnarScanStats, - RaftStoreProxyPtr, RawRustPtr, RawVoidPtr, RustStrWithView, RustStrWithViewVec, + BaseBuffView, ColumnarLateMaterializationInterfaces, ColumnarReaderErrorType, + ColumnarReaderPtr, ColumnarScanStats, RaftStoreProxyPtr, RawRustPtr, RawVoidPtr, + RustStrWithView, RustStrWithViewVec, }, RawRustPtrType, }; @@ -249,6 +250,160 @@ pub unsafe extern "C" fn ffi_read_column( build_from_string(reader.as_mut().ffi_read_column(col_id)) } +const LM_SELECTION_ALL: u8 = 0; +const LM_SELECTION_NONE: u8 = 1; +const LM_SELECTION_BYTES: u8 = 2; + +pub unsafe extern "C" fn ffi_read_early_block( + mut reader: ColumnarReaderPtr, + limit: u64, + early_column_ids: BaseBuffView, + batch_id: *mut u64, + physical_table_id: *mut i64, +) -> u64 { + if batch_id.is_null() || physical_table_id.is_null() || early_column_ids.len % 8 != 0 { + error!("invalid late-materialization read_early_block arguments"); + return u64::MAX; + } + let early_column_ids = early_column_ids + .to_slice() + .chunks_exact(8) + .map(|bytes| i64::from_le_bytes(bytes.try_into().unwrap())) + .collect::>(); + match reader + .as_mut() + .ffi_read_early_block(limit as usize, &early_column_ids) + { + Ok(Some((id, rows, table_id))) => { + *batch_id = id; + *physical_table_id = table_id; + rows as u64 + } + Ok(None) => 0, + Err(err) => { + error!("ffi_read_early_block failed, limit={}: {}", limit, err); + u64::MAX + } + } +} + +pub unsafe extern "C" fn ffi_read_early_column( + mut reader: ColumnarReaderPtr, + batch_id: u64, + col_id: i64, +) -> RustStrWithView { + match reader.as_mut().ffi_read_early_column(batch_id, col_id) { + Ok(data) => build_from_string(data), + Err(err) => { + error!( + "ffi_read_early_column failed, batch_id={}, col_id={}: {}", + batch_id, col_id, err + ); + RustStrWithView::default() + } + } +} + +pub unsafe extern "C" fn ffi_materialize_selected( + mut reader: ColumnarReaderPtr, + batch_id: u64, + selection_kind: u8, + selection: BaseBuffView, +) -> u64 { + let selection = match selection_kind { + LM_SELECTION_ALL if selection.len == 0 => LateMaterializationSelection::All, + LM_SELECTION_NONE if selection.len == 0 => LateMaterializationSelection::None, + LM_SELECTION_BYTES => LateMaterializationSelection::Bytes(selection.to_slice()), + _ => { + error!("invalid late-materialization selection kind or payload"); + return u64::MAX; + } + }; + match reader + .as_mut() + .ffi_materialize_selected(batch_id, selection) + { + Ok(rows) => rows as u64, + Err(err) => { + error!( + "ffi_materialize_selected failed, batch_id={}: {}", + batch_id, err + ); + u64::MAX + } + } +} + +pub unsafe extern "C" fn ffi_read_late_column( + mut reader: ColumnarReaderPtr, + batch_id: u64, + col_id: i64, +) -> RustStrWithView { + match reader.as_mut().ffi_read_late_column(batch_id, col_id) { + Ok(data) => build_from_string(data), + Err(err) => { + error!( + "ffi_read_late_column failed, batch_id={}, col_id={}: {}", + batch_id, col_id, err + ); + RustStrWithView::default() + } + } +} + +pub unsafe extern "C" fn ffi_finish_materialized_block( + mut reader: ColumnarReaderPtr, + batch_id: u64, +) -> u8 { + match reader.as_mut().ffi_finish_materialized_block(batch_id) { + Ok(()) => 1, + Err(err) => { + error!( + "ffi_finish_materialized_block failed, batch_id={}: {}", + batch_id, err + ); + 0 + } + } +} + +pub unsafe extern "C" fn ffi_discard_late_materialization_batch( + mut reader: ColumnarReaderPtr, + batch_id: u64, +) -> u8 { + match reader + .as_mut() + .ffi_discard_late_materialization_batch(batch_id) + { + Ok(()) => 1, + Err(err) => { + error!( + "ffi_discard_late_materialization_batch failed, batch_id={}: {}", + batch_id, err + ); + 0 + } + } +} + +static LATE_MATERIALIZATION_INTERFACES: ColumnarLateMaterializationInterfaces = + ColumnarLateMaterializationInterfaces { + version: 1, + size: std::mem::size_of::() as u32, + fn_read_early_block: Some(ffi_read_early_block), + fn_read_early_column: Some(ffi_read_early_column), + fn_materialize_selected: Some(ffi_materialize_selected), + fn_read_late_column: Some(ffi_read_late_column), + fn_finish_materialized_block: Some(ffi_finish_materialized_block), + fn_discard_late_materialization_batch: Some(ffi_discard_late_materialization_batch), + }; + +#[no_mangle] +pub extern "C" fn tiflash_columnar_get_late_materialization_interfaces( +) -> *const ColumnarLateMaterializationInterfaces { + &LATE_MATERIALIZATION_INTERFACES +} + pub unsafe extern "C" fn ffi_physical_table_id(mut reader: ColumnarReaderPtr) -> i64 { reader.as_mut().ffi_physical_table_id() } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs index 94d4f123292..f5688545b9e 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs @@ -363,6 +363,49 @@ pub mod root { } #[repr(C)] #[derive(Debug)] + pub struct ColumnarLateMaterializationInterfaces { + pub version: u32, + pub size: u32, + pub fn_read_early_block: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::ColumnarReaderPtr, + arg2: u64, + arg3: root::DB::BaseBuffView, + arg4: *mut u64, + arg5: *mut i64, + ) -> u64, + >, + pub fn_read_early_column: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::ColumnarReaderPtr, + arg2: u64, + arg3: i64, + ) -> root::DB::RustStrWithView, + >, + pub fn_materialize_selected: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::ColumnarReaderPtr, + arg2: u64, + arg3: u8, + arg4: root::DB::BaseBuffView, + ) -> u64, + >, + pub fn_read_late_column: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::ColumnarReaderPtr, + arg2: u64, + arg3: i64, + ) -> root::DB::RustStrWithView, + >, + pub fn_finish_materialized_block: ::std::option::Option< + unsafe extern "C" fn(arg1: root::DB::ColumnarReaderPtr, arg2: u64) -> u8, + >, + pub fn_discard_late_materialization_batch: ::std::option::Option< + unsafe extern "C" fn(arg1: root::DB::ColumnarReaderPtr, arg2: u64) -> u8, + >, + } + #[repr(C)] + #[derive(Debug)] pub struct CloudStorageEngineInterfaces { pub fn_get_keyspace_encryption: ::std::option::Option< unsafe extern "C" fn(arg1: root::DB::RaftStoreProxyPtr, arg2: u32) -> bool, diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.h b/dbms/src/Flash/Coprocessor/DAGUtils.h index 6b49bda44f1..e668bea8f1c 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.h +++ b/dbms/src/Flash/Coprocessor/DAGUtils.h @@ -47,6 +47,7 @@ String getFieldTypeName(Int32 tp); String getJoinExecTypeName(const tipb::JoinExecType & tp); bool isColumnExpr(const tipb::Expr & expr); String getColumnNameForColumnExpr(const tipb::Expr & expr, const std::vector & input_col); +ColumnID getColumnIDForColumnExpr(const tipb::Expr & expr, const std::vector & input_col); void getColumnIDsFromExpr( const tipb::Expr & expr, const std::vector & input_col, diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 4388a78d5d5..b3ca4b75877 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -337,6 +337,7 @@ struct Settings M(SettingInt64, max_buffered_bytes_in_executor, 100LL * 1024 * 1024, "The max buffered size in each executor, 0 mean unlimited, use 100MB as the default value") \ M(SettingDouble, auto_memory_revoke_trigger_threshold, 0.0, "Trigger auto memory revocation when the memory usage is above this percentage.") \ M(SettingUInt64, remote_read_queue_size, 0, "size of remote read queue, 0 means it is determined automatically") \ + M(SettingBool, enable_columnar_l2_late_materialization, false, "Enable L2-only late materialization for cloud columnar reads") \ M(SettingBool, enable_cop_stream_for_remote_read, false, "Enable cop stream for remote read") \ M(SettingUInt64, cop_timeout_for_remote_read, 60, "cop timeout seconds for remote read") \ M(SettingUInt64, auto_spill_check_min_interval_ms, 10, "The minimum interval in millisecond between two successive auto spill check, default value is 100, 0 means no limit") \ diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index d0e1657a8ae..1c80e520a9b 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -21,11 +21,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -126,6 +129,8 @@ struct RNColumnarReaderSharedContext String executor_id; String table_scan_data; String filter_conditions_data; + google::protobuf::RepeatedPtrField exact_filter_conditions; + std::vector scan_columns; String table_info_data; String ann_query_info_data; String fts_query_info_data; @@ -167,6 +172,51 @@ namespace using ColumnarPhysicalTableRanges = std::vector>; using BucketSplitUnit = std::pair; +constexpr UInt32 COLUMNAR_LATE_MATERIALIZATION_ABI_VERSION = 1; + +const ColumnarLateMaterializationInterfaces * getLateMaterializationInterfaces() +{ +#if defined(__GNUC__) || defined(__clang__) + if (tiflash_columnar_get_late_materialization_interfaces == nullptr) + return nullptr; + const auto * interfaces = tiflash_columnar_get_late_materialization_interfaces(); + if (interfaces == nullptr || interfaces->version != COLUMNAR_LATE_MATERIALIZATION_ABI_VERSION + || interfaces->size < sizeof(ColumnarLateMaterializationInterfaces) + || interfaces->fn_read_early_block == nullptr || interfaces->fn_read_early_column == nullptr + || interfaces->fn_materialize_selected == nullptr || interfaces->fn_read_late_column == nullptr + || interfaces->fn_finish_materialized_block == nullptr + || interfaces->fn_discard_late_materialization_batch == nullptr) + return nullptr; + return interfaces; +#else + return nullptr; +#endif +} + +void remapColumnRefsForLateMaterialization( + tipb::Expr & expr, + const std::vector & scan_columns, + const std::unordered_map & early_column_indexes) +{ + if (expr.tp() == tipb::ExprType::ColumnRef) + { + const auto column_id = getColumnIDForColumnExpr(expr, scan_columns); + const auto it = early_column_indexes.find(column_id); + if (it == early_column_indexes.end()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Late-materialization predicate column {} is absent from the early projection", + column_id); + + WriteBufferFromOwnString buffer; + encodeDAGInt64(static_cast(it->second), buffer); + expr.set_val(buffer.releaseStr()); + } + + for (int i = 0; i < expr.children_size(); ++i) + remapColumnRefsForLateMaterialization(*expr.mutable_children(i), scan_columns, early_column_indexes); +} + void normalizeTimestampCompareDateTimeLiteralToUTC(tipb::Expr & expr, const TimezoneInfo & timezone_info); struct BucketSplitResult @@ -407,6 +457,7 @@ std::shared_ptr buildColumnarReaderSharedContext( } std::tie(shared_context->column_defines, shared_context->extra_table_id_index) = genColumnDefinesForDisaggregatedReadThroughColumnar(table_scan); + shared_context->scan_columns = table_scan.getColumns(); auto table_scan_pb = *table_scan.getTableScanPB(); const auto & timezone_info = context.getTimezoneInfo(); @@ -435,6 +486,11 @@ std::shared_ptr buildColumnarReaderSharedContext( shared_context->filter_conditions_data.append(reinterpret_cast(&len), sizeof(len)); shared_context->filter_conditions_data.append(data.data(), data.size()); } + shared_context->exact_filter_conditions = conditions; + const auto & pushed_down_filters = table_scan_pb.tp() == tipb::TypePartitionTableScan + ? table_scan_pb.partition_table_scan().pushed_down_filter_conditions() + : table_scan_pb.tbl_scan().pushed_down_filter_conditions(); + shared_context->exact_filter_conditions.MergeFrom(pushed_down_filters); tipb::TableInfo table_info; bool is_partition_scan = table_scan.isPartitionTableScan(); @@ -932,6 +988,79 @@ const String & RNColumnarReadTask::getExecutorID() const return shared_reader_context->executor_id; } +const google::protobuf::RepeatedPtrField & RNColumnarReadTask::getExactFilterConditions() const +{ + return shared_reader_context->exact_filter_conditions; +} + +google::protobuf::RepeatedPtrField RNColumnarReadTask::getLateMaterializationFilterConditions( + const Block & early_block) const +{ + std::unordered_map early_column_indexes; + early_column_indexes.reserve(early_block.columns()); + for (size_t index = 0; index < early_block.columns(); ++index) + { + const auto [it, inserted] = early_column_indexes.emplace(early_block.getByPosition(index).column_id, index); + if (!inserted) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Late-materialization early projection contains duplicate column ID {} at indexes {} and {}", + it->first, + it->second, + index); + } + + auto conditions = shared_reader_context->exact_filter_conditions; + for (int i = 0; i < conditions.size(); ++i) + remapColumnRefsForLateMaterialization( + *conditions.Mutable(i), + shared_reader_context->scan_columns, + early_column_indexes); + return conditions; +} + +std::unordered_set RNColumnarReadTask::getExactFilterColumnIDs() const +{ + std::unordered_set column_ids; + for (const auto & condition : shared_reader_context->exact_filter_conditions) + getColumnIDsFromExpr(condition, shared_reader_context->scan_columns, column_ids); + return column_ids; +} + +std::unordered_set RNColumnarReadTask::getLateMaterializationEarlyColumnIDs() const +{ + auto column_ids = getExactFilterColumnIDs(); + column_ids.insert(MutSup::extra_handle_id); + column_ids.insert(MutSup::version_col_id); + for (const auto & column : shared_reader_context->scan_columns) + { + if (column.hasPriKeyFlag()) + column_ids.insert(column.id); + } + return column_ids; +} + +bool RNColumnarReadTask::isLateMaterializationFilterEligible() const +{ + if (shared_reader_context->exact_filter_conditions.empty()) + return false; + const auto column_ids = getExactFilterColumnIDs(); + if (column_ids.find(MutSup::extra_table_id_col_id) != column_ids.end()) + return false; + for (const auto & column : shared_reader_context->scan_columns) + { + if (column_ids.find(column.id) == column_ids.end()) + continue; + if (column.hasGeneratedColumnFlag()) + return false; + const bool needs_timezone_cast + = !shared_reader_context->context->getTimezoneInfo().is_utc_timezone && column.tp == TiDB::TypeTimestamp; + if (needs_timezone_cast || column.tp == TiDB::TypeTime) + return false; + } + return true; +} + void RNColumnarReadTask::replaceReaderWork( const RNColumnarReaderWorkPtr & reader_work, std::vector replanned_reader_plans) @@ -1338,6 +1467,7 @@ bool RNColumnarInputStream::ensureReader() { current_reader_work = fixed_reader_work; reader.emplace(task->getOrCreateReader(fixed_reader_work)); + initializeLateMaterialization(); return true; } @@ -1347,9 +1477,37 @@ bool RNColumnarInputStream::ensureReader() current_reader_work = next_reader_work.value(); reader.emplace(task->getOrCreateReader(next_reader_work.value())); + initializeLateMaterialization(); return true; } +void RNColumnarInputStream::initializeLateMaterialization() +{ + if (late_materialization_initialized) + return; + late_materialization_initialized = true; + if (!context.getSettingsRef().enable_columnar_l2_late_materialization + || !task->isLateMaterializationFilterEligible()) + return; + + const auto predicate_column_ids = task->getExactFilterColumnIDs(); + bool has_late_column = false; + for (const auto & column : header) + { + if (column.column_id == MutSup::extra_table_id_col_id || column.column_id == MutSup::extra_handle_id) + continue; + if (predicate_column_ids.find(column.column_id) == predicate_column_ids.end()) + { + has_late_column = true; + break; + } + } + if (!has_late_column) + return; + + late_materialization_interfaces = getLateMaterializationInterfaces(); +} + void RNColumnarInputStream::releaseReader() { mergeReaderStats(); @@ -1357,6 +1515,8 @@ void RNColumnarInputStream::releaseReader() RustGcHelper::instance().gcRustPtr(reader->inner.ptr, reader->inner.type); reader.reset(); current_reader_work.reset(); + late_materialization_interfaces = nullptr; + late_materialization_initialized = false; } void RNColumnarInputStream::mergeReaderStats() @@ -1456,7 +1616,32 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } Stopwatch w{CLOCK_MONOTONIC_COARSE}; - UInt64 rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); + UInt64 batch_id = 0; + TableID physical_table_id = -1; + bool pending_late_materialization = false; + bool use_late_materialization = late_materialization_interfaces != nullptr; + std::unordered_set early_column_ids; + UInt64 rows = 0; + if (use_late_materialization) + { + early_column_ids = task->getLateMaterializationEarlyColumnIDs(); + std::vector encoded_ids(early_column_ids.begin(), early_column_ids.end()); + rows = late_materialization_interfaces->fn_read_early_block( + reader.value(), + batch_size, + BaseBuffView{reinterpret_cast(encoded_ids.data()), encoded_ids.size() * sizeof(Int64)}, + &batch_id, + &physical_table_id); + pending_late_materialization = rows != 0 && rows != std::numeric_limits::max(); + } + else + { + rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); + } + SCOPE_EXIT({ + if (pending_late_materialization) + late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id); + }); duration_read_sec += w.elapsedSecondsFromLastTime(); LOG_DEBUG(log, "Read {} rows from columnar", rows); if (rows == std::numeric_limits::max()) @@ -1479,9 +1664,122 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ // before deserializing more column data into TiFlash memory. CurrentMemoryTracker::checkRssLimit(); - TableID physical_table_id = -1; Block header = getHeader(); const ColumnsWithTypeAndName & col_type_and_name = header.getColumnsWithTypeAndName(); + if (use_late_materialization) + { + Block early_block; + for (const auto & column : col_type_and_name) + { + if (column.column_id == MutSup::extra_table_id_col_id + || early_column_ids.find(column.column_id) == early_column_ids.end()) + continue; + auto col_data + = late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, column.column_id); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + auto mutable_column = column.type->createColumn(); + column.type->deserializeBinaryBulkWithMultipleStreams( + *mutable_column, + [&](const IDataType::SubstreamPath &) { return &buf; }, + rows, + -1.0, + true, + {}); + early_block.insert( + ColumnWithTypeAndName{std::move(mutable_column), column.type, column.name, column.column_id}); + } + + NamesAndTypes early_names_and_types; + early_names_and_types.reserve(early_block.columns()); + for (const auto & column : early_block) + early_names_and_types.emplace_back(column.name, column.type); + DAGExpressionAnalyzer lm_analyzer(std::move(early_names_and_types), context); + auto filter_conditions = task->getLateMaterializationFilterConditions(early_block); + auto filter_actions = lm_analyzer.buildPushDownFilter(filter_conditions, true); + const auto & before_where = std::get<0>(filter_actions); + const auto & filter_column_name = std::get<1>(filter_actions); + FilterTransformAction filter_action(early_block, before_where, filter_column_name); + Block evaluation_block = early_block; + FilterPtr selection = nullptr; + bool any_selected = !filter_action.alwaysFalse(); + if (any_selected) + { + any_selected = filter_action.transform(evaluation_block, selection, true); + if (!evaluation_block || evaluation_block.rows() == 0) + any_selected = false; + } + + UInt8 selection_kind = 0; + const char * selection_data = nullptr; + uint64_t selection_size = 0; + if (!any_selected) + selection_kind = 1; + else if (selection != nullptr) + { + selection_kind = 2; + selection_data = reinterpret_cast(selection->data()); + selection_size = selection->size(); + } + const auto selected_rows = late_materialization_interfaces->fn_materialize_selected( + reader.value(), + batch_id, + selection_kind, + BaseBuffView{selection_data, selection_size}); + if (selected_rows == std::numeric_limits::max()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "materialize selected rows for batch {} failed", batch_id); + if (selected_rows == 0) + { + if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) + == 0) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "discard empty late-materialization batch {} failed", + batch_id); + pending_late_materialization = false; + continue; + } + + MutableColumns columns = header.cloneEmptyColumns(); + for (size_t i = 0; i < col_type_and_name.size(); ++i) + { + const auto & column = col_type_and_name[i]; + if (column.column_id == MutSup::extra_table_id_col_id) + continue; + if (early_column_ids.find(column.column_id) != early_column_ids.end()) + { + ColumnPtr value = early_block.getByName(column.name).column; + if (!any_selected) + value = value->cut(0, 0); + else if (selection != nullptr) + value = value->filter(*selection, selected_rows); + columns[i] = value->assumeMutable(); + continue; + } + auto col_data + = late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, column.column_id); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + column.type->deserializeBinaryBulkWithMultipleStreams( + *columns[i], + [&](const IDataType::SubstreamPath &) { return &buf; }, + selected_rows, + -1.0, + true, + {}); + } + if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); + pending_late_materialization = false; + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + + Block block = header.cloneWithColumns(std::move(columns)); + action.fill(block, physical_table_id == -1 ? table_id : physical_table_id); + block.setRSResult(DM::RSResult::All); + block.checkNumberOfRows(); + total_bytes += block.bytes(); + return block; + } // Construct block from columnar column data. MutableColumns columns = header.cloneEmptyColumns(); for (UInt32 i = 0; i < col_type_and_name.size(); ++i) @@ -1496,7 +1794,9 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ Int64 col_id = col_type_and_name[i].column_id; if (col_id == MutSup::extra_handle_id) { - RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); + RustStrWithView col_data = use_late_materialization + ? late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, col_id) + : proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1515,8 +1815,11 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } else { - RustStrWithView col_data - = proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); + RustStrWithView col_data = use_late_materialization + ? (early_column_ids.find(col_id) != early_column_ids.end() + ? late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, col_id) + : late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, col_id)) + : proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1533,6 +1836,13 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + if (pending_late_materialization) + { + if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); + pending_late_materialization = false; + } + Block block = header.cloneWithColumns(std::move(columns)); LOG_DEBUG(log, "Read block rows={}, structure={}", block.rows(), block.dumpStructure()); if (physical_table_id == -1) diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index 3fecb3dd6cb..a4f652e3048 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -43,6 +43,7 @@ #include #include #include +#include #pragma GCC diagnostic pop namespace DB @@ -153,6 +154,17 @@ class RNColumnarReadTask const String & getExecutorID() const; + const google::protobuf::RepeatedPtrField & getExactFilterConditions() const; + + google::protobuf::RepeatedPtrField getLateMaterializationFilterConditions( + const Block & early_block) const; + + std::unordered_set getExactFilterColumnIDs() const; + + std::unordered_set getLateMaterializationEarlyColumnIDs() const; + + bool isLateMaterializationFilterEligible() const; + RNColumnarReadTask( std::vector reader_plans, size_t source_num, @@ -223,6 +235,7 @@ class RNColumnarInputStream : public IProfilingBlockInputStream private: bool ensureReader(); + void initializeLateMaterialization(); void mergeReaderStats(); void releaseReader(); @@ -236,6 +249,8 @@ class RNColumnarInputStream : public IProfilingBlockInputStream TableID table_id; const String executor_id; Block header; + const ColumnarLateMaterializationInterfaces * late_materialization_interfaces = nullptr; + bool late_materialization_initialized = false; bool done = false; From 109c056c1b64ef3b0dd38662d3412d4f9380483d Mon Sep 17 00:00:00 2001 From: yongman Date: Thu, 20 Aug 2026 18:12:11 +0800 Subject: [PATCH 02/10] add stats for late materialization packs Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 1 + .../hub-runtime/src/columnar_impls.rs | 1 + .../hub-runtime/src/interfaces.rs | 1 + .../Flash/Coprocessor/ColumnarScanContext.h | 5 ++ .../Storages/StorageDisaggregatedColumnar.cpp | 46 ++++++++++++------- .../Storages/StorageDisaggregatedColumnar.h | 2 + 7 files changed, 41 insertions(+), 17 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 1e882735ec3..663880bd1b4 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 1e882735ec30d96a7cfdb4ff6c7bb8040287db02 +Subproject commit 663880bd1b4c2997f187f56da2c23e86d9f1fa79 diff --git a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 72959feb15f..86334874811 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -223,6 +223,7 @@ struct ColumnarScanStats { uint64_t rough_check_selected_packs; uint64_t rough_check_skipped_packs; uint64_t rough_check_unknown_packs; + uint64_t late_materialization_skipped_packs; uint64_t remote_segments; uint64_t total_segments; }; diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs index 7a848289bab..51f9b0ae489 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs @@ -88,6 +88,7 @@ impl From for ColumnarScanStats rough_check_selected_packs: stats.rough_check_selected_packs, rough_check_skipped_packs: stats.rough_check_skipped_packs, rough_check_unknown_packs: stats.rough_check_unknown_packs, + late_materialization_skipped_packs: stats.late_materialization_skipped_packs, remote_segments: stats.remote_segments, total_segments: stats.total_segments, } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs index f5688545b9e..b78873cb4bb 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs @@ -292,6 +292,7 @@ pub mod root { pub rough_check_selected_packs: u64, pub rough_check_skipped_packs: u64, pub rough_check_unknown_packs: u64, + pub late_materialization_skipped_packs: u64, pub remote_segments: u64, pub total_segments: u64, } diff --git a/dbms/src/Flash/Coprocessor/ColumnarScanContext.h b/dbms/src/Flash/Coprocessor/ColumnarScanContext.h index 58fe112926f..e4f43bf5a2e 100644 --- a/dbms/src/Flash/Coprocessor/ColumnarScanContext.h +++ b/dbms/src/Flash/Coprocessor/ColumnarScanContext.h @@ -47,6 +47,7 @@ class ColumnarScanContext std::atomic rough_check_selected_packs{0}; std::atomic rough_check_skipped_packs{0}; std::atomic rough_check_unknown_packs{0}; + std::atomic late_materialization_skipped_packs{0}; std::atomic remote_segments{0}; std::atomic total_segments{0}; @@ -118,6 +119,7 @@ class ColumnarScanContext rough_check_selected_packs += other.rough_check_selected_packs.load(); rough_check_skipped_packs += other.rough_check_skipped_packs.load(); rough_check_unknown_packs += other.rough_check_unknown_packs.load(); + late_materialization_skipped_packs += other.late_materialization_skipped_packs.load(); remote_segments += other.remote_segments.load(); total_segments += other.total_segments.load(); } @@ -159,6 +161,7 @@ class ColumnarScanContext rough_check_selected_packs += other.rough_check_selected_packs; rough_check_skipped_packs += other.rough_check_skipped_packs; rough_check_unknown_packs += other.rough_check_unknown_packs; + late_materialization_skipped_packs += other.late_materialization_skipped_packs; remote_segments += other.remote_segments; total_segments += other.total_segments; } @@ -173,6 +176,7 @@ class ColumnarScanContext R"(,"user_read_bytes":{},"read_block":"{:.3f}ms","serialize_block":"{:.3f}ms")" R"(,"init_reader":"{:.3f}ms","prefetch":"{:.3f}ms","deserialize_block":"{:.3f}ms")" R"(,"rough_check":{{"total":{},"selected":{},"skipped":{},"unknown":{}}})" + R"(,"late_materialization_skipped_packs":{})" R"(,"remote_segments":{},"total_segments":{}}})", mvcc_input_rows.load(), mvcc_input_bytes.load(), @@ -191,6 +195,7 @@ class ColumnarScanContext rough_check_selected_packs.load(), rough_check_skipped_packs.load(), rough_check_unknown_packs.load(), + late_materialization_skipped_packs.load(), remote_segments.load(), total_segments.load()); } diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index 1c80e520a9b..760e7a8cef1 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -130,6 +130,7 @@ struct RNColumnarReaderSharedContext String table_scan_data; String filter_conditions_data; google::protobuf::RepeatedPtrField exact_filter_conditions; + bool has_pushed_down_filter_conditions = false; std::vector scan_columns; String table_info_data; String ann_query_info_data; @@ -138,6 +139,7 @@ struct RNColumnarReaderSharedContext ClearSharedSnapAccessByStartTsFn clear_shared_snap_access_by_start_ts = nullptr; std::shared_ptr output_lock = std::make_shared(); bool registered_for_start_ts = false; + std::atomic_bool late_materialization_logged{false}; ~RNColumnarReaderSharedContext() noexcept { @@ -490,6 +492,7 @@ std::shared_ptr buildColumnarReaderSharedContext( const auto & pushed_down_filters = table_scan_pb.tp() == tipb::TypePartitionTableScan ? table_scan_pb.partition_table_scan().pushed_down_filter_conditions() : table_scan_pb.tbl_scan().pushed_down_filter_conditions(); + shared_context->has_pushed_down_filter_conditions = !pushed_down_filters.empty(); shared_context->exact_filter_conditions.MergeFrom(pushed_down_filters); tipb::TableInfo table_info; @@ -1042,7 +1045,8 @@ std::unordered_set RNColumnarReadTask::getLateMaterializationEarlyColu bool RNColumnarReadTask::isLateMaterializationFilterEligible() const { - if (shared_reader_context->exact_filter_conditions.empty()) + if (!shared_reader_context->has_pushed_down_filter_conditions + || shared_reader_context->exact_filter_conditions.empty()) return false; const auto column_ids = getExactFilterColumnIDs(); if (column_ids.find(MutSup::extra_table_id_col_id) != column_ids.end()) @@ -1061,6 +1065,11 @@ bool RNColumnarReadTask::isLateMaterializationFilterEligible() const return true; } +bool RNColumnarReadTask::shouldLogLateMaterialization() +{ + return !shared_reader_context->late_materialization_logged.exchange(true); +} + void RNColumnarReadTask::replaceReaderWork( const RNColumnarReaderWorkPtr & reader_work, std::vector replanned_reader_plans) @@ -1486,26 +1495,31 @@ void RNColumnarInputStream::initializeLateMaterialization() if (late_materialization_initialized) return; late_materialization_initialized = true; - if (!context.getSettingsRef().enable_columnar_l2_late_materialization - || !task->isLateMaterializationFilterEligible()) - return; - - const auto predicate_column_ids = task->getExactFilterColumnIDs(); - bool has_late_column = false; - for (const auto & column : header) + if (context.getSettingsRef().enable_columnar_l2_late_materialization && task->isLateMaterializationFilterEligible()) { - if (column.column_id == MutSup::extra_table_id_col_id || column.column_id == MutSup::extra_handle_id) - continue; - if (predicate_column_ids.find(column.column_id) == predicate_column_ids.end()) + const auto predicate_column_ids = task->getExactFilterColumnIDs(); + bool has_late_column = false; + for (const auto & column : header) { - has_late_column = true; - break; + if (column.column_id == MutSup::extra_table_id_col_id || column.column_id == MutSup::extra_handle_id) + continue; + if (predicate_column_ids.find(column.column_id) == predicate_column_ids.end()) + { + has_late_column = true; + break; + } } + if (has_late_column) + late_materialization_interfaces = getLateMaterializationInterfaces(); } - if (!has_late_column) - return; - late_materialization_interfaces = getLateMaterializationInterfaces(); + if (task->shouldLogLateMaterialization()) + LOG_INFO( + log, + "Columnar late materialization enabled={}, executor_id={}, table_id={}", + late_materialization_interfaces != nullptr, + executor_id, + table_id); } void RNColumnarInputStream::releaseReader() diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index a4f652e3048..d73f2906159 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -165,6 +165,8 @@ class RNColumnarReadTask bool isLateMaterializationFilterEligible() const; + bool shouldLogLateMaterialization(); + RNColumnarReadTask( std::vector reader_plans, size_t source_num, From 778a9d241f3c9fdb9ae9f2138e3813ccdab8fae8 Mon Sep 17 00:00:00 2001 From: yongman Date: Fri, 21 Aug 2026 19:35:37 +0800 Subject: [PATCH 03/10] polish code Signed-off-by: yongman --- .../Storages/StorageDisaggregatedColumnar.cpp | 313 +++++++++--------- .../Storages/StorageDisaggregatedColumnar.h | 1 + 2 files changed, 160 insertions(+), 154 deletions(-) diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index 760e7a8cef1..63e69541a31 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -1613,6 +1613,153 @@ Block RNColumnarInputStream::readImpl() return readImpl(filter_ignored, false); } +Block RNColumnarInputStream::readLateMaterializedBlock() +{ + Stopwatch w{CLOCK_MONOTONIC_COARSE}; + const auto early_column_ids = task->getLateMaterializationEarlyColumnIDs(); + const std::vector encoded_ids(early_column_ids.begin(), early_column_ids.end()); + UInt64 batch_id = 0; + TableID physical_table_id = -1; + const UInt64 rows = late_materialization_interfaces->fn_read_early_block( + reader.value(), + batch_size, + BaseBuffView{reinterpret_cast(encoded_ids.data()), encoded_ids.size() * sizeof(Int64)}, + &batch_id, + &physical_table_id); + bool pending_late_materialization = rows != 0 && rows != std::numeric_limits::max(); + SCOPE_EXIT({ + if (pending_late_materialization) + late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id); + }); + duration_read_sec += w.elapsedSecondsFromLastTime(); + LOG_DEBUG(log, "Read {} rows from columnar", rows); + if (rows == std::numeric_limits::max()) + { + LOG_WARNING(log, "Read block from columnar failed"); + throw Exception("read_block failed in columnar", ErrorCodes::LOGICAL_ERROR); + } + if (rows == 0) + { + releaseReader(); + done = fixed_reader_work != nullptr; + return {}; + } + + // Check RSS pressure once the columnar reader has materialized a non-empty block, + // before deserializing more column data into TiFlash memory. + CurrentMemoryTracker::checkRssLimit(); + + Block header = getHeader(); + const ColumnsWithTypeAndName & col_type_and_name = header.getColumnsWithTypeAndName(); + Block early_block; + for (const auto & column : col_type_and_name) + { + if (column.column_id == MutSup::extra_table_id_col_id + || early_column_ids.find(column.column_id) == early_column_ids.end()) + continue; + auto col_data + = late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, column.column_id); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + auto mutable_column = column.type->createColumn(); + column.type->deserializeBinaryBulkWithMultipleStreams( + *mutable_column, + [&](const IDataType::SubstreamPath &) { return &buf; }, + rows, + -1.0, + true, + {}); + early_block.insert( + ColumnWithTypeAndName{std::move(mutable_column), column.type, column.name, column.column_id}); + } + + NamesAndTypes early_names_and_types; + early_names_and_types.reserve(early_block.columns()); + for (const auto & column : early_block) + early_names_and_types.emplace_back(column.name, column.type); + DAGExpressionAnalyzer lm_analyzer(std::move(early_names_and_types), context); + auto filter_conditions = task->getLateMaterializationFilterConditions(early_block); + auto filter_actions = lm_analyzer.buildPushDownFilter(filter_conditions, true); + const auto & before_where = std::get<0>(filter_actions); + const auto & filter_column_name = std::get<1>(filter_actions); + FilterTransformAction filter_action(early_block, before_where, filter_column_name); + Block evaluation_block = early_block; + FilterPtr selection = nullptr; + bool any_selected = !filter_action.alwaysFalse(); + if (any_selected) + { + any_selected = filter_action.transform(evaluation_block, selection, true); + if (!evaluation_block || evaluation_block.rows() == 0) + any_selected = false; + } + + UInt8 selection_kind = 0; + const char * selection_data = nullptr; + uint64_t selection_size = 0; + if (!any_selected) + selection_kind = 1; + else if (selection != nullptr) + { + selection_kind = 2; + selection_data = reinterpret_cast(selection->data()); + selection_size = selection->size(); + } + const auto selected_rows = late_materialization_interfaces->fn_materialize_selected( + reader.value(), + batch_id, + selection_kind, + BaseBuffView{selection_data, selection_size}); + if (selected_rows == std::numeric_limits::max()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "materialize selected rows for batch {} failed", batch_id); + if (selected_rows == 0) + { + if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "discard empty late-materialization batch {} failed", batch_id); + pending_late_materialization = false; + return {}; + } + + MutableColumns columns = header.cloneEmptyColumns(); + for (size_t i = 0; i < col_type_and_name.size(); ++i) + { + const auto & column = col_type_and_name[i]; + if (column.column_id == MutSup::extra_table_id_col_id) + continue; + if (early_column_ids.find(column.column_id) != early_column_ids.end()) + { + ColumnPtr value = early_block.getByName(column.name).column; + if (!any_selected) + value = value->cut(0, 0); + else if (selection != nullptr) + value = value->filter(*selection, selected_rows); + columns[i] = value->assumeMutable(); + continue; + } + auto col_data + = late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, column.column_id); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + column.type->deserializeBinaryBulkWithMultipleStreams( + *columns[i], + [&](const IDataType::SubstreamPath &) { return &buf; }, + selected_rows, + -1.0, + true, + {}); + } + if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); + pending_late_materialization = false; + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + + Block block = header.cloneWithColumns(std::move(columns)); + action.fill(block, physical_table_id == -1 ? table_id : physical_table_id); + block.setRSResult(DM::RSResult::All); + block.checkNumberOfRows(); + total_bytes += block.bytes(); + return block; +} + Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [[maybe_unused]] bool return_filter) { if (done) @@ -1629,33 +1776,17 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ return {}; } - Stopwatch w{CLOCK_MONOTONIC_COARSE}; - UInt64 batch_id = 0; - TableID physical_table_id = -1; - bool pending_late_materialization = false; - bool use_late_materialization = late_materialization_interfaces != nullptr; - std::unordered_set early_column_ids; - UInt64 rows = 0; - if (use_late_materialization) + if (late_materialization_interfaces != nullptr) { - early_column_ids = task->getLateMaterializationEarlyColumnIDs(); - std::vector encoded_ids(early_column_ids.begin(), early_column_ids.end()); - rows = late_materialization_interfaces->fn_read_early_block( - reader.value(), - batch_size, - BaseBuffView{reinterpret_cast(encoded_ids.data()), encoded_ids.size() * sizeof(Int64)}, - &batch_id, - &physical_table_id); - pending_late_materialization = rows != 0 && rows != std::numeric_limits::max(); - } - else - { - rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); + Block block = readLateMaterializedBlock(); + if (block || done) + return block; + continue; } - SCOPE_EXIT({ - if (pending_late_materialization) - late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id); - }); + + Stopwatch w{CLOCK_MONOTONIC_COARSE}; + TableID physical_table_id = -1; + const UInt64 rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); duration_read_sec += w.elapsedSecondsFromLastTime(); LOG_DEBUG(log, "Read {} rows from columnar", rows); if (rows == std::numeric_limits::max()) @@ -1680,120 +1811,6 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ Block header = getHeader(); const ColumnsWithTypeAndName & col_type_and_name = header.getColumnsWithTypeAndName(); - if (use_late_materialization) - { - Block early_block; - for (const auto & column : col_type_and_name) - { - if (column.column_id == MutSup::extra_table_id_col_id - || early_column_ids.find(column.column_id) == early_column_ids.end()) - continue; - auto col_data - = late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, column.column_id); - SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); - ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); - auto mutable_column = column.type->createColumn(); - column.type->deserializeBinaryBulkWithMultipleStreams( - *mutable_column, - [&](const IDataType::SubstreamPath &) { return &buf; }, - rows, - -1.0, - true, - {}); - early_block.insert( - ColumnWithTypeAndName{std::move(mutable_column), column.type, column.name, column.column_id}); - } - - NamesAndTypes early_names_and_types; - early_names_and_types.reserve(early_block.columns()); - for (const auto & column : early_block) - early_names_and_types.emplace_back(column.name, column.type); - DAGExpressionAnalyzer lm_analyzer(std::move(early_names_and_types), context); - auto filter_conditions = task->getLateMaterializationFilterConditions(early_block); - auto filter_actions = lm_analyzer.buildPushDownFilter(filter_conditions, true); - const auto & before_where = std::get<0>(filter_actions); - const auto & filter_column_name = std::get<1>(filter_actions); - FilterTransformAction filter_action(early_block, before_where, filter_column_name); - Block evaluation_block = early_block; - FilterPtr selection = nullptr; - bool any_selected = !filter_action.alwaysFalse(); - if (any_selected) - { - any_selected = filter_action.transform(evaluation_block, selection, true); - if (!evaluation_block || evaluation_block.rows() == 0) - any_selected = false; - } - - UInt8 selection_kind = 0; - const char * selection_data = nullptr; - uint64_t selection_size = 0; - if (!any_selected) - selection_kind = 1; - else if (selection != nullptr) - { - selection_kind = 2; - selection_data = reinterpret_cast(selection->data()); - selection_size = selection->size(); - } - const auto selected_rows = late_materialization_interfaces->fn_materialize_selected( - reader.value(), - batch_id, - selection_kind, - BaseBuffView{selection_data, selection_size}); - if (selected_rows == std::numeric_limits::max()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "materialize selected rows for batch {} failed", batch_id); - if (selected_rows == 0) - { - if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) - == 0) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "discard empty late-materialization batch {} failed", - batch_id); - pending_late_materialization = false; - continue; - } - - MutableColumns columns = header.cloneEmptyColumns(); - for (size_t i = 0; i < col_type_and_name.size(); ++i) - { - const auto & column = col_type_and_name[i]; - if (column.column_id == MutSup::extra_table_id_col_id) - continue; - if (early_column_ids.find(column.column_id) != early_column_ids.end()) - { - ColumnPtr value = early_block.getByName(column.name).column; - if (!any_selected) - value = value->cut(0, 0); - else if (selection != nullptr) - value = value->filter(*selection, selected_rows); - columns[i] = value->assumeMutable(); - continue; - } - auto col_data - = late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, column.column_id); - SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); - ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); - column.type->deserializeBinaryBulkWithMultipleStreams( - *columns[i], - [&](const IDataType::SubstreamPath &) { return &buf; }, - selected_rows, - -1.0, - true, - {}); - } - if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); - pending_late_materialization = false; - duration_deserialize_sec += w.elapsedSecondsFromLastTime(); - - Block block = header.cloneWithColumns(std::move(columns)); - action.fill(block, physical_table_id == -1 ? table_id : physical_table_id); - block.setRSResult(DM::RSResult::All); - block.checkNumberOfRows(); - total_bytes += block.bytes(); - return block; - } // Construct block from columnar column data. MutableColumns columns = header.cloneEmptyColumns(); for (UInt32 i = 0; i < col_type_and_name.size(); ++i) @@ -1808,9 +1825,7 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ Int64 col_id = col_type_and_name[i].column_id; if (col_id == MutSup::extra_handle_id) { - RustStrWithView col_data = use_late_materialization - ? late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, col_id) - : proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); + RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1829,11 +1844,8 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } else { - RustStrWithView col_data = use_late_materialization - ? (early_column_ids.find(col_id) != early_column_ids.end() - ? late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, col_id) - : late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, col_id)) - : proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); + RustStrWithView col_data + = proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1850,13 +1862,6 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } duration_deserialize_sec += w.elapsedSecondsFromLastTime(); - if (pending_late_materialization) - { - if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); - pending_late_materialization = false; - } - Block block = header.cloneWithColumns(std::move(columns)); LOG_DEBUG(log, "Read block rows={}, structure={}", block.rows(), block.dumpStructure()); if (physical_table_id == -1) diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index d73f2906159..0eb05c3f2b4 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -238,6 +238,7 @@ class RNColumnarInputStream : public IProfilingBlockInputStream private: bool ensureReader(); void initializeLateMaterialization(); + Block readLateMaterializedBlock(); void mergeReaderStats(); void releaseReader(); From 0f3f38dcb7b31921e2dfdb57b9a3691a1efc0cd3 Mon Sep 17 00:00:00 2001 From: yongman Date: Mon, 24 Aug 2026 09:44:21 +0800 Subject: [PATCH 04/10] disable lm when has multiple tables in one region Signed-off-by: yongman --- dbms/src/Storages/StorageDisaggregatedColumnar.cpp | 7 +++++++ dbms/src/Storages/StorageDisaggregatedColumnar.h | 1 + 2 files changed, 8 insertions(+) diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index 63e69541a31..caadf932361 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -61,6 +61,7 @@ #include #include +#include #include #include #include @@ -943,6 +944,10 @@ RNColumnarReadTask::RNColumnarReadTask( std::shared_ptr shared_reader_context_) : reader_count(reader_plans.size()) , source_num(source_num_) + , has_multi_table_reader_plan(std::any_of( + reader_plans.begin(), + reader_plans.end(), + [](const auto & reader_plan) { return reader_plan.physical_table_ranges.size() > 1; })) , shared_reader_context(std::move(shared_reader_context_)) { RUNTIME_CHECK(source_num > 0); @@ -1045,6 +1050,8 @@ std::unordered_set RNColumnarReadTask::getLateMaterializationEarlyColu bool RNColumnarReadTask::isLateMaterializationFilterEligible() const { + if (has_multi_table_reader_plan) + return false; if (!shared_reader_context->has_pushed_down_filter_conditions || shared_reader_context->exact_filter_conditions.empty()) return false; diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index 0eb05c3f2b4..373c47283bb 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -183,6 +183,7 @@ class RNColumnarReadTask size_t reader_count; size_t source_num; + bool has_multi_table_reader_plan; std::shared_ptr shared_reader_context; mutable std::mutex pending_reader_works_mutex; std::deque pending_reader_works; From 9308dc52c957129a83ba24099dc5d00dfb21fd98 Mon Sep 17 00:00:00 2001 From: yongman Date: Mon, 24 Aug 2026 16:31:18 +0800 Subject: [PATCH 05/10] optimize perf Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 2 + .../hub-runtime/src/columnar_impls.rs | 26 ++++++ .../hub-runtime/src/interfaces.rs | 6 ++ .../Storages/StorageDisaggregatedColumnar.cpp | 85 ++++++++++++++++--- .../Storages/StorageDisaggregatedColumnar.h | 2 + 6 files changed, 109 insertions(+), 14 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 663880bd1b4..a96382685ea 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 663880bd1b4c2997f187f56da2c23e86d9f1fa79 +Subproject commit a96382685ea2dd1ab61c7e33b4106df1030a97fa diff --git a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 86334874811..ef321d19b32 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -279,6 +279,8 @@ struct ColumnarLateMaterializationInterfaces { uint8_t (*fn_finish_materialized_block)(ColumnarReaderPtr, uint64_t); uint8_t (*fn_discard_late_materialization_batch)(ColumnarReaderPtr, uint64_t); + uint8_t (*fn_is_late_materialization_supported)(ColumnarReaderPtr, + BaseBuffView); }; #if defined(__GNUC__) || defined(__clang__) diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs index 51f9b0ae489..d11760b951d 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs @@ -387,6 +387,31 @@ pub unsafe extern "C" fn ffi_discard_late_materialization_batch( } } +pub unsafe extern "C" fn ffi_is_late_materialization_supported( + mut reader: ColumnarReaderPtr, + early_column_ids: BaseBuffView, +) -> u8 { + if early_column_ids.len % 8 != 0 { + error!("invalid late-materialization capability arguments"); + return 0; + } + let early_column_ids = early_column_ids + .to_slice() + .chunks_exact(8) + .map(|bytes| i64::from_le_bytes(bytes.try_into().unwrap())) + .collect::>(); + match reader + .as_mut() + .ffi_is_late_materialization_supported(&early_column_ids) + { + Ok(supported) => u8::from(supported), + Err(err) => { + error!("late-materialization capability check failed: {}", err); + 0 + } + } +} + static LATE_MATERIALIZATION_INTERFACES: ColumnarLateMaterializationInterfaces = ColumnarLateMaterializationInterfaces { version: 1, @@ -397,6 +422,7 @@ static LATE_MATERIALIZATION_INTERFACES: ColumnarLateMaterializationInterfaces = fn_read_late_column: Some(ffi_read_late_column), fn_finish_materialized_block: Some(ffi_finish_materialized_block), fn_discard_late_materialization_batch: Some(ffi_discard_late_materialization_batch), + fn_is_late_materialization_supported: Some(ffi_is_late_materialization_supported), }; #[no_mangle] diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs index b78873cb4bb..766a37181f2 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs @@ -404,6 +404,12 @@ pub mod root { pub fn_discard_late_materialization_batch: ::std::option::Option< unsafe extern "C" fn(arg1: root::DB::ColumnarReaderPtr, arg2: u64) -> u8, >, + pub fn_is_late_materialization_supported: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::ColumnarReaderPtr, + arg2: root::DB::BaseBuffView, + ) -> u8, + >, } #[repr(C)] #[derive(Debug)] diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index caadf932361..56124cfdcfc 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -188,7 +188,8 @@ const ColumnarLateMaterializationInterfaces * getLateMaterializationInterfaces() || interfaces->fn_read_early_block == nullptr || interfaces->fn_read_early_column == nullptr || interfaces->fn_materialize_selected == nullptr || interfaces->fn_read_late_column == nullptr || interfaces->fn_finish_materialized_block == nullptr - || interfaces->fn_discard_late_materialization_batch == nullptr) + || interfaces->fn_discard_late_materialization_batch == nullptr + || interfaces->fn_is_late_materialization_supported == nullptr) return nullptr; return interfaces; #else @@ -1517,7 +1518,23 @@ void RNColumnarInputStream::initializeLateMaterialization() } } if (has_late_column) - late_materialization_interfaces = getLateMaterializationInterfaces(); + { + const auto * interfaces = getLateMaterializationInterfaces(); + if (interfaces != nullptr) + { + const auto early_column_ids = task->getLateMaterializationEarlyColumnIDs(); + const std::vector encoded_ids(early_column_ids.begin(), early_column_ids.end()); + const bool supported = interfaces->fn_is_late_materialization_supported( + reader.value(), + BaseBuffView{ + reinterpret_cast(encoded_ids.data()), + encoded_ids.size() * sizeof(Int64)}); + if (supported) + late_materialization_interfaces = interfaces; + else + LOG_DEBUG(log, "Columnar late materialization is unavailable for this reader"); + } + } } if (task->shouldLogLateMaterialization()) @@ -1537,6 +1554,7 @@ void RNColumnarInputStream::releaseReader() reader.reset(); current_reader_work.reset(); late_materialization_interfaces = nullptr; + late_materialization_filter_action.reset(); late_materialization_initialized = false; } @@ -1664,8 +1682,10 @@ Block RNColumnarInputStream::readLateMaterializedBlock() if (column.column_id == MutSup::extra_table_id_col_id || early_column_ids.find(column.column_id) == early_column_ids.end()) continue; + (void)w.elapsedSecondsFromLastTime(); auto col_data = late_materialization_interfaces->fn_read_early_column(reader.value(), batch_id, column.column_id); + duration_read_sec += w.elapsedSecondsFromLastTime(); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); auto mutable_column = column.type->createColumn(); @@ -1676,20 +1696,30 @@ Block RNColumnarInputStream::readLateMaterializedBlock() -1.0, true, {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); early_block.insert( ColumnWithTypeAndName{std::move(mutable_column), column.type, column.name, column.column_id}); } - NamesAndTypes early_names_and_types; - early_names_and_types.reserve(early_block.columns()); - for (const auto & column : early_block) - early_names_and_types.emplace_back(column.name, column.type); - DAGExpressionAnalyzer lm_analyzer(std::move(early_names_and_types), context); - auto filter_conditions = task->getLateMaterializationFilterConditions(early_block); - auto filter_actions = lm_analyzer.buildPushDownFilter(filter_conditions, true); - const auto & before_where = std::get<0>(filter_actions); - const auto & filter_column_name = std::get<1>(filter_actions); - FilterTransformAction filter_action(early_block, before_where, filter_column_name); + if (!late_materialization_filter_action) + { + // The filter expression and action graph are invariant for one input + // stream. Building them for every 10K-row batch makes LM spend most of + // its time in planner setup instead of row filtering. + Block filter_header = early_block.cloneEmpty(); + NamesAndTypes early_names_and_types; + early_names_and_types.reserve(filter_header.columns()); + for (const auto & column : filter_header) + early_names_and_types.emplace_back(column.name, column.type); + DAGExpressionAnalyzer lm_analyzer(std::move(early_names_and_types), context); + auto filter_conditions = task->getLateMaterializationFilterConditions(filter_header); + auto filter_actions = lm_analyzer.buildPushDownFilter(filter_conditions, true); + late_materialization_filter_action = std::make_unique( + filter_header, + std::get<0>(filter_actions), + std::get<1>(filter_actions)); + } + auto & filter_action = *late_materialization_filter_action; Block evaluation_block = early_block; FilterPtr selection = nullptr; bool any_selected = !filter_action.alwaysFalse(); @@ -1723,9 +1753,16 @@ Block RNColumnarInputStream::readLateMaterializedBlock() if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) == 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "discard empty late-materialization batch {} failed", batch_id); pending_late_materialization = false; + // Account for early deserialization, exact filtering, selection, and + // discard even when this batch produces no output rows. + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); return {}; } + // Exact filter evaluation and selection are post-read processing and + // therefore belong to deserialize_cost. + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + MutableColumns columns = header.cloneEmptyColumns(); for (size_t i = 0; i < col_type_and_name.size(); ++i) { @@ -1740,10 +1777,25 @@ Block RNColumnarInputStream::readLateMaterializedBlock() else if (selection != nullptr) value = value->filter(*selection, selected_rows); columns[i] = value->assumeMutable(); - continue; } + } + // Account for filtering/copying the early columns before measuring the + // late-column callbacks individually. + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + + for (size_t i = 0; i < col_type_and_name.size(); ++i) + { + const auto & column = col_type_and_name[i]; + if (column.column_id == MutSup::extra_table_id_col_id + || early_column_ids.find(column.column_id) != early_column_ids.end()) + continue; + // A late-column callback may perform L2 pack IO, decompression and Rust + // serialization. Keep that work in read_cost; only the C++ decode below + // belongs to deserialize_cost. + (void)w.elapsedSecondsFromLastTime(); auto col_data = late_materialization_interfaces->fn_read_late_column(reader.value(), batch_id, column.column_id); + duration_read_sec += w.elapsedSecondsFromLastTime(); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); column.type->deserializeBinaryBulkWithMultipleStreams( @@ -1753,6 +1805,7 @@ Block RNColumnarInputStream::readLateMaterializedBlock() -1.0, true, {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); } if (late_materialization_interfaces->fn_finish_materialized_block(reader.value(), batch_id) == 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "finish late-materialization batch {} failed", batch_id); @@ -1832,7 +1885,9 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ Int64 col_id = col_type_and_name[i].column_id; if (col_id == MutSup::extra_handle_id) { + (void)w.elapsedSecondsFromLastTime(); RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); + duration_read_sec += w.elapsedSecondsFromLastTime(); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1844,6 +1899,7 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar true, {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); } else if (col_id == MutSup::extra_table_id_col_id) { @@ -1851,8 +1907,10 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ } else { + (void)w.elapsedSecondsFromLastTime(); RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); + duration_read_sec += w.elapsedSecondsFromLastTime(); SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); @@ -1864,6 +1922,7 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar true, {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); LOG_DEBUG(log, "Read column data done, col size={}", col.size()); } } diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index 373c47283bb..a5bd2860186 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -254,6 +255,7 @@ class RNColumnarInputStream : public IProfilingBlockInputStream const String executor_id; Block header; const ColumnarLateMaterializationInterfaces * late_materialization_interfaces = nullptr; + std::unique_ptr late_materialization_filter_action; bool late_materialization_initialized = false; bool done = false; From f82ae577aa02e6b03e94d6ee7a085d8515d02f14 Mon Sep 17 00:00:00 2001 From: RayYan Date: Mon, 24 Aug 2026 20:30:19 +0800 Subject: [PATCH 06/10] add more check before enable lm Signed-off-by: RayYan --- dbms/src/Interpreters/Settings.h | 2 + .../Storages/StorageDisaggregatedColumnar.cpp | 265 +++++++++++------- .../Storages/StorageDisaggregatedColumnar.h | 2 + 3 files changed, 162 insertions(+), 107 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index b3ca4b75877..457d6d4889e 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -338,6 +338,8 @@ struct Settings M(SettingDouble, auto_memory_revoke_trigger_threshold, 0.0, "Trigger auto memory revocation when the memory usage is above this percentage.") \ M(SettingUInt64, remote_read_queue_size, 0, "size of remote read queue, 0 means it is determined automatically") \ M(SettingBool, enable_columnar_l2_late_materialization, false, "Enable L2-only late materialization for cloud columnar reads") \ + M(SettingFloat, columnar_l2_late_materialization_min_selection_skip_ratio, 0.5, "Minimum exact selection skip ratio required for L2 late materialization") \ + M(SettingFloat, columnar_l2_late_materialization_min_late_to_early_ratio, 10.0, "Minimum late-column to (early-column minus handle/version) ratio for L2 late materialization") \ M(SettingBool, enable_cop_stream_for_remote_read, false, "Enable cop stream for remote read") \ M(SettingUInt64, cop_timeout_for_remote_read, 60, "cop timeout seconds for remote read") \ M(SettingUInt64, auto_spill_check_min_interval_ms, 10, "The minimum interval in millisecond between two successive auto spill check, default value is 100, 0 means no limit") \ diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index 56124cfdcfc..f92575e0ddb 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -1505,24 +1505,27 @@ void RNColumnarInputStream::initializeLateMaterialization() late_materialization_initialized = true; if (context.getSettingsRef().enable_columnar_l2_late_materialization && task->isLateMaterializationFilterEligible()) { - const auto predicate_column_ids = task->getExactFilterColumnIDs(); - bool has_late_column = false; + const auto early_column_ids = task->getLateMaterializationEarlyColumnIDs(); + size_t late_column_count = 0; + size_t early_column_count = 0; for (const auto & column : header) { - if (column.column_id == MutSup::extra_table_id_col_id || column.column_id == MutSup::extra_handle_id) + if (column.column_id == MutSup::extra_table_id_col_id) continue; - if (predicate_column_ids.find(column.column_id) == predicate_column_ids.end()) - { - has_late_column = true; - break; - } + if (early_column_ids.find(column.column_id) == early_column_ids.end()) + ++late_column_count; + else + ++early_column_count; } - if (has_late_column) + const auto early_without_system_columns = std::max(1, early_column_count - 2); + const auto late_to_early_ratio + = static_cast(late_column_count) / static_cast(early_without_system_columns); + if (late_column_count > 0 + && late_to_early_ratio > context.getSettingsRef().columnar_l2_late_materialization_min_late_to_early_ratio) { const auto * interfaces = getLateMaterializationInterfaces(); if (interfaces != nullptr) { - const auto early_column_ids = task->getLateMaterializationEarlyColumnIDs(); const std::vector encoded_ids(early_column_ids.begin(), early_column_ids.end()); const bool supported = interfaces->fn_is_late_materialization_supported( reader.value(), @@ -1535,6 +1538,13 @@ void RNColumnarInputStream::initializeLateMaterialization() LOG_DEBUG(log, "Columnar late materialization is unavailable for this reader"); } } + LOG_DEBUG( + log, + "Columnar late materialization eligibility: late_columns={}, early_columns={}, late_to_early_ratio={:.3f}, min_ratio={:.3f}", + late_column_count, + early_column_count, + late_to_early_ratio, + context.getSettingsRef().columnar_l2_late_materialization_min_late_to_early_ratio); } if (task->shouldLogLateMaterialization()) @@ -1556,6 +1566,7 @@ void RNColumnarInputStream::releaseReader() late_materialization_interfaces = nullptr; late_materialization_filter_action.reset(); late_materialization_initialized = false; + late_materialization_probed = false; } void RNColumnarInputStream::mergeReaderStats() @@ -1748,6 +1759,42 @@ Block RNColumnarInputStream::readLateMaterializedBlock() BaseBuffView{selection_data, selection_size}); if (selected_rows == std::numeric_limits::max()) throw Exception(ErrorCodes::LOGICAL_ERROR, "materialize selected rows for batch {} failed", batch_id); + + if (!late_materialization_probed) + { + late_materialization_probed = true; + const auto skip_ratio = rows == 0 ? 0.0 : 1.0 - static_cast(selected_rows) / static_cast(rows); + const auto min_skip_ratio + = context.getSettingsRef().columnar_l2_late_materialization_min_selection_skip_ratio; + if (skip_ratio < min_skip_ratio) + { + if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "discard LM probe batch {} failed", batch_id); + pending_late_materialization = false; + late_materialization_interfaces = nullptr; + LOG_INFO( + log, + "Disable columnar late materialization after probe: rows={}, selected_rows={}, skip_ratio={:.4f}, min_skip_ratio={:.4f}", + rows, + selected_rows, + skip_ratio, + min_skip_ratio); + const auto * proxy_helper + = context.getGlobalContext().getSharedContextDisagg()->getColumnarProxyHelper(); + RUNTIME_CHECK_MSG(proxy_helper != nullptr, "columnar helper is not initialized"); + // The probe consumed an independent LM batch. Continue immediately with the + // legacy reader so the caller does not mistake the discarded probe for EOF. + return readLegacyBlock(proxy_helper); + } + LOG_DEBUG( + log, + "Columnar late materialization probe: rows={}, selected_rows={}, skip_ratio={:.4f}, min_skip_ratio={:.4f}", + rows, + selected_rows, + skip_ratio, + min_skip_ratio); + } + if (selected_rows == 0) { if (late_materialization_interfaces->fn_discard_late_materialization_batch(reader.value(), batch_id) == 0) @@ -1820,6 +1867,104 @@ Block RNColumnarInputStream::readLateMaterializedBlock() return block; } +Block RNColumnarInputStream::readLegacyBlock(const TiFlashRaftProxyHelper * proxy_helper) +{ + Stopwatch w{CLOCK_MONOTONIC_COARSE}; + TableID physical_table_id = -1; + const UInt64 rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); + duration_read_sec += w.elapsedSecondsFromLastTime(); + LOG_DEBUG(log, "Read {} rows from columnar", rows); + if (rows == std::numeric_limits::max()) + { + LOG_WARNING(log, "Read block from columnar failed"); + throw Exception("read_block failed in columnar", ErrorCodes::LOGICAL_ERROR); + } + if (rows == 0) + { + releaseReader(); + if (fixed_reader_work != nullptr) + done = true; + return {}; + } + + // Check RSS pressure once the columnar reader has materialized a non-empty block, + // before deserializing more column data into TiFlash memory. + CurrentMemoryTracker::checkRssLimit(); + + Block header = getHeader(); + const ColumnsWithTypeAndName & col_type_and_name = header.getColumnsWithTypeAndName(); + // Construct block from columnar column data. + MutableColumns columns = header.cloneEmptyColumns(); + for (UInt32 i = 0; i < col_type_and_name.size(); ++i) + { + LOG_DEBUG( + log, + "Read column id={} name={} type={}", + col_type_and_name[i].column_id, + col_type_and_name[i].name, + col_type_and_name[i].type->getName()); + // Read column data from columnar + Int64 col_id = col_type_and_name[i].column_id; + if (col_id == MutSup::extra_handle_id) + { + (void)w.elapsedSecondsFromLastTime(); + RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); + duration_read_sec += w.elapsedSecondsFromLastTime(); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + auto & col = *columns[i]; + col_type_and_name[i].type->deserializeBinaryBulkWithMultipleStreams( + col, + [&](const IDataType::SubstreamPath &) { return &buf; }, + rows, + -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar + true, + {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + } + else if (col_id == MutSup::extra_table_id_col_id) + { + continue; + } + else + { + (void)w.elapsedSecondsFromLastTime(); + RustStrWithView col_data + = proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); + duration_read_sec += w.elapsedSecondsFromLastTime(); + SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); + physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); + ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); + auto & col = *columns[i]; + col_type_and_name[i].type->deserializeBinaryBulkWithMultipleStreams( + col, + [&](const IDataType::SubstreamPath &) { return &buf; }, + rows, + -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar + true, + {}); + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + LOG_DEBUG(log, "Read column data done, col size={}", col.size()); + } + } + duration_deserialize_sec += w.elapsedSecondsFromLastTime(); + + Block block = header.cloneWithColumns(std::move(columns)); + LOG_DEBUG(log, "Read block rows={}, structure={}", block.rows(), block.dumpStructure()); + if (physical_table_id == -1) + { + LOG_WARNING(log, "physical_table_id is not set, use table_id {} instead", table_id); + physical_table_id = table_id; + } + // Fill extra table id column. + action.fill(block, physical_table_id); + block.checkNumberOfRows(); + + total_bytes += block.bytes(); + return block; +} + Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [[maybe_unused]] bool return_filter) { if (done) @@ -1844,103 +1989,9 @@ Block RNColumnarInputStream::readImpl([[maybe_unused]] FilterPtr & res_filter, [ continue; } - Stopwatch w{CLOCK_MONOTONIC_COARSE}; - TableID physical_table_id = -1; - const UInt64 rows = proxy_helper->cloud_storage_engine_interfaces.fn_read_block(reader.value(), batch_size); - duration_read_sec += w.elapsedSecondsFromLastTime(); - LOG_DEBUG(log, "Read {} rows from columnar", rows); - if (rows == std::numeric_limits::max()) - { - LOG_WARNING(log, "Read block from columnar failed"); - throw Exception("read_block failed in columnar", ErrorCodes::LOGICAL_ERROR); - } - if (rows == 0) - { - releaseReader(); - if (fixed_reader_work != nullptr) - { - done = true; - return {}; - } - continue; - } - - // Check RSS pressure once the columnar reader has materialized a non-empty block, - // before deserializing more column data into TiFlash memory. - CurrentMemoryTracker::checkRssLimit(); - - Block header = getHeader(); - const ColumnsWithTypeAndName & col_type_and_name = header.getColumnsWithTypeAndName(); - // Construct block from columnar column data. - MutableColumns columns = header.cloneEmptyColumns(); - for (UInt32 i = 0; i < col_type_and_name.size(); ++i) - { - LOG_DEBUG( - log, - "Read column id={} name={} type={}", - col_type_and_name[i].column_id, - col_type_and_name[i].name, - col_type_and_name[i].type->getName()); - // Read column data from columnar - Int64 col_id = col_type_and_name[i].column_id; - if (col_id == MutSup::extra_handle_id) - { - (void)w.elapsedSecondsFromLastTime(); - RustStrWithView col_data = proxy_helper->cloud_storage_engine_interfaces.fn_read_handle(reader.value()); - duration_read_sec += w.elapsedSecondsFromLastTime(); - SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); - physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); - ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); - auto & col = *columns[i]; - col_type_and_name[i].type->deserializeBinaryBulkWithMultipleStreams( - col, - [&](const IDataType::SubstreamPath &) { return &buf; }, - rows, - -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar - true, - {}); - duration_deserialize_sec += w.elapsedSecondsFromLastTime(); - } - else if (col_id == MutSup::extra_table_id_col_id) - { - continue; - } - else - { - (void)w.elapsedSecondsFromLastTime(); - RustStrWithView col_data - = proxy_helper->cloud_storage_engine_interfaces.fn_read_column(reader.value(), col_id); - duration_read_sec += w.elapsedSecondsFromLastTime(); - SCOPE_EXIT({ RustGcHelper::instance().gcRustPtr(col_data.inner.ptr, col_data.inner.type); }); - physical_table_id = proxy_helper->cloud_storage_engine_interfaces.fn_physical_table_id(reader.value()); - ReadBufferFromMemory buf(col_data.buff.data, static_cast(col_data.buff.len)); - auto & col = *columns[i]; - col_type_and_name[i].type->deserializeBinaryBulkWithMultipleStreams( - col, - [&](const IDataType::SubstreamPath &) { return &buf; }, - rows, - -1.0, // avg_value_size_hint set to -1 to indicate Decimal format from columnar - true, - {}); - duration_deserialize_sec += w.elapsedSecondsFromLastTime(); - LOG_DEBUG(log, "Read column data done, col size={}", col.size()); - } - } - duration_deserialize_sec += w.elapsedSecondsFromLastTime(); - - Block block = header.cloneWithColumns(std::move(columns)); - LOG_DEBUG(log, "Read block rows={}, structure={}", block.rows(), block.dumpStructure()); - if (physical_table_id == -1) - { - LOG_WARNING(log, "physical_table_id is not set, use table_id {} instead", table_id); - physical_table_id = table_id; - } - // Fill extra table id column. - action.fill(block, physical_table_id); - block.checkNumberOfRows(); - - total_bytes += block.bytes(); - return block; + Block block = readLegacyBlock(proxy_helper); + if (block || done) + return block; } } diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index a5bd2860186..1ff8726fac7 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -241,6 +241,7 @@ class RNColumnarInputStream : public IProfilingBlockInputStream bool ensureReader(); void initializeLateMaterialization(); Block readLateMaterializedBlock(); + Block readLegacyBlock(const TiFlashRaftProxyHelper * proxy_helper); void mergeReaderStats(); void releaseReader(); @@ -257,6 +258,7 @@ class RNColumnarInputStream : public IProfilingBlockInputStream const ColumnarLateMaterializationInterfaces * late_materialization_interfaces = nullptr; std::unique_ptr late_materialization_filter_action; bool late_materialization_initialized = false; + bool late_materialization_probed = false; bool done = false; From dd302e6c6f8977ad2f3035c37ea38a5f4ce71062 Mon Sep 17 00:00:00 2001 From: RayYan Date: Wed, 26 Aug 2026 16:43:31 +0800 Subject: [PATCH 07/10] perf optimize Signed-off-by: RayYan Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- contrib/tiflash-columnar-hub/Cargo.lock | 28 ++-- contrib/tiflash-proxy-cmake/CMakeLists.txt | 18 +++ .../Storages/StorageDisaggregatedColumnar.cpp | 142 +++++++++++++++++- .../Storages/StorageDisaggregatedColumnar.h | 4 +- 5 files changed, 166 insertions(+), 28 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index a96382685ea..7404c56f9a9 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit a96382685ea2dd1ab61c7e33b4106df1030a97fa +Subproject commit 7404c56f9a9b1089222a4bb4e01926bb8bb43177 diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 8e4740ab765..5d6f0b92bbb 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -1038,7 +1038,7 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex 1.3.0", + "shlex", "which", ] @@ -1220,7 +1220,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 1.3.0", + "shlex", ] [[package]] @@ -5685,7 +5685,7 @@ dependencies = [ [[package]] name = "rusoto_core" version = "0.46.0" -source = "git+https://github.com/tikv/rusoto?branch=cse#d6ffce8898ce5d03ada531401437985b727393a4" +source = "git+https://github.com/tikv/rusoto?branch=cse#187f471cc34e1eeb5bb2f3c0e9564284a441593f" dependencies = [ "async-trait", "base64 0.13.1", @@ -5709,7 +5709,7 @@ dependencies = [ [[package]] name = "rusoto_credential" version = "0.46.0" -source = "git+https://github.com/tikv/rusoto?branch=cse#d6ffce8898ce5d03ada531401437985b727393a4" +source = "git+https://github.com/tikv/rusoto?branch=cse#187f471cc34e1eeb5bb2f3c0e9564284a441593f" dependencies = [ "async-trait", "chrono", @@ -5718,7 +5718,7 @@ dependencies = [ "hyper 0.14.26", "serde", "serde_json", - "shlex 0.1.1", + "shlex", "tokio", "zeroize", ] @@ -5726,7 +5726,7 @@ dependencies = [ [[package]] name = "rusoto_mock" version = "0.46.0" -source = "git+https://github.com/tikv/rusoto?branch=cse#d6ffce8898ce5d03ada531401437985b727393a4" +source = "git+https://github.com/tikv/rusoto?branch=cse#187f471cc34e1eeb5bb2f3c0e9564284a441593f" dependencies = [ "async-trait", "chrono", @@ -5740,7 +5740,7 @@ dependencies = [ [[package]] name = "rusoto_s3" version = "0.46.0" -source = "git+https://github.com/tikv/rusoto?branch=cse#d6ffce8898ce5d03ada531401437985b727393a4" +source = "git+https://github.com/tikv/rusoto?branch=cse#187f471cc34e1eeb5bb2f3c0e9564284a441593f" dependencies = [ "async-trait", "bytes", @@ -5752,7 +5752,7 @@ dependencies = [ [[package]] name = "rusoto_signature" version = "0.46.0" -source = "git+https://github.com/tikv/rusoto?branch=cse#d6ffce8898ce5d03ada531401437985b727393a4" +source = "git+https://github.com/tikv/rusoto?branch=cse#187f471cc34e1eeb5bb2f3c0e9564284a441593f" dependencies = [ "base64 0.13.1", "bytes", @@ -6273,12 +6273,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - [[package]] name = "shlex" version = "1.3.0" @@ -7357,7 +7351,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tipb" version = "0.0.1" -source = "git+https://github.com/pingcap/tipb.git#a4d204a193b4f9aa776343ac75281cf8442343ba" +source = "git+https://github.com/pingcap/tipb.git#7ed1acfec6949dda0395f6615671b542779e1b79" dependencies = [ "futures 0.3.32", "grpcio", @@ -7669,7 +7663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.7.3", + "rand 0.8.6", "static_assertions", ] @@ -8602,7 +8596,7 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yatp" version = "0.0.1" -source = "git+https://github.com/tikv/yatp.git?branch=master#64ccad68f86f06c2cbb5a95fb3258c2c092ecea0" +source = "git+https://github.com/tikv/yatp.git?branch=master#7ca1723bea900d7f0258de73f44d93cbc7b4326a" dependencies = [ "crossbeam-deque", "crossbeam-skiplist", diff --git a/contrib/tiflash-proxy-cmake/CMakeLists.txt b/contrib/tiflash-proxy-cmake/CMakeLists.txt index 283fad864e6..07dff1a1728 100644 --- a/contrib/tiflash-proxy-cmake/CMakeLists.txt +++ b/contrib/tiflash-proxy-cmake/CMakeLists.txt @@ -174,6 +174,24 @@ set(_TIFLASH_PROXY_CUSTOM_DEPENDS "${_TIFLASH_PROXY_MAKEFILE}" "${_TIFLASH_PROXY_CARGO_MANIFEST}" "${_TIFLASH_PROXY_RUST_TOOLCHAIN}") + +# In columnar next-gen mode, the hub links crates from the local +# cloud-storage-engine checkout through Cargo's [patch] entries. Those files +# are outside _TIFLASH_PROXY_SOURCE_DIR, so they must be added explicitly; +# otherwise CMake considers libtiflash_proxy.so up to date after a local Rust +# change and never invokes Cargo. +if (ENABLE_NEXT_GEN_COLUMNAR) + set(_CLOUD_STORAGE_ENGINE_SOURCE_DIR "${TiFlash_SOURCE_DIR}/contrib/cloud-storage-engine") + if (EXISTS "${_CLOUD_STORAGE_ENGINE_SOURCE_DIR}") + file(GLOB_RECURSE _CLOUD_STORAGE_ENGINE_SRCS CONFIGURE_DEPENDS + "${_CLOUD_STORAGE_ENGINE_SOURCE_DIR}/*.rs" + "${_CLOUD_STORAGE_ENGINE_SOURCE_DIR}/*.toml" + "${_CLOUD_STORAGE_ENGINE_SOURCE_DIR}/*.proto") + list(FILTER _CLOUD_STORAGE_ENGINE_SRCS EXCLUDE REGEX "/target/") + list(APPEND _TIFLASH_PROXY_CUSTOM_DEPENDS ${_CLOUD_STORAGE_ENGINE_SRCS}) + endif() +endif() + if (EXISTS "${_TIFLASH_PROXY_CARGO_LOCK}") list(APPEND _TIFLASH_PROXY_CUSTOM_DEPENDS "${_TIFLASH_PROXY_CARGO_LOCK}") endif() diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index f92575e0ddb..ce207f0a698 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -140,7 +140,11 @@ struct RNColumnarReaderSharedContext ClearSharedSnapAccessByStartTsFn clear_shared_snap_access_by_start_ts = nullptr; std::shared_ptr output_lock = std::make_shared(); bool registered_for_start_ts = false; - std::atomic_bool late_materialization_logged{false}; + // Reader capability is checked independently for every region reader. Report each state + // at most once so an empty region cannot hide a later enabled reader without producing one + // INFO line for every region in a large scan. + std::atomic_bool late_materialization_enabled_logged{false}; + std::atomic_bool late_materialization_disabled_logged{false}; ~RNColumnarReaderSharedContext() noexcept { @@ -1049,33 +1053,93 @@ std::unordered_set RNColumnarReadTask::getLateMaterializationEarlyColu return column_ids; } -bool RNColumnarReadTask::isLateMaterializationFilterEligible() const +bool RNColumnarReadTask::isLateMaterializationFilterEligible(String * reason) const { + const auto setReason = [reason](const char * value) { + if (reason != nullptr) + *reason = value; + }; + if (has_multi_table_reader_plan) + { + setReason("multi_table_reader_plan"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is ineligible: reason=multi_table_reader_plan, executor_id={}, table_id={}", + getExecutorID(), + getLogicalTableID()); return false; + } if (!shared_reader_context->has_pushed_down_filter_conditions || shared_reader_context->exact_filter_conditions.empty()) + { + setReason("no_pushed_down_filter"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is ineligible: reason=no_pushed_down_filter, has_pushed_down_filter={}, exact_filter_conditions={}, executor_id={}, table_id={}", + shared_reader_context->has_pushed_down_filter_conditions, + shared_reader_context->exact_filter_conditions.size(), + getExecutorID(), + getLogicalTableID()); return false; + } const auto column_ids = getExactFilterColumnIDs(); if (column_ids.find(MutSup::extra_table_id_col_id) != column_ids.end()) + { + setReason("filter_uses_extra_table_id"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is ineligible: reason=filter_uses_extra_table_id, executor_id={}, table_id={}", + getExecutorID(), + getLogicalTableID()); return false; + } for (const auto & column : shared_reader_context->scan_columns) { if (column_ids.find(column.id) == column_ids.end()) continue; if (column.hasGeneratedColumnFlag()) + { + setReason("filter_uses_generated_column"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is ineligible: reason=filter_uses_generated_column, column_id={}, executor_id={}, table_id={}", + column.id, + getExecutorID(), + getLogicalTableID()); return false; + } const bool needs_timezone_cast = !shared_reader_context->context->getTimezoneInfo().is_utc_timezone && column.tp == TiDB::TypeTimestamp; if (needs_timezone_cast || column.tp == TiDB::TypeTime) + { + setReason("unsupported_filter_column_type"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is ineligible: reason=unsupported_filter_column_type, column_id={}, timezone_cast={}, executor_id={}, table_id={}", + column.id, + needs_timezone_cast, + getExecutorID(), + getLogicalTableID()); return false; + } } + setReason("eligible"); + LOG_DEBUG( + shared_reader_context->log, + "Columnar late materialization filter is eligible: filter_columns={}, exact_filter_conditions={}, executor_id={}, table_id={}", + column_ids.size(), + shared_reader_context->exact_filter_conditions.size(), + getExecutorID(), + getLogicalTableID()); return true; } -bool RNColumnarReadTask::shouldLogLateMaterialization() +bool RNColumnarReadTask::shouldLogLateMaterialization(bool enabled) { - return !shared_reader_context->late_materialization_logged.exchange(true); + auto & logged = enabled ? shared_reader_context->late_materialization_enabled_logged + : shared_reader_context->late_materialization_disabled_logged; + return !logged.exchange(true); } void RNColumnarReadTask::replaceReaderWork( @@ -1503,7 +1567,21 @@ void RNColumnarInputStream::initializeLateMaterialization() if (late_materialization_initialized) return; late_materialization_initialized = true; - if (context.getSettingsRef().enable_columnar_l2_late_materialization && task->isLateMaterializationFilterEligible()) + const bool setting_enabled = context.getSettingsRef().enable_columnar_l2_late_materialization; + bool filter_eligible = false; + String disable_reason = setting_enabled ? "filter_ineligible" : "setting_disabled"; + if (setting_enabled) + filter_eligible = task->isLateMaterializationFilterEligible(&disable_reason); + + LOG_DEBUG( + log, + "Columnar late materialization prerequisites: setting_enabled={}, filter_eligible={}, executor_id={}, table_id={}", + setting_enabled, + filter_eligible, + executor_id, + table_id); + + if (setting_enabled && filter_eligible) { const auto early_column_ids = task->getLateMaterializationEarlyColumnIDs(); size_t late_column_count = 0; @@ -1533,10 +1611,47 @@ void RNColumnarInputStream::initializeLateMaterialization() reinterpret_cast(encoded_ids.data()), encoded_ids.size() * sizeof(Int64)}); if (supported) + { late_materialization_interfaces = interfaces; + disable_reason = "enabled"; + LOG_DEBUG( + log, + "Columnar late materialization enabled after reader support check: executor_id={}, table_id={}", + executor_id, + table_id); + } else - LOG_DEBUG(log, "Columnar late materialization is unavailable for this reader"); + { + disable_reason = "reader_unsupported"; + LOG_DEBUG( + log, + "Columnar late materialization is unavailable for this reader: executor_id={}, table_id={}", + executor_id, + table_id); + } } + else + { + disable_reason = "interfaces_unavailable"; + LOG_DEBUG( + log, + "Columnar late materialization is unavailable: interfaces missing or ABI incompatible, executor_id={}, table_id={}", + executor_id, + table_id); + } + } + else + { + disable_reason = late_column_count == 0 ? "no_late_columns" : "late_to_early_ratio_below_threshold"; + LOG_DEBUG( + log, + "Columnar late materialization skipped by column ratio: late_columns={}, early_columns={}, late_to_early_ratio={:.3f}, min_ratio={:.3f}, executor_id={}, table_id={}", + late_column_count, + early_column_count, + late_to_early_ratio, + context.getSettingsRef().columnar_l2_late_materialization_min_late_to_early_ratio, + executor_id, + table_id); } LOG_DEBUG( log, @@ -1546,12 +1661,23 @@ void RNColumnarInputStream::initializeLateMaterialization() late_to_early_ratio, context.getSettingsRef().columnar_l2_late_materialization_min_late_to_early_ratio); } + else if (!setting_enabled) + { + LOG_DEBUG( + log, + "Columnar late materialization skipped: setting enable_columnar_l2_late_materialization is false, executor_id={}, table_id={}", + executor_id, + table_id); + } - if (task->shouldLogLateMaterialization()) + if (task->shouldLogLateMaterialization(late_materialization_interfaces != nullptr)) LOG_INFO( log, - "Columnar late materialization enabled={}, executor_id={}, table_id={}", + "Columnar late materialization enabled={}, scope=reader, reason={}, setting_enabled={}, filter_eligible={}, executor_id={}, table_id={}", late_materialization_interfaces != nullptr, + disable_reason, + setting_enabled, + filter_eligible, executor_id, table_id); } diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index 1ff8726fac7..c56dbd3815d 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -164,9 +164,9 @@ class RNColumnarReadTask std::unordered_set getLateMaterializationEarlyColumnIDs() const; - bool isLateMaterializationFilterEligible() const; + bool isLateMaterializationFilterEligible(String * reason = nullptr) const; - bool shouldLogLateMaterialization(); + bool shouldLogLateMaterialization(bool enabled); RNColumnarReadTask( std::vector reader_plans, From 5e2cb85e3583f2370e2fefc29ec37b7744a4e86b Mon Sep 17 00:00:00 2001 From: yongman Date: Wed, 26 Aug 2026 17:01:10 +0800 Subject: [PATCH 08/10] polish Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- dbms/src/Storages/StorageDisaggregatedColumnar.cpp | 5 ----- dbms/src/Storages/StorageDisaggregatedColumnar.h | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 7404c56f9a9..9542a9eb612 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 7404c56f9a9b1089222a4bb4e01926bb8bb43177 +Subproject commit 9542a9eb6123a46f91ab31257882a3ed51d2a635 diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index ce207f0a698..37d58777616 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -1001,11 +1001,6 @@ const String & RNColumnarReadTask::getExecutorID() const return shared_reader_context->executor_id; } -const google::protobuf::RepeatedPtrField & RNColumnarReadTask::getExactFilterConditions() const -{ - return shared_reader_context->exact_filter_conditions; -} - google::protobuf::RepeatedPtrField RNColumnarReadTask::getLateMaterializationFilterConditions( const Block & early_block) const { diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.h b/dbms/src/Storages/StorageDisaggregatedColumnar.h index c56dbd3815d..10974df86a0 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.h +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.h @@ -155,8 +155,6 @@ class RNColumnarReadTask const String & getExecutorID() const; - const google::protobuf::RepeatedPtrField & getExactFilterConditions() const; - google::protobuf::RepeatedPtrField getLateMaterializationFilterConditions( const Block & early_block) const; From 3e14f6951101a18dda3aa4cacbd8985c584fefe7 Mon Sep 17 00:00:00 2001 From: yongman Date: Wed, 26 Aug 2026 17:23:45 +0800 Subject: [PATCH 09/10] add design doc Signed-off-by: yongman --- ...8-l2-only-columnar-late-materialization.md | 906 ++++++++++++++++++ 1 file changed, 906 insertions(+) create mode 100644 docs/design/2026-08-18-l2-only-columnar-late-materialization.md diff --git a/docs/design/2026-08-18-l2-only-columnar-late-materialization.md b/docs/design/2026-08-18-l2-only-columnar-late-materialization.md new file mode 100644 index 00000000000..25ea9990229 --- /dev/null +++ b/docs/design/2026-08-18-l2-only-columnar-late-materialization.md @@ -0,0 +1,906 @@ +# L2-Only Late Materialization for Columnar Reads + +- Author(s): TBD +- Status: Implemented (default off) +- Last Updated: 2026-08-26 +- Discussion PR: TBD +- Tracking Issue: TBD + +## Table of Contents + +- [Introduction](#introduction) +- [Motivation or Background](#motivation-or-background) +- [Goals](#goals) +- [Non-Goals](#non-goals) +- [Terminology](#terminology) +- [Current Implementation](#current-implementation) +- [Detailed Design](#detailed-design) + - [Scope and Eligibility](#scope-and-eligibility) + - [Column Projections](#column-projections) + - [Hybrid Batch Representation](#hybrid-batch-representation) + - [Source Reader Behavior](#source-reader-behavior) + - [L2 Physical Row References](#l2-physical-row-references) + - [Merge and MVCC](#merge-and-mvcc) + - [Exact Predicate Evaluation](#exact-predicate-evaluation) + - [Late Pack Planning](#late-pack-planning) + - [Variable-Width Column Packs](#variable-width-column-packs) + - [FFI Protocol](#ffi-protocol) + - [Pending Batch State Machine](#pending-batch-state-machine) + - [Cost Model and Metrics](#cost-model-and-metrics) + - [Implementation Mapping](#implementation-mapping) +- [Correctness Invariants](#correctness-invariants) +- [Compatibility](#compatibility) +- [Failure Handling](#failure-handling) +- [Test Design](#test-design) +- [Impacts & Risks](#impacts--risks) +- [Investigation & Alternatives](#investigation--alternatives) +- [Rollout Plan](#rollout-plan) +- [Unresolved Questions](#unresolved-questions) + +## Introduction + +This implementation adds late materialization to the cloud columnar read path, but only +for rows read by the level-2 (L2) `ColumnarConcatReader`. Memtable rows, +unconverted L0 row data, and L0/L1 columnar files continue to read all requested +columns eagerly. A scan does not need to contain only L2 data. + +The optimization divides requested physical columns into an early projection +and a late projection. All sources participate in the existing global merge and +MVCC pass using the early projection. Every row that survives MVCC carries +either an eager late-value reference or an L2 physical-row reference. TiFlash +evaluates the exact predicate over the merged, MVCC-visible early rows. The Rust +reader then combines selected eager values with late values loaded from L2. + +The late L2 I/O unit is a complete pack of one late column. A pack is read once +if it contains at least one selected row and is skipped completely otherwise. +The late path never calls `ColumnarColumnReader::set_row_idx()`. Pack-local row +selection happens after the complete pack has been read, decrypted, and +decompressed. + +## Motivation or Background + +The current columnar path constructs a complete Rust `Block` before it crosses +the FFI boundary. TiFlash then deserializes all requested columns and executes +the exact pushed-down predicate. For a selective query over a wide table, most +bytes in non-predicate columns may therefore be read, decompressed, serialized, +copied across FFI, and deserialized only to be filtered out immediately. + +L2 is the best initial scope for this optimization: + +- L2 files are consumed by `ColumnarConcatReader` in key order. +- L2 contains most of the data in the target deployments. This is an expected + workload property and must be validated by level-specific metrics; it is not + a format invariant. +- L2 rows can be identified by a snapshot-pinned file and an absolute physical + row index. +- L0/L1 and row sources can retain their current eager I/O behavior, which + avoids implementing random late access for every source type. + +The restricted scope does not remove the need for a hybrid merge protocol. +`ColumnarMergeReader` merges all sources by handle and version, and +`ColumnarMvccReader` decides which version is visible. Exact predicate selection +must happen after both operations. + +For example: + +```text +L1: handle=42, version=200, predicate=false +L2: handle=42, version=100, predicate=true +read_ts >= 200 +``` + +The L1 row suppresses the older L2 row during MVCC, even though the L1 row does +not satisfy the predicate. Filtering the L2 row before the global merge would +incorrectly return version 100. The required order is: + +```text +memtable/L0/L1: full data retained in an eager sidecar +L2 concat: early columns plus a physical-row reference + | + global merge and MVCC + | + TiFlash exact predicate selection + | + eager gather plus selected complete-pack L2 reads + | + final TiFlash block assembly +``` + +## Goals + +1. Avoid reading complete L2 late-column packs that contain no selected rows. +2. Avoid serializing unselected late values across FFI for all source types. +3. Preserve the current global merge order and MVCC visibility semantics across + memtable, unconverted L0, L0/L1 columnar files, and L2 files. +4. Reuse TiFlash's current expression analyzer and execution semantics for exact + predicate evaluation. +5. Support fixed-width and variable-width late columns without assuming their + pack boundaries are aligned. +6. Keep the legacy full-materialization reader and FFI contract available as a + safe fallback. +7. Make performance benefits and regressions observable by source level, + column kind, and touched-pack ratio. + +## Non-Goals + +- Late materialization for memtable, unconverted L0, or L0/L1 columnar data. +- A requirement that a scan, range, or batch contain only L2 rows. +- Row-granular or byte-range reads within a compressed column pack. +- Calling `set_row_idx()` once per selected row or range. +- Reusing handle-pack identities for string, bytes, or JSON columns. +- Implementing a second SQL expression evaluator in `kvengine`. +- Supporting ANN, vector-distance projection, or FTS readers in the first + version. +- Changing the columnar on-disk format or compaction rules. + +## Terminology + +- **Early column**: a physical column required by merge/MVCC, exact predicate + evaluation, or both. +- **Late column**: a requested output column not needed to produce the exact + predicate selection. +- **Eager row**: a row from memtable, unconverted L0, or an L0/L1 columnar file. + Its late values are read during the early phase and retained in Rust. +- **Deferred row**: an L2 row whose late values are represented by a physical + row reference until selection is known. +- **Relevant pack**: a late-column pack containing at least one deferred row in + the pending batch before exact selection. +- **Touched pack**: a relevant pack containing at least one selected deferred + row. +- **Pending batch**: the Rust-owned early or materialized state between + `read_early` and either `finish_materialized_block` or `discard_batch`. + +## Current Implementation + +The implemented legacy read sequence is: + +```text +RNColumnarInputStream + -> CloudStorageEngineInterfaces::fn_read_block + -> CloudColumnarReader / CloudColumnarReaders + -> ColumnarMvccReader + -> ColumnarMergeReader + -> row readers for memtable and unconverted L0 + -> one ColumnarTableReader per L0/L1 file + -> one ColumnarConcatReader over L2 files + -> complete Rust Block + -> per-column FFI serialization and TiFlash deserialization + -> extraCast + -> exact pushed-down filter +``` + +When LM is enabled for one reader work, `RNColumnarInputStream` uses the +optional two-phase extension instead. It reads and deserializes only the early +header columns, evaluates a `FilterTransformAction`, passes its exact +selection to CSE, obtains selected late columns, and then rebuilds the original +header. The existing downstream filter remains in the pipeline and is applied +again to the assembled block; LM uses the first evaluation only to avoid late +column work, not to change filter ownership. + +The relevant implementation is summarized below. + +| Responsibility | Current code | Relevant behavior | +| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| TiFlash block read | [`StorageDisaggregatedColumnar.cpp`](../../dbms/src/Storages/StorageDisaggregatedColumnar.cpp) | `RNColumnarInputStream::readImpl` calls `fn_read_block`, then fetches and deserializes every header column. | +| Exact filter construction | [`StorageDisaggregatedColumnar.cpp`](../../dbms/src/Storages/StorageDisaggregatedColumnar.cpp), [`InterpreterUtils.cpp`](../../dbms/src/Flash/Coprocessor/InterpreterUtils.cpp) | External filter conditions and `TableScan` pushed filters are combined and executed with `DAGExpressionAnalyzer`. | +| Reader construction | [`read.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/read.rs) | `new_columnar_mvcc_reader_impl` builds row, L0/L1 table, L2 concat, merge, and MVCC readers. | +| Multi-table buffering | [`read.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/read.rs) | `CloudColumnarReaders` and `BlockResult` eagerly extract every column from a completed block in concurrent mode. | +| Global merge | [`reader.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/reader.rs) | `ColumnarMergeReader::read` appends rows in handle-ascending and version-descending order. | +| MVCC | [`reader.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/reader.rs) | `ColumnarMvccReader::try_read_block` removes future versions, older versions, tombstones, and rows beyond the range. | +| L2 concatenation | [`reader.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/reader.rs) | `ColumnarConcatReader` finishes one L2 `ColumnarTableReader` before advancing to the next file. | +| L2 ordering | [`columnar.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/columnar.rs) | `ColumnarLevel::sort` sorts level 2 by each file's smallest key. | +| Pack positioning | [`reader.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/reader.rs), [`columnar.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/columnar.rs) | `set_row_idx` uses `PackOffsets::search_pack_idx`, which is a forward linear search followed by pack loading. | +| Pack construction | [`builder.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/builder.rs) | Fixed-width columns follow handle pack completion; variable-width columns split independently by row and byte limits. | +| Rough filtering | [`filter.rs`](../../contrib/cloud-storage-engine/components/kvengine/src/table/columnar/filter.rs) | Rust evaluates pack-level min/max conditions, not exact row-level SQL predicates. | + +Changing only `CloudColumnarReader::ffi_read_column` cannot provide the desired +optimization. At that point the complete Rust block has already loaded and +decompressed all requested L2 columns. + +## Detailed Design + +### Scope and Eligibility + +The implementation enables LM per `RNColumnarInputStream`, rather than once +for an entire query. It requires all of the following: + +1. `enable_columnar_l2_late_materialization` is true. +2. The weak symbol `tiflash_columnar_get_late_materialization_interfaces` is + present and returns ABI version 1, a descriptor at least as large as the + v1 descriptor, and every required callback. +3. At least one TableScan pushed-down filter exists. The exact condition set is + `filter_conditions.conditions` followed by the TableScan pushed filters. +4. The exact condition set has a non-empty physical column dependency set, and + the source header has at least one late column. +5. The exact predicate does not reference the synthetic extra table-ID column + or a generated column. `TIME`, and `TIMESTAMP` outside UTC, are also + rejected because the local early-action representation has not been proven + equivalent for those cases. +6. `late_column_count / max(1, early_column_count - 2)` is strictly greater + than `columnar_l2_late_materialization_min_late_to_early_ratio`. The two + subtracted early columns are handle and version. +7. CSE accepts the reader: it rejects ANN/FTS, multi-table sequential reads, + worker aggregation, missing schema/L2 files, missing handle/version in the + early projection, and overlapping L2 files for the target table. + +After CSE has produced the first non-empty early batch, TiFlash evaluates the +exact selection. If its skip ratio is below +`columnar_l2_late_materialization_min_selection_skip_ratio`, TiFlash discards +that pending LM batch, disables LM for this input stream, and immediately reads +the next block through the legacy ABI. This is a runtime probe, not a +creation-time byte or pack cost model. + +Memtable, unconverted L0, and L0/L1 data do not disable LM. They produce eager +rows in the same hybrid batch. A capable reader can also produce an eager-only +batch; it remains valid and still uses the two-phase protocol. Pack-clean is +not selected by an LM-specific chooser: the hybrid reader deliberately does not +use pack-clean because it requires real handle and version values. + +### Column Projections + +Reader creation derives three ordered sets from the existing physical source +header: + +```text +full_output_columns = physical columns currently returned by the source +predicate_columns = transitive physical dependencies of the exact predicate +early_input_columns = internal handle/version plus predicate_columns +early_output_columns = full_output_columns intersect early_input_columns +late_columns = full_output_columns minus early_output_columns +``` + +The internal handle and version are always early because merge and MVCC require +them, even when they are not part of the query output. All sources use the same +early schema so that `ColumnarMergeReader` continues to compare homogeneous +blocks. The original output order is stored separately and used during final +assembly. + +Column identities are physical column IDs, not names or positions. PK-handle +aliases and internal handle/version columns must be normalized during plan +construction. Default-value columns remain valid late columns; they generate +selected values by count and require no pack read. + +### Hybrid Batch Representation + +The merge path gains an LM-only representation. The existing `Block` and +normal `read` methods remain unchanged. + +```rust +enum MaterializationRef { + Eager { + eager_row_idx: u32, + }, + L2Deferred { + l2_reader_id: u32, + file_index: u32, + physical_row_idx: u32, + }, +} + +struct HybridBatch { + early: Block, + refs: Vec, + eager_late: Vec, +} +``` + +`early.rows() == refs.len()` is mandatory. `eager_late` contains only late +columns and only rows copied from eager sources. `eager_row_idx` indexes the +batch-owned `ColumnBuffer` vectors. + +The batch-owned eager block is deliberate. A raw `{source_id, +source_block_row}` reference is unsafe because `ColumnarMergeReader` may exhaust +and refill a source more than once while constructing one output batch. At the +same append point where an eager early slice is added to the merged block, its +late slice is appended to `eager_late` and stable indexes are emitted. Source +buffers may then refill normally. + +After MVCC, the implementation retains the eager sidecar and its stable +`eager_row_idx` values; it filters only `early` and `refs`. Exact selection +builds `HybridMaterializationPlan` once, mapping selected output rows to either +an eager sidecar index or an L2 deferred slot. No eager-sidecar compaction or +index rewriting occurs. + +### Source Reader Behavior + +| Source | Early phase | Materialization reference | Late phase | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------- | ----------------------------------------------------------------------------- | +| Memtable and unconverted L0 row reader | Decode the same full row as today; append early columns and copy late values into the batch sidecar. | `Eager` | Gather selected rows from `eager_late`; no second source read. | +| L0/L1 `ColumnarTableReader` | Read the same full requested block as today; append early columns and copy late values into the batch sidecar. | `Eager` | Gather selected rows from `eager_late`; no random access. | +| L2 `ColumnarConcatReader` | Read handle, version, and early columns only. | `L2Deferred` | Read complete touched packs for every late column and gather selected values. | + +Only L2 avoids late-column I/O. Eager sources may still benefit from not +serializing unselected late rows across FFI, but their disk/decode work is not +reported as an LM I/O saving. + +### L2 Physical Row References + +`ColumnarTableReader::read_with_physical_rows` is the LM companion API. It +creates an early block and returns physical rows while preserving normal +`read` behavior. `ColumnarConcatReader::read_with_physical_rows` attaches its +stable file index to those rows. + +For each appended L2 row, the early reader records the absolute row index in +that file/table. The index is derived from the aligned version reader (or handle +reader) pack's absolute row start plus its pack-local cursor before the append; +it is not derived from a variable-width predicate column. Packs rejected by +rough filtering produce no rows and no references. `Unknown` rough-filter +results are still read. + +`ColumnarConcatReader` assigns a stable `l2_reader_id` and a stable index in its +snapshot-pinned, table-filtered file vector. A batch may contain references to +multiple L2 files because merge output can cross a concat boundary or interleave +L1 and L2 rows. Late reading groups references by `(l2_reader_id, file_index)`; +it does not relocate rows by handle. + +Late accessors are built from the pinned `ColumnarFile` and column metadata, not +from the concat reader's current cursor. Advancing the concat reader therefore +does not invalidate an earlier physical reference. + +### Merge and MVCC + +`HybridMergeReader` is a dedicated counterpart to `ColumnarMergeReader`. It +uses the same handle-ascending/version-descending heap ordering, but emits +`HybridBatch`: eager sources retain their late `ColumnBuffer` sidecars and L2 +sources emit `L2Deferred` references. Its comparator and heap behavior do not +change. + +`ColumnarMvccState::apply_hybrid_visibility` reuses the normal reader's +visibility-range calculation. The normal path applies that result to a complete +`Block`; the hybrid path applies it to: + +- the early block; +- the materialization-reference vector; and + +The eager sidecar is intentionally not compacted. + +There must not be separate implementations of read-ts checks, same-handle +deduplication, tombstone handling, range-end handling, or int/common-handle +logic. Predicate selection is produced only after this shared MVCC step. + +### Exact Predicate Evaluation + +Rust rough filtering remains a pack-elimination optimization. It must not be +extended into an exact SQL evaluator. Exact selection remains owned by +TiFlash's `DAGExpressionAnalyzer` and the current expression actions. + +When LM is enabled, `RNColumnarInputStream` builds one cached +`FilterTransformAction` for that input stream. It copies and remaps the same +combined condition set used by `filterConditionsWithPushedDownFilters`: + +```text +filter_conditions.conditions AND table_scan.getPushedDownFilters() +``` + +`ColumnRef` operands in the copied expressions are remapped from the TableScan +column index to the early-header index. The original protobuf expressions are +not mutated. For each pending batch, TiFlash performs the following steps: + +1. Deserialize the early physical columns into `early_block`. +2. Copy it into `evaluation_block` and execute the cached filter action there. +3. Send `All`, `None`, or the resulting `UInt8` filter to CSE with the batch ID. +4. Apply that same selection to the unmodified early columns. +5. Deserialize selected late columns and assemble the original full source + header. + +The evaluation copy prevents predicate-only casts and temporary columns from +changing the early columns used in final assembly. `action.fill` still adds the +physical table ID, and the existing downstream filter is intentionally retained +in both stream and pipeline paths. Therefore an LM-result block is filtered +once for selection and again by the normal pipeline. The second application is +the current compatibility guard; it must not be removed without an explicit +semantic-equivalence and profiling change. + +If a reader is ineligible before its first LM batch, `readImpl` uses the legacy +full-block ABI. If the first exact-selection probe is too dense, TiFlash +discards that batch and makes the same per-stream fallback. Neither fallback +changes downstream filter ownership. + +### Late Pack Planning + +No selected row invokes `set_row_idx()`. Rust builds a batch-level plan after it +receives the exact selection: + +```rust +struct LateReadSlot { + physical_row_idx: u32, + selected_output_idx: u32, +} + +struct LatePackPlan { + pack_idx: u32, + slots: Vec, +} + +struct LateColumnPlan { + column_id: i64, + packs: Vec, +} + +struct LateFilePlan { + l2_reader_id: u32, + file_index: u32, + columns: Vec, +} +``` + +Plan construction and execution are: + +1. Scan selected materialization references. Eager references are gathered from + `eager_late`; L2 references are grouped by reader and file. Every selected + row is assigned its stable `selected_output_idx` in merged/MVCC order. +2. For each late column independently, map every physical row to a pack using + that column's `ColumnMeta.pack_offsets.find_pack_idx`, which binary-searches + that column's `row_offsets`. It does not use the cursor-oriented + `search_pack_idx`. +3. Coalesce slots with the same `(file, column, pack)` into one + `LatePackPlan`. Sort pack plans by pack index to preserve sequential access + opportunities. +4. Construct the late `ColumnarColumnReader` or dedicated + `LateColumnAccessor` with `packs_filter = None`. +5. Load, decrypt, and decompress each touched pack exactly once for the pending + batch. Extract selected pack-local rows from the complete `ColumnBuffer`. +6. Store fragments with `selected_output_idx`, then assemble each late column + in selected merged/MVCC order together with eager fragments. +7. Serialize only selected rows to TiFlash. + +If the selection is empty, Rust releases the pending batch without reading any +late pack. If it selects every row, the same plan remains correct and will touch +all relevant packs. The implementation may coalesce adjacent pack reads in the +future, but the logical accounting and at-most-once rule remain per column pack. + +### Variable-Width Column Packs + +String, bytes, JSON, and other variable-width columns do not share pack +boundaries with handle or fixed-width columns. + +This follows directly from the current builder and reader: + +- `ColumnarColumnBuilder::append` passes the handle `finish_pack` decision to + fixed-width columns. +- `append_var` splits a variable-width column according to its own accumulated + row count and byte size. +- `ColumnarTableReader::new` sets `packs_filter = None` for columns whose fixed + size is zero. + +Therefore, `physical_row_idx` is the only shared identity across late columns. +Pack identity is scoped to `(file, column)`. The implementation must obey these +rules: + +1. Every late column performs its own physical-row-to-pack lookup. +2. A handle or fixed-column pack index is never reused for a variable-width + column. +3. A fixed-column rough-filter bitmap is never passed to a variable-width late + accessor. +4. Touched-pack count, compressed bytes, and read ranges are calculated per + column before being aggregated for the batch. +5. A selected row may map to unrelated pack IDs in two late columns, and the + plan must represent that normally. + +The complete-pack rule is especially important here. If one selected row falls +in a large string pack, the complete string pack is read and decompressed once; +the implementation does not attempt a row-sized I/O operation inside it. + +### FFI Protocol + +The current `fn_get_columnar_reader`, `fn_read_block`, and per-column read +functions remain unchanged. LM is an implemented optional extension, exported +from the Hub through the weak C symbol +`tiflash_columnar_get_late_materialization_interfaces()`. The v1 descriptor is +validated by its `version`, minimum `size`, and all callback pointers; it does +not change `CloudStorageEngineInterfaces` or `RaftStoreProxyFFIHelper` layout. +Its callback contract is: + +```text +read_early_block(reader, limit, early_column_ids, &batch_id, &physical_table_id) + -> row_count | 0 (EOF) | UINT64_MAX (error) + +read_early_column(reader, batch_id, column_id) + +materialize_selected(reader, batch_id, selection_kind, selection_bytes) + -> selected_row_count | UINT64_MAX (error) + +read_late_column(reader, batch_id, column_id) +finish_materialized_block(reader, batch_id) -> 1 | 0 +discard_late_materialization_batch(reader, batch_id) -> 1 | 0 +is_late_materialization_supported(reader, early_column_ids) -> 1 | 0 +``` + +`selection_kind` is `All`, `None`, or `Bytes`. `Bytes` uses the existing +TiFlash filter representation: one `UInt8` per input row, where zero is false +and non-zero is true. `All` and `None` carry no payload and avoid allocating a +uniform selection buffer. + +The protocol requirements are: + +- LM entry points are used only when every required function pointer is + non-null and the interface version is supported. +- `batch_id` is unique within a reader instance and is validated by every batch + operation. +- For `Bytes`, CSE validates that payload length equals the pending + MVCC-output row count. `All` and `None` have an empty payload. +- `materialize_selected` returns the number of selected rows: the number of + non-zero bytes for `Bytes`, the pending row count for `All`, and zero for + `None`. +- `read_late_column` is valid only after successful materialization and before + the batch is finished. Each late column can be taken only once. +- Every returned early and late column is validated against its expected row + count after TiFlash deserialization. +- Early and late Rust buffers use the existing Rust GC ownership convention. + TiFlash releases each returned buffer after deserialization. +- `physical_table_id` is attached to the early batch and cannot change while it + is pending. +- A reader that has returned a non-empty early batch cannot switch to the + legacy reader for that batch. The only density fallback discards the first + probe batch before reading a separate legacy block. + +The FFI header, generated Rust bindings, proxy implementation, and columnar Hub +are updated together. Future descriptor revisions must retain this lockstep +rule. A binary without the complete LM extension continues to use the legacy +full-materialization path. + +### Pending Batch State Machine + +Each Rust reader permits at most one globally merged pending batch: + +```text +Idle --read_early(non-empty)------> PendingEarly(batch_id) +Idle --read_early(EOF)------------> Drained +PendingEarly --materialize(valid)-> Materialized(batch_id) +PendingEarly --discard(valid)-----> Idle +Materialized --finish(valid)------> Idle +Materialized --discard(valid)-----> Idle +PendingEarly/Materialized --drop/cancel/error--> caller discards, then Idle +``` + +While either batch state is active, the reader rejects another early read, +seek, reset, or range switch. `PendingEarly` owns the early buffers, references, +eager sidecar, and pinned L2 metadata. Successful materialization replaces that +state with `Materialized`, which owns the selected late buffers until TiFlash +takes every expected late column and calls `finish_materialized_block`. Because +eager values are copied into the sidecar during merge, source readers may refill +while the batch is being constructed; they simply cannot be advanced by a +second batch after the first batch becomes pending. + +TiFlash keeps a scope guard armed from early-batch acquisition through +`finish_materialized_block`; any exception before finish calls +`discard_late_materialization_batch`. CSE returns protocol or storage errors to +the caller; it has no persistent `Failed` state. A partially assembled block is +not returned because TiFlash throws before final assembly and the scope guard +discards the pending batch. + +### Cost Model and Metrics + +Row selectivity alone is not a reliable estimate because late I/O is saved only +when complete packs are skipped. For every late column, the implementation +uses the following batch-local quantities: + +```text +relevant_packs = packs containing any deferred row in the pending batch +touched_packs = relevant packs containing any selected deferred row + +touched_pack_ratio = touched_packs / relevant_packs +touched_byte_ratio = compressed bytes of touched packs + / compressed bytes of relevant packs +``` + +The implemented admission controls are intentionally simpler than a byte-cost +model: the early/late column-count ratio is checked before reading, and the +first exact batch must satisfy the configured skip ratio. A dense probe is +discarded and the reader switches to legacy mode for its remaining lifetime. + +The currently exported metric is +`late_materialization_skipped_packs`. For each late column, CSE counts candidate +packs referenced by deferred rows minus packs actually loaded for selected +rows, and aggregates the result in `ColumnarRuntimeStats` and TiFlash's +`ColumnarScanContext`. Existing read, serialization, rough-filter and +deserialize timings remain available. Per-column byte ratios, eager-sidecar +bytes, repeated-load counters, and pending-batch memory metrics are not +implemented yet and must not be treated as rollout signals. + +### Implementation Mapping + +| Area | Implemented behavior | +| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `components/kvengine/src/read.rs` | Builds `HybridMvccReader`, owns the pending-batch protocol, validates IDs and selections, and materializes all late columns concurrently (bounded to four) on the first late-column request. | +| `table/columnar/reader.rs` | Defines `HybridBatch`, `HybridMergeReader`, `HybridMvccReader`, physical L2 references, `HybridMaterializationPlan`, and `LateColumnAccessor`. Normal readers remain available. | +| `table/columnar/columnar.rs` | Provides `PackOffsets::find_pack_idx`, the binary physical-row-to-pack lookup. | +| `StorageDisaggregatedColumnar.cpp/.h` | Derives early IDs, checks TiFlash-side eligibility, resolves the optional ABI, executes the cached filter action, probes selection density, calls the two-phase protocol, and preserves the legacy reader plus downstream filter. | +| Settings | Adds the default-off `enable_columnar_l2_late_materialization`, the minimum early/late ratio (default `10.0`), and the minimum first-batch skip ratio (default `0.5`). | +| Hub FFI | Exports the v1 extension descriptor and bridges callbacks to `CloudColumnarReaders`; the main Cloud Storage Engine interface remains ABI-stable. | +| Build dependencies | `contrib/tiflash-proxy-cmake/CMakeLists.txt` tracks local CSE Rust/TOML/proto inputs when next-gen columnar is enabled, so a CSE change rebuilds the linked Hub. | + +## Correctness Invariants + +The implementation must assert or test the following invariants: + +1. Exact selection is evaluated in the row space produced by the global merge + and MVCC pass, never in a source-local row space. +2. Each MVCC-visible row has exactly one `MaterializationRef`. +3. `HybridBatch.early.rows() == HybridBatch.refs.len()` before and after MVCC. +4. MVCC applies the same retained ranges to early rows and references. +5. Each selected row produces exactly one late value for every late output + column, whether the row is eager or deferred. +6. Final row order is merged/MVCC order restricted by the exact TiFlash + selection. File/column/pack read order never changes output order. +7. An L2 physical row reference is resolved only against its snapshot-pinned + file and table metadata. +8. Pack identity is per `(file, column)`; only physical row identity is shared + across columns. +9. A touched `(file, column, pack)` is loaded at most once per pending batch, + and an untouched pack is not loaded by the late path. +10. Late access never calls `set_row_idx()`; the early L2 reader can still use + its existing skip-pack repositioning. +11. Rough filtering may reduce I/O but cannot replace or weaken exact + filtering. LM selection and the existing downstream filter must use the + same combined conditions. +12. No pending batch survives reader release, cancellation, retry, or error. + +## Compatibility + +### Query Semantics + +The result rows, order, column types, errors, timezone behavior, collation +behavior, null semantics, and JSON guard behavior must match the current full +reader. The design reuses the current TiFlash expression actions specifically +to avoid semantic drift. + +Partition reads remain supported when the predicate does not depend on the +synthetic extra table-ID column. The early batch carries `physical_table_id`, +and final assembly retains the current extra table-ID fill behavior. + +Generated output columns may continue to use the current placeholder and later +generation flow. Predicates depending on generated columns are excluded from +V1 because their early dependency and evaluation ordering are not currently +represented by the Rust reader contract. + +### FFI and Upgrade Compatibility + +Legacy function pointers and behavior remain available. LM is enabled only when +TiFlash observes the complete supported extension. Missing functions, an +unsupported interface version, or a CSE capability rejection selects the full +reader before any early batch is acquired. A dense first batch is separately +discarded before legacy fallback. + +All in-process FFI definitions must be regenerated and shipped consistently for +TiFlash, proxy variants, and the columnar hub. Mixed binaries that cannot prove +the extension layout use the legacy interface; they must not infer capability +from one non-null function pointer. + +No persistent data format changes are introduced, so downgrade only requires +disabling the feature or running a binary without the extension. + +### Other Features + +- ANN, vector-distance projection, and FTS use the current reader path. +- Pack-clean and LM are mutually exclusive reader modes. +- Rough filtering remains conservative and may run before exact selection. +- Encryption and IA/remote segment reads remain in the existing `PackLoader`; + the late accessor changes which complete packs are requested, not how a pack + is decoded. +- Lock resolution, region retry, snapshot lifetime, and range replanning retain + their current outer control flow. Pending state must be discarded before a + reader is released for retry. + +## Failure Handling + +| Failure | Required behavior | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| Static query is not eligible | Build the existing pipeline and reader. | +| FFI extension missing or unsupported | Use the legacy full reader; do not acquire an early batch. | +| Snapshot has no eligible L2 file or the overlap check fails | Report LM unsupported to TiFlash; it keeps the legacy reader for that reader work. | +| Invalid batch ID, selection length, or column request | Return a protocol error; TiFlash's scope guard discards the pending batch. | +| TiFlash predicate evaluation throws | Discard the pending batch with a scope guard, then rethrow the original exception. | +| Empty selection | Discard the pending batch without late-pack reads and continue to the next early batch. | +| Late pack I/O, key lookup, decrypt, checksum, or decompress failure | Propagate the storage error; the scope guard discards pending state. | +| TiFlash late-column row count mismatch | Treat as a logical/protocol error and release the reader. | +| Cancellation or reader drop while pending | Abort and release eager sidecar, refs, accessors, and FFI buffers. | +| Dense or fragmented selection | Complete the same pack plan, potentially reading all relevant packs; never switch readers mid-batch. | + +## Test Design + +### Functional Tests + +Implemented Rust unit coverage includes hybrid early/ref retention, stable eager +sidecar indexes, selected L2 late-value materialization, and the strict L2 +non-overlap gate. The following broader cases remain required: + +1. L1 version 200 with predicate false and L2 version 100 with predicate true; + the old L2 version must not appear. Include the reverse predicate outcome, + future versions, tombstones, and read-ts boundaries. +2. Interleaved memtable, unconverted L0, L0, L1, and multiple L2 files. Compare + early rows, refs, selected late values, and final order with the complete + reader. +3. Eager sidecar stability when one eager source refills multiple times while a + merged batch is being built. +4. Sparse, contiguous, empty, and all-row L2 selections across one and multiple + files and packs. +5. Fixed-width, nullable, default, decimal, string, bytes, and JSON late + columns. +6. Fixed and string pack misalignment produced with a small byte limit. Verify + that string rows use the string column's own offsets and no fixed-column + rough-filter bitmap. +7. One complete load per touched `(file, column, pack)`, zero loads for + untouched packs, and no `set_row_idx` call from late access. +8. L2 rough-filter `None` and `Unknown` results mixed with eager rows, with no + false negative. +9. Invalid state transitions, duplicate materialization, bad selection sizes, + wrong batch IDs, late I/O failure, cancellation, and drop. +10. Gating for no predicate, no late column, no L2, overlapping L2 ranges, ANN, + FTS, vector projection, generated/extra-table-ID predicate, pack-clean, + missing FFI, and internal concurrent mode. + +TiFlash unit and integration tests still need to cover: + +1. Early projection extraction and original-header reconstruction. +2. Exact selection for AND, OR, NOT, NULL, casts, timestamp/timezone, + collation, and guarded JSON expressions. +3. Combined external filter conditions and `TableScan` pushed filters in the + current order, including LM early-block selection and the retained downstream + filter. +4. Empty/all/partial selections and early/late row-count validation. +5. Full-read local filtering when an individual reader does not support LM, + while the pipeline uses materialization-action filter ownership. +6. Exception scope guards that discard a pending Rust batch. + +### Scenario Tests + +End-to-end tests compare feature-on and feature-off results for: + +- mixed L1/L2 versions and tombstones; +- partition tables and multiple physical tables; +- multiple key ranges and batches crossing L2 file boundaries; +- region retry, lock error, source cancellation, and reader release; +- local, remote, IA-cached, encrypted, and cache-miss pack reads; +- scans where a batch contains only eager rows even though the reader is LM + capable. + +### Compatibility Tests + +- New TiFlash with a complete LM-capable proxy/hub. +- New TiFlash with the LM extension absent or disabled. +- Feature flag toggled off with new binaries. +- Restart and downgrade after LM queries, confirming that no persistent state + or format migration exists. +- Both legacy stream and pipeline execution models. + +### Benchmark Tests + +Benchmarks must vary: + +- L2 share and eager-source share; +- projected late-column width; +- fixed-width versus variable-width late data; +- row selectivity and actual touched-pack ratio; +- clustered versus scattered selected rows; +- one versus many L2 files; +- local, remote, IA cache-hit, and IA cache-miss reads; +- pack sizes, including packs above the direct-read cache threshold; +- batch size and L1/L2 interleaving frequency. + +Report wall time, CPU, peak memory, I/O bytes, decompressed bytes, FFI bytes, +pack metrics, and stage timing. Promotion beyond default-off requires a clear +reduction in L2 late bytes at low touched-pack ratios without material +regression for dense or eager-heavy batches. + +## Impacts & Risks + +### Expected Impacts + +- Selective wide scans can skip complete L2 late-column packs. +- Only selected late values cross FFI, including values from eager sources. +- L0/L1/memtable I/O behavior remains unchanged. +- LM evaluates an additional early-block selection before the normal downstream + filter, which adds CPU work but preserves existing final filter ownership. +- Rust retains a batch-owned eager late sidecar and L2 metadata until selection + completes. + +### Risks + +1. **Dense selection regression:** if almost every relevant pack is touched, + two-phase calls, early evaluation, sidecar management, and final assembly add + overhead without L2 I/O savings. +2. **Variable-width amplification:** one selected large string may cause a large + complete pack read even at low row selectivity. +3. **Memory growth:** eager source payloads and pending early data coexist until + materialization; mixed-source batches can increase peak memory. +4. **Expression drift:** remapped ColumnRefs or independently rebuilt actions + could change the selection. The normal downstream filter is retained as a + compatibility guard; equivalence tests are still required. +5. **State-machine bugs:** cancellation, retry, or exception paths could leave a + pending batch or stale FFI buffer alive. +6. **Pack-plan bugs:** treating pack IDs as cross-column identities would return + wrong variable-width values. +7. **Concurrency loss:** disabling `CloudColumnarReaders` internal worker mode + may offset LM gains for multi-table reads. +8. **Pack-clean regression:** reading real handle/version columns for LM can be + slower than the existing clean-pack path. +9. **Remote access fragmentation:** many touched packs can create more small + remote/cache operations even though each pack is loaded only once. + +The default-off flag, strict gates, per-reason metrics, and workload-based +rollout are required mitigations. + +## Investigation & Alternatives + +### Defer All Source Types + +This would maximize possible I/O savings but requires stable late access for +memtable, row data, L0, and overlapping L1 files. It also expands physical-row +identity and lifetime rules substantially. L2-only deferral captures the +expected dominant data level with a smaller correctness surface. + +### Require Pure-L2 Scans + +This avoids an eager sidecar but rejects normal scans containing recent L1 or +memtable updates. More importantly, it is unnecessary: a hybrid batch can keep +eager payloads while preserving one global merge and MVCC pass. + +### Filter Each L2 File Before Merge + +This is incorrect because newer eager or L1 versions must suppress older L2 +versions before predicate filtering. The version-200/version-100 example in the +background section demonstrates the failure. + +### Reevaluate the Exact Predicate in Rust + +The current Rust filter parser is a conservative pack-level rough filter. A +second row-level evaluator would need to reproduce TiFlash casts, collations, +timezone behavior, null semantics, JSON guards, and errors. Reusing TiFlash's +current expression actions is both smaller and safer. + +### Reposition with `set_row_idx()` + +Calling `set_row_idx()` for selected rows or short ranges can repeatedly search +pack metadata and load/decode packs. It is also difficult to reason about when +fixed and variable-width packs are misaligned. The proposed plan maps rows once +per column, loads each touched pack once, and extracts locally. + +### Use Handle Pack IDs for Every Late Column + +This is invalid for variable-width columns because their pack boundaries are +independent. Physical row index is the cross-column identity; pack index is not. + +### Read Full L2 Late Columns and Filter Only at FFI + +This reduces serialization but does not save storage reads, decryption, +decompression, or Rust block memory. It may be a useful fallback behavior for +eager sources, but it does not meet the primary goal. + +### Row-Sized Reads Inside a Pack + +Compressed packs are the existing independent decode unit. Reading a row-sized +fragment cannot decode the row without the rest of the pack and would add +format-specific random I/O complexity. Complete touched packs are the V1 unit. + +## Rollout Plan + +1. **Completed: protocol and L2 implementation.** Capability plumbing, hybrid + batches, the one-pending-batch protocol, L2 complete-pack access, dense-probe + fallback, and the default-off settings are implemented. +2. **Current: controlled validation.** Validate touched-pack savings, + dense-selection regressions, variable-width behavior, remote/IA access, and + the simple ratio/skip-ratio gates against end-to-end workloads. +3. **Future: limited production rollout.** Enable by workload or tenant + allowlist, monitor rejection reasons, errors, memory, and stage latency, and + retain an immediate runtime disable switch. +4. **Future: broader enablement.** Consider default-on only after correctness + equivalence and regression thresholds are met. Extending deferral to other + levels requires a separate proposal. + +## Unresolved Questions + +1. Should the simple early/late ratio and first-batch skip-ratio gates be + replaced with a pack or byte-aware model? +2. Should a future implementation compact eager sidecar rows after MVCC, or is + retaining them until batch completion faster and sufficiently bounded? +3. What pending batch byte limit should supplement the existing row batch size + for wide eager-source data? +4. Can internal `CloudColumnarReaders` concurrency later preserve one pending + batch per worker without eagerly draining columns into `BlockResult`? +5. Should adjacent touched packs be coalesced into larger remote reads while + retaining per-pack decode and metrics? +6. Which generated-column and hidden-column predicates can be admitted after + proving an early evaluation header equivalent to the current pipeline? +7. How should source-level exact-filter profile time be attributed so query + profiles remain comparable with the current downstream filter operator? From 1a5f0cb2256b5bcebd9c018b6ff09feccdd64bb7 Mon Sep 17 00:00:00 2001 From: yongman Date: Wed, 26 Aug 2026 18:22:50 +0800 Subject: [PATCH 10/10] remove skipped packs count Signed-off-by: yongman --- contrib/cloud-storage-engine | 2 +- .../hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 1 - .../tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs | 1 - contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs | 1 - dbms/src/Flash/Coprocessor/ColumnarScanContext.h | 5 ----- .../2026-08-18-l2-only-columnar-late-materialization.md | 6 +----- 6 files changed, 2 insertions(+), 14 deletions(-) diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 9542a9eb612..09db4d27101 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 9542a9eb6123a46f91ab31257882a3ed51d2a635 +Subproject commit 09db4d27101ed5692b57b777fa375550fb2e6d15 diff --git a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index ef321d19b32..10bc9c4c657 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -223,7 +223,6 @@ struct ColumnarScanStats { uint64_t rough_check_selected_packs; uint64_t rough_check_skipped_packs; uint64_t rough_check_unknown_packs; - uint64_t late_materialization_skipped_packs; uint64_t remote_segments; uint64_t total_segments; }; diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs index d11760b951d..a6cc6df656d 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs @@ -88,7 +88,6 @@ impl From for ColumnarScanStats rough_check_selected_packs: stats.rough_check_selected_packs, rough_check_skipped_packs: stats.rough_check_skipped_packs, rough_check_unknown_packs: stats.rough_check_unknown_packs, - late_materialization_skipped_packs: stats.late_materialization_skipped_packs, remote_segments: stats.remote_segments, total_segments: stats.total_segments, } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs index 766a37181f2..8bd900d82ec 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs @@ -292,7 +292,6 @@ pub mod root { pub rough_check_selected_packs: u64, pub rough_check_skipped_packs: u64, pub rough_check_unknown_packs: u64, - pub late_materialization_skipped_packs: u64, pub remote_segments: u64, pub total_segments: u64, } diff --git a/dbms/src/Flash/Coprocessor/ColumnarScanContext.h b/dbms/src/Flash/Coprocessor/ColumnarScanContext.h index e4f43bf5a2e..58fe112926f 100644 --- a/dbms/src/Flash/Coprocessor/ColumnarScanContext.h +++ b/dbms/src/Flash/Coprocessor/ColumnarScanContext.h @@ -47,7 +47,6 @@ class ColumnarScanContext std::atomic rough_check_selected_packs{0}; std::atomic rough_check_skipped_packs{0}; std::atomic rough_check_unknown_packs{0}; - std::atomic late_materialization_skipped_packs{0}; std::atomic remote_segments{0}; std::atomic total_segments{0}; @@ -119,7 +118,6 @@ class ColumnarScanContext rough_check_selected_packs += other.rough_check_selected_packs.load(); rough_check_skipped_packs += other.rough_check_skipped_packs.load(); rough_check_unknown_packs += other.rough_check_unknown_packs.load(); - late_materialization_skipped_packs += other.late_materialization_skipped_packs.load(); remote_segments += other.remote_segments.load(); total_segments += other.total_segments.load(); } @@ -161,7 +159,6 @@ class ColumnarScanContext rough_check_selected_packs += other.rough_check_selected_packs; rough_check_skipped_packs += other.rough_check_skipped_packs; rough_check_unknown_packs += other.rough_check_unknown_packs; - late_materialization_skipped_packs += other.late_materialization_skipped_packs; remote_segments += other.remote_segments; total_segments += other.total_segments; } @@ -176,7 +173,6 @@ class ColumnarScanContext R"(,"user_read_bytes":{},"read_block":"{:.3f}ms","serialize_block":"{:.3f}ms")" R"(,"init_reader":"{:.3f}ms","prefetch":"{:.3f}ms","deserialize_block":"{:.3f}ms")" R"(,"rough_check":{{"total":{},"selected":{},"skipped":{},"unknown":{}}})" - R"(,"late_materialization_skipped_packs":{})" R"(,"remote_segments":{},"total_segments":{}}})", mvcc_input_rows.load(), mvcc_input_bytes.load(), @@ -195,7 +191,6 @@ class ColumnarScanContext rough_check_selected_packs.load(), rough_check_skipped_packs.load(), rough_check_unknown_packs.load(), - late_materialization_skipped_packs.load(), remote_segments.load(), total_segments.load()); } diff --git a/docs/design/2026-08-18-l2-only-columnar-late-materialization.md b/docs/design/2026-08-18-l2-only-columnar-late-materialization.md index 25ea9990229..5c1f3637ea4 100644 --- a/docs/design/2026-08-18-l2-only-columnar-late-materialization.md +++ b/docs/design/2026-08-18-l2-only-columnar-late-materialization.md @@ -589,11 +589,7 @@ model: the early/late column-count ratio is checked before reading, and the first exact batch must satisfy the configured skip ratio. A dense probe is discarded and the reader switches to legacy mode for its remaining lifetime. -The currently exported metric is -`late_materialization_skipped_packs`. For each late column, CSE counts candidate -packs referenced by deferred rows minus packs actually loaded for selected -rows, and aggregates the result in `ColumnarRuntimeStats` and TiFlash's -`ColumnarScanContext`. Existing read, serialization, rough-filter and +Existing read, serialization, rough-filter and deserialize timings remain available. Per-column byte ratios, eager-sidecar bytes, repeated-load counters, and pending-batch memory metrics are not implemented yet and must not be treated as rollout signals.