Skip to content

[WIP] columnar: support late materialization #11053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
yongman wants to merge 6 commits into
base: master
Choose a base branch
from
+843 −185
Draft

[WIP] columnar: support late materialization #11053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f6eb96f
columnar: support late materialization
yongman Aug 19, 2026
109c056
add stats for late materialization packs
yongman Aug 20, 2026
778a9d2
polish code
yongman Aug 21, 2026
0f3f38d
disable lm when has multiple tables in one region
yongman Aug 24, 2026
9308dc5
optimize perf
yongman Aug 24, 2026
f82ae57
add more check before enable lm
yongman Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion contrib/cloud-storage-engine
Open in desktop
Submodule cloud-storage-engine updated from 219b17 to a96382
111 changes: 16 additions & 95 deletions contrib/tiflash-columnar-hub/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 27 additions & 0 deletions contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ struct ColumnarScanStats {
uint64_t rough_check_selected_packs;
uint64_t rough_check_skipped_packs;
uint64_t rough_check_unknown_packs;
uint64_t late_materialization_skipped_packs;
uint64_t remote_segments;
uint64_t total_segments;
};
Expand Down Expand Up @@ -261,6 +262,32 @@ struct CloudStorageEngineInterfaces {
ColumnarScanStats (*fn_columnar_scan_stats)(ColumnarReaderPtr);
};

// Optional extension for the two-phase columnar reader protocol. This is
// intentionally discovered through an independently exported, weak symbol so
// that extending it cannot change RaftStoreProxyFFIHelper's ABI.
struct ColumnarLateMaterializationInterfaces {
uint32_t version;
uint32_t size;
uint64_t (*fn_read_early_block)(ColumnarReaderPtr, uint64_t, BaseBuffView,
uint64_t *, int64_t *);
RustStrWithView (*fn_read_early_column)(ColumnarReaderPtr, uint64_t,
int64_t);
uint64_t (*fn_materialize_selected)(ColumnarReaderPtr, uint64_t, uint8_t,
BaseBuffView);
RustStrWithView (*fn_read_late_column)(ColumnarReaderPtr, uint64_t,
int64_t);
uint8_t (*fn_finish_materialized_block)(ColumnarReaderPtr, uint64_t);
uint8_t (*fn_discard_late_materialization_batch)(ColumnarReaderPtr,
uint64_t);
uint8_t (*fn_is_late_materialization_supported)(ColumnarReaderPtr,
BaseBuffView);
};

#if defined(__GNUC__) || defined(__clang__)
extern "C" const ColumnarLateMaterializationInterfaces *
tiflash_columnar_get_late_materialization_interfaces() __attribute__((weak));
#endif

enum class MsgPBType : uint32_t {
ReadIndexResponse = 0,
ServerInfoResponse,
Expand Down
Loading