From 4cac664310dab7fc7fba0105a61e5f7dfc9fe8b9 Mon Sep 17 00:00:00 2001 From: Yi Shan Date: Fri, 29 May 2026 12:13:45 -0700 Subject: [PATCH] Git actions for CI/CD --- .github/workflows/ci.yaml | 60 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- benchmark/Cargo.lock | 55 +++++++--------------------------- fuzz/Cargo.lock | 45 +++++----------------------- src/snapshot.rs | 6 ++-- src/storage.rs | 2 +- src/tests/concurrent.rs | 24 ++++++++++++--- src/utils/mapping_table.rs | 2 +- src/utils/mod.rs | 14 +++++++-- 9 files changed, 116 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..f0060dc --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,60 @@ +name: bftree + +on: + push: + branches: + - main + pull_request: + branches: [main] + +jobs: + check-fmt-build: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - run: rustup component add rustfmt clippy + - run: cargo install -f cargo-fuzz + # Temporarily allow warnings due to spdk feature disabled in benchmark. + - run: cargo clippy --features "metrics-rt, tracing" + - run: cargo clippy --features "metrics-rt-debug-all, tracing" + - run: cargo clippy --features "metrics-rt-debug-timer, tracing" + - run: cargo fmt --all -- --check + - run: cd benchmark && cargo clippy + - run: cd benchmark && cargo clippy --features "metrics-rt" + - run: cd benchmark && cargo clippy --features "metrics-rt-debug-all" + - run: cd benchmark && cargo clippy --features "metrics-rt-debug-timer" + - run: cd fuzz && cargo clippy + - run: cd fuzz && cargo fmt --all -- --check + - run: cd fuzz && cargo build + + test_sans: + needs: check-fmt-build + runs-on: ubuntu-22.04 + timeout-minutes: 60 + env: + RUST_BACKTRACE: 1 + steps: + - uses: actions/checkout@v3 + - run: rustup component add rust-src + - run: rustup toolchain install nightly --component rust-src + - run: sudo apt update && sudo apt install -y llvm-dev + - run: echo "PATH=/usr/lib/llvm-14/bin:$PATH" >> $GITHUB_ENV + - run: cargo test + - run: > + env ASAN_OPTIONS="detect_odr_violation=0" RUSTFLAGS="-Z sanitizer=address" + cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu --tests + # Temporarily disabled + # I believe it is a bug elsewhere, not from us. + # - run: > + # env RUSTFLAGS="-Z sanitizer=memory" + # cargo test -Zbuild-std --target x86_64-unknown-linux-gnu --tests + - run: > + env RUST_BACKTRACE=1 + cargo test --features "shuttle" --release shuttle_bf_tree_concurrent_operations + - run: > + env RUST_BACKTRACE=1 + cargo test --features "shuttle" --release shuttle_cpr_snapshot_disk + - run: > + env RUST_BACKTRACE=1 + cargo test --features "shuttle" --release shuttle_cpr_snapshot_cache_only \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 2b27c94..92049d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bf-tree" -version = "0.5.3" +version = "0.5.4" edition = "2021" license = "MIT" description = "Bf-Tree is a modern read-write-optimized concurrent larger-than-memory range index in Rust from Microsoft Research." diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index fc77432..97ae22e 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.3.4", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -104,7 +104,7 @@ dependencies = [ "libc", "lmdb-rkv", "mimalloc", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "redb", "rocksdb", @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "bf-tree" -version = "0.5.1" +version = "0.5.3" dependencies = [ "auto_ops", "cfg-if", @@ -125,8 +125,7 @@ dependencies = [ "hdrhistogram", "io-uring", "libc", - "rand 0.8.5", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", "thread_local", @@ -411,17 +410,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "getrandom" version = "0.3.4" @@ -575,7 +563,7 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.4", + "getrandom", "libc", ] @@ -946,31 +934,19 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "libc", - "rand_chacha 0.3.1", "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.3", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -986,9 +962,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] [[package]] name = "rand_core" @@ -996,7 +969,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.4", + "getrandom", ] [[package]] @@ -1006,7 +979,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -1320,7 +1293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom", "once_cell", "rustix", "windows-sys 0.61.2", @@ -1435,12 +1408,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 3dfa1f2..217cfee 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -26,13 +26,12 @@ dependencies = [ [[package]] name = "bf-tree" -version = "0.5.1" +version = "0.5.3" dependencies = [ "cfg-if", "io-uring", "libc", - "rand 0.8.5", - "rand 0.9.2", + "rand", "serde", "serde_json", "thread_local", @@ -234,33 +233,12 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -270,16 +248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.12", + "rand_core", ] [[package]] diff --git a/src/snapshot.rs b/src/snapshot.rs index 416c3f4..1225302 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -1179,9 +1179,9 @@ impl CPRSnapShotMgr { recovery_snapshot_vfs.read(*offset, &mut inner_node_page_buffer); let inner_node = InnerNodeBuilder::new().build_from_slice(&inner_node_page_buffer); if unsafe { (*inner_node).is_root() } { - println!("Here"); root_cnt += 1; } + InnerNode::free_node(inner_node); } assert_eq!(root_cnt, 1, "Root count in inner mapping: {}", root_cnt); @@ -2076,7 +2076,7 @@ mod tests { let mut runner = shuttle::PortfolioRunner::new(true, shuttle_config); let available_cores = std::thread::available_parallelism().unwrap().get().min(4); for _ in 0..available_cores { - runner.add(shuttle::scheduler::PctScheduler::new(5, 400)); + runner.add(shuttle::scheduler::PctScheduler::new(10, 1000)); } runner.run(|| { @@ -2203,7 +2203,7 @@ mod tests { let mut runner = shuttle::PortfolioRunner::new(true, shuttle_config); let available_cores = std::thread::available_parallelism().unwrap().get().min(4); for _ in 0..available_cores { - runner.add(shuttle::scheduler::PctScheduler::new(5, 400)); + runner.add(shuttle::scheduler::PctScheduler::new(10, 100)); } runner.run(|| { diff --git a/src/storage.rs b/src/storage.rs index d7a84d1..2c21589 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -17,7 +17,7 @@ use crate::{ fs::{MemoryVfs, StdVfs, VfsImpl}, mini_page_op::{LeafEntrySLocked, LeafEntryXLocked}, nodes::{LeafNode, PageID}, - snapshot::{CPRSnapShotMgr}, + snapshot::CPRSnapShotMgr, utils::{rw_lock::RwLock, MappingTable}, Config, StorageBackend, }; diff --git a/src/tests/concurrent.rs b/src/tests/concurrent.rs index 67c5b81..8c0dc77 100644 --- a/src/tests/concurrent.rs +++ b/src/tests/concurrent.rs @@ -23,6 +23,19 @@ use shuttle::rand::{thread_rng, Rng}; #[cfg(not(feature = "shuttle"))] use rand::Rng; +macro_rules! rand_range { + ($rng:expr, $range:expr) => {{ + #[cfg(feature = "shuttle")] + { + $rng.gen_range($range) + } + #[cfg(not(feature = "shuttle"))] + { + $rng.random_range($range) + } + }}; +} + fn make_key(key: u32, len: usize) -> Vec { let bytes = key.to_ne_bytes(); bytes.into_iter().cycle().take(len).collect::>() @@ -58,22 +71,25 @@ fn concurrent_ops() { let handle = thread::spawn(move || { let mut buffer = vec![0u8; 4096]; + #[cfg(feature = "shuttle")] + let mut rng = thread_rng(); + #[cfg(not(feature = "shuttle"))] let mut rng = rand::rng(); let current_tid = thread::current().id(); black_box(current_tid); for op_n in 0..config.op_cnt_per_thread { black_box(op_n); - match rng.random_range(0..OP_RANGE) { + match rand_range!(rng, 0..OP_RANGE) { 0..=1 => { // insert - let key = rng.random_range(0..config.key_range); + let key = rand_range!(rng, 0..config.key_range); let kv = make_key(key, config.key_len); let _unused = tree_clone.insert(&kv, &kv); } 2 => { // read - let key = rng.random_range(0..config.key_range); + let key = rand_range!(rng, 0..config.key_range); let kv = make_key(key, config.key_len); let cnt = tree_clone.read(&kv, &mut buffer); @@ -86,7 +102,7 @@ fn concurrent_ops() { } 3 => { // delete - let key = rng.random_range(0..config.key_range); + let key = rand_range!(rng, 0..config.key_range); let kv = make_key(key, config.key_len); tree_clone.delete(&kv); } diff --git a/src/utils/mapping_table.rs b/src/utils/mapping_table.rs index 5cd0147..05b0092 100644 --- a/src/utils/mapping_table.rs +++ b/src/utils/mapping_table.rs @@ -52,7 +52,7 @@ impl Default for MappingTable { impl Drop for MappingTable { fn drop(&mut self) { let mut states = self.states.lock().unwrap(); - let initialized_records = states.next_id - 1; + let initialized_records = states.next_id; for i in 0..initialized_records { let batch_id = self.get_batch_id(i); let record_id = self.get_record_id(i); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 4183def..ba00db5 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -147,8 +147,18 @@ impl RngCore for SmallThreadRng { } #[cfg(all(feature = "shuttle", test))] -pub(crate) fn get_rng() -> shuttle::rand::rngs::ThreadRng { - shuttle::rand::thread_rng() +pub(crate) fn get_rng() -> ShuttleThreadRng { + ShuttleThreadRng(shuttle::rand::thread_rng()) +} + +#[cfg(all(feature = "shuttle", test))] +pub(crate) struct ShuttleThreadRng(shuttle::rand::rngs::ThreadRng); + +#[cfg(all(feature = "shuttle", test))] +impl ShuttleThreadRng { + pub(crate) fn random_range(&mut self, range: std::ops::Range) -> usize { + shuttle::rand::Rng::gen_range(&mut self.0, range) + } } pub(crate) enum NodeInfo {