From 7aa6af26440499e7b979e20a6c7cab63d9fbe114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Tue, 7 Apr 2026 20:02:01 +0800 Subject: [PATCH 01/19] chore: update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10 --- .github/workflows/check.yml | 8 +- .github/workflows/ci.yml | 20 +++++ .github/workflows/deploy.yml | 87 -------------------- .github/workflows/release-plz.yml | 72 +++++++++++++++++ .github/workflows/release.yml | 128 ------------------------------ .github/workflows/test.yml | 4 +- Cargo.toml | 4 +- benches/buddy_allocator.rs | 10 +-- benches/global_allocator.rs | 6 +- benches/slab_allocator.rs | 14 ++-- benches/stability.rs | 24 +++--- 11 files changed, 127 insertions(+), 250 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/release-plz.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4cc8d5c..e827f11 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -62,7 +62,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f08e078 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: + pull_request: + push: + branches-ignore: + - main + tags-ignore: + - "*" + +permissions: + contents: read + +jobs: + check: + uses: ./.github/workflows/check.yml + + test: + uses: ./.github/workflows/test.yml + needs: check diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index d3850e2..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Deploy - -on: - push: - branches: - - "**" - tags-ignore: - - "v*" - - "v*-pre.*" - pull_request: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - check: - uses: ./.github/workflows/check.yml - - test: - uses: ./.github/workflows/test.yml - needs: check - - benchmark: - name: Benchmark check - runs-on: ubuntu-latest - needs: check - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2026-02-25 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Build benchmarks - run: cargo bench --no-run - - build-doc: - name: Build documentation - runs-on: ubuntu-latest - needs: test - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2026-02-25 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Build docs - run: cargo doc --no-deps --features log,tracking - - - name: Create index redirect - run: | - printf '' > target/doc/index.html - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: target/doc - - deploy-doc: - name: Deploy to GitHub Pages - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build-doc - if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml new file mode 100644 index 0000000..0d36a95 --- /dev/null +++ b/.github/workflows/release-plz.yml @@ -0,0 +1,72 @@ +name: Release-plz + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + check: + uses: ./.github/workflows/check.yml + + test: + uses: ./.github/workflows/test.yml + needs: check + + release-plz-release: + name: Release-plz release + runs-on: ubuntu-latest + needs: [check, test] + permissions: + contents: write + pull-requests: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run release-plz + uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128 + with: + command: release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # If you switch to crates.io trusted publishing, remove this line + # and keep `id-token: write` above. + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + release-plz-pr: + name: Release-plz PR + runs-on: ubuntu-latest + needs: [check, test] + permissions: + contents: write + pull-requests: write + concurrency: + group: release-plz-${{ github.ref }} + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run release-plz + uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128 + with: + command: release-pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index e807c93..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Release - -on: - push: - tags: - - "v*.*.*" - - "v*.*.*-pre.*" - -permissions: - contents: write - -jobs: - check: - uses: ./.github/workflows/check.yml - - test: - uses: ./.github/workflows/test.yml - needs: check - - create-release: - name: Create GitHub Release - runs-on: ubuntu-latest - needs: [check, test] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Validate tag and branch (HEAD-based) - shell: bash - run: | - set -e - - TAG="${{ github.ref_name }}" - TAG_COMMIT=$(git rev-list -n 1 "$TAG") - - git fetch origin main dev - - MAIN_HEAD=$(git rev-parse origin/main) - DEV_HEAD=$(git rev-parse origin/dev) - - echo "Tag: $TAG" - echo "Tag commit: $TAG_COMMIT" - echo "main HEAD: $MAIN_HEAD" - echo "dev HEAD: $DEV_HEAD" - - if [[ "$TAG" == *-pre.* ]]; then - if [ "$TAG_COMMIT" != "$DEV_HEAD" ]; then - echo "❌ prerelease tag must be created from dev HEAD" - exit 1 - fi - echo "✅ prerelease tag validated on dev" - else - if [ "$TAG_COMMIT" != "$MAIN_HEAD" ]; then - echo "❌ stable release tag must be created from main HEAD" - exit 1 - fi - echo "✅ stable release tag validated on main" - fi - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - draft: false - prerelease: ${{ contains(github.ref_name, '-pre.') }} - body: | - ## ${{ github.ref_name }} - - - [Documentation](https://docs.rs/buddy-slab-allocator) - - [crates.io](https://crates.io/crates/buddy-slab-allocator) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - publish-crates: - name: Publish to crates.io - runs-on: ubuntu-latest - needs: test - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Validate tag and branch (HEAD-based) - shell: bash - run: | - set -e - - TAG="${{ github.ref_name }}" - TAG_COMMIT=$(git rev-list -n 1 "$TAG") - - git fetch origin main dev - - MAIN_HEAD=$(git rev-parse origin/main) - DEV_HEAD=$(git rev-parse origin/dev) - - echo "Tag: $TAG" - echo "Tag commit: $TAG_COMMIT" - echo "main HEAD: $MAIN_HEAD" - echo "dev HEAD: $DEV_HEAD" - - if [[ "$TAG" == *-pre.* ]]; then - if [ "$TAG_COMMIT" != "$DEV_HEAD" ]; then - echo "❌ prerelease tag must be created from dev HEAD" - exit 1 - fi - echo "✅ prerelease tag validated on dev" - else - if [ "$TAG_COMMIT" != "$MAIN_HEAD" ]; then - echo "❌ stable release tag must be created from main HEAD" - exit 1 - fi - echo "✅ stable release tag validated on main" - fi - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2026-02-25 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Publish to crates.io - run: cargo publish --no-verify --token ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b1b763..51ad5d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,12 +9,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: - toolchain: nightly-2026-02-25 + toolchain: stable - name: Rust Cache uses: Swatinem/rust-cache@v2 diff --git a/Cargo.toml b/Cargo.toml index ee1f184..5235fad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,8 +22,8 @@ cfg-if = "1.0" log = { version = "0.4", optional = true } [dev-dependencies] -criterion = { version = "0.4", features = ["html_reports"] } -rand = { version = "0.8", features = ["small_rng"] } +criterion = { version = "0.8", features = ["html_reports"] } +rand = { version = "0.10", features = ["std_rng"] } [[bench]] name = "global_allocator" diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index b156a49..947a4ff 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -7,7 +7,7 @@ use buddy_slab_allocator::{ BaseAllocator, BuddyPageAllocator, CompositePageAllocator, PageAllocator, }; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{Rng, SeedableRng}; +use rand::{RngExt, SeedableRng}; use std::alloc::Layout; use std::alloc::{alloc, dealloc}; @@ -172,23 +172,23 @@ fn bench_random_pattern(c: &mut Criterion) { c.bench_function("buddy_random_pattern", |b| { let mut allocator = BuddyPageAllocator::::new(); allocator.init(heap_ptr as usize, HEAP_SIZE); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { let mut allocated = Vec::new(); // Random alloc/dealloc pattern for _ in 0..1000 { - if allocated.is_empty() || rng.gen_bool(0.7) { + if allocated.is_empty() || rng.random_bool(0.7) { // Allocate - let pages = 1 << rng.gen_range(0..5); // 1, 2, 4, 8, 16 pages + let pages = 1 << rng.random_range(0..5); // 1, 2, 4, 8, 16 pages match allocator.alloc_pages(pages, PAGE_SIZE) { Ok(addr) => allocated.push((addr, pages)), Err(_) => break, // Out of memory } } else { // Deallocate random item - let idx = rng.gen_range(0..allocated.len()); + let idx = rng.random_range(0..allocated.len()); let (addr, pages) = allocated.swap_remove(idx); allocator.dealloc_pages(addr, pages); } diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index aa5d4e1..792abd4 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -5,7 +5,7 @@ use buddy_slab_allocator::GlobalAllocator; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{Rng, SeedableRng}; +use rand::{RngExt, SeedableRng}; use std::alloc::Layout; use std::alloc::{alloc, dealloc}; @@ -145,12 +145,12 @@ fn bench_random_allocations(c: &mut Criterion) { c.bench_function("global_random_allocations", |b| { let mut allocator = GlobalAllocator::::new(); allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { // 100 random allocations for _ in 0..100 { - let size: usize = rng.gen_range(8..2048); + let size: usize = rng.random_range(8..2048); let size = size.next_power_of_two(); let layout = Layout::from_size_align(size, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index cf1accc..8b2b445 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -7,7 +7,7 @@ use buddy_slab_allocator::{ BaseAllocator, ByteAllocator, CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator, }; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{Rng, SeedableRng}; +use rand::{RngExt, SeedableRng}; use std::alloc::Layout; use std::alloc::{alloc, dealloc}; @@ -156,11 +156,11 @@ fn bench_random_allocations(c: &mut Criterion) { slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); c.bench_function("slab_random_allocations", |b| { - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { for _ in 0..100 { - let size: usize = rng.gen_range(8..2049); + let size: usize = rng.random_range(8..2049); let size = if size.is_power_of_two() { size } else { @@ -240,23 +240,23 @@ fn bench_stress_test(c: &mut Criterion) { slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); c.bench_function("slab_stress_test", |b| { - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { let mut allocated = Vec::new(); // Stress test: rapid alloc/dealloc with varying sizes for _ in 0..1000 { - if allocated.is_empty() || rng.gen_bool(0.7) { + if allocated.is_empty() || rng.random_bool(0.7) { // Allocate - let size = rng.gen_range(8..2049); + let size = rng.random_range(8..2049); let layout = Layout::from_size_align(size, 8).unwrap(); if let Ok(ptr) = slab_alloc.alloc(layout) { allocated.push((ptr, layout)); } } else { // Deallocate random - let idx = rng.gen_range(0..allocated.len()); + let idx = rng.random_range(0..allocated.len()); let (ptr, layout) = allocated.swap_remove(idx); slab_alloc.dealloc(ptr, layout); } diff --git a/benches/stability.rs b/benches/stability.rs index 4139f1f..6de7cf0 100644 --- a/benches/stability.rs +++ b/benches/stability.rs @@ -9,7 +9,7 @@ use buddy_slab_allocator::GlobalAllocator; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{Rng, SeedableRng}; +use rand::{RngExt, SeedableRng}; use std::alloc::Layout; use std::alloc::{alloc, dealloc}; @@ -41,15 +41,15 @@ fn bench_random_pattern_stability(c: &mut Criterion) { group.bench_with_input(BenchmarkId::from_parameter(iter), &iter, |b, _| { let mut allocator = GlobalAllocator::::new(); allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { let mut allocated = Vec::new(); for _ in 0..iter { - if allocated.is_empty() || rng.gen_bool(0.6) { + if allocated.is_empty() || rng.random_bool(0.6) { // 60% allocate - let size = rng.gen_range(8..8193); + let size = rng.random_range(8..8193); let size = if size <= 2048 { size } else { @@ -62,7 +62,7 @@ fn bench_random_pattern_stability(c: &mut Criterion) { } } else { // 40% deallocate - let idx = rng.gen_range(0..allocated.len()); + let idx = rng.random_range(0..allocated.len()); let (ptr, layout) = allocated.swap_remove(idx); allocator.dealloc(ptr, layout); } @@ -222,11 +222,11 @@ fn bench_long_running(c: &mut Criterion) { group.bench_with_input(BenchmarkId::from_parameter(duration), &duration, |b, _| { let mut allocator = GlobalAllocator::::new(); allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { for _ in 0..duration { - let size = rng.gen_range(8..2049); + let size = rng.random_range(8..2049); let layout = Layout::from_size_align(size, 8).unwrap(); let ptr = allocator.alloc(layout); @@ -259,14 +259,14 @@ fn bench_rapid_mixed_sizes(c: &mut Criterion) { c.bench_function("stress_rapid_mixed", |b| { let mut allocator = GlobalAllocator::::new(); allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); b.iter(|| { let mut allocated = Vec::new(); // Rapid mixed-size allocations for _ in 0..5000 { - let size = rng.gen_range(8..16385); + let size = rng.random_range(8..16385); let aligned_size = if size <= 2048 { size } else { @@ -275,7 +275,7 @@ fn bench_rapid_mixed_sizes(c: &mut Criterion) { let layout = Layout::from_size_align(aligned_size, 8).unwrap(); if let Ok(ptr) = allocator.alloc(layout) { - if rng.gen_bool(0.5) { + if rng.random_bool(0.5) { // Immediately free 50% of allocations allocator.dealloc(ptr, layout); } else { @@ -354,11 +354,11 @@ fn bench_memory_leak_detection(c: &mut Criterion) { b.iter(|| { let mut ptrs = Vec::new(); - let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); // Perform random allocations for _ in 0..1000 { - let size = rng.gen_range(8..2049); + let size = rng.random_range(8..2049); let layout = Layout::from_size_align(size, 8).unwrap(); if let Ok(ptr) = allocator.alloc(layout) { ptrs.push((ptr, layout)); From 5cbbdc99b9f4cf322376f1a3b815ed4618f9f18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Tue, 7 Apr 2026 20:07:42 +0800 Subject: [PATCH 02/19] feat: enhance logging support by integrating log crate and updating documentation --- Cargo.toml | 3 +-- README.md | 18 ++++-------------- README_CN.md | 18 ++++-------------- src/buddy/buddy_allocator.rs | 1 - src/buddy/buddy_set.rs | 1 - src/buddy/pooled_list.rs | 1 - src/buddy/stats.rs | 1 - src/global_allocator.rs | 1 - src/lib.rs | 29 +---------------------------- src/page_allocator.rs | 1 - src/slab/slab_byte_allocator.rs | 1 - src/slab/slab_cache.rs | 1 - src/slab/slab_node.rs | 1 - 13 files changed, 10 insertions(+), 67 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5235fad..1a64980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,13 +13,12 @@ categories = ["memory-management", "no-std", "embedded"] [features] default = [] -log = ["dep:log"] tracking = [] [dependencies] axallocator = "0.2" cfg-if = "1.0" -log = { version = "0.4", optional = true } +log = "0.4" [dev-dependencies] criterion = { version = "0.8", features = ["html_reports"] } diff --git a/README.md b/README.md index e3ceb89..f3205d8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ A high-performance page-level and byte-level memory allocator designed for embed - **Composite Page Allocator**: Unified multi-region page allocation interface - **Global Allocator**: Coordinates page and byte allocators with unified allocation interface - **Zero `std` Dependency**: Fully `#![no_std]`, suitable for embedded and kernel environments -- **Conditional Logging**: Support `log` feature for logging, no dependencies by default +- **Logging Support**: Built on the `log` crate and available by default - **Memory Tracking**: Support `tracking` feature for detailed statistics ## Quick Start @@ -21,9 +21,6 @@ Add to your `Cargo.toml`: ```toml [dependencies] buddy-slab-allocator = "0.1.0" - -# Optional features -buddy-slab-allocator = { version = "0.1.0", features = ["log"] } # Enable logging buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # Enable tracking ``` @@ -101,22 +98,18 @@ slab_alloc.dealloc(&mut page_alloc, ptr, layout); ## Features Details -### Conditional Logging - -Enable logging via `log` feature: +### Logging ```toml -buddy-slab-allocator = { version = "0.1.0", features = ["log"] } +buddy-slab-allocator = "0.1.0" ``` -After enabling, you can use standard `log` crate macros to log allocation events: +You can use standard `log` crate macros to log allocation events: ```rust log::info!("Allocated memory at {:x}", addr); ``` -When disabled, log calls are compiled to no-ops with zero runtime overhead. - ### Memory Tracking Enable detailed memory usage tracking via `tracking` feature: @@ -144,9 +137,6 @@ Run the test suite: # Run all tests cargo test --package buddy-slab-allocator -# Run tests with logging enabled -cargo test --package buddy-slab-allocator --features log - # Run tests with tracking enabled cargo test --package buddy-slab-allocator --features tracking ``` diff --git a/README_CN.md b/README_CN.md index bf9e49b..7464cb9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -9,7 +9,7 @@ - **复合页分配器**：统一的多区域页分配接口 - **全局分配器**：协调页分配器和字节分配器，提供统一的分配接口 - **零 `std` 依赖**：完全 `#![no_std]`，适合嵌入式和内核环境 -- **条件日志**：支持 `log` feature 启用日志，默认无依赖 +- **日志支持**：基于 `log` crate，默认可用 - **内存追踪**：支持 `tracking` feature 收集详细统计信息 ## 快速开始 @@ -21,9 +21,6 @@ ```toml [dependencies] buddy-slab-allocator = "0.1.0" - -# 可选功能 -buddy-slab-allocator = { version = "0.1.0", features = ["log"] } # 启用日志 buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # 启用追踪 ``` @@ -101,22 +98,18 @@ slab_alloc.dealloc(&mut page_alloc, ptr, layout); ## 特性详解 -### 条件日志 - -通过 `log` feature 启用日志功能： +### 日志 ```toml -buddy-slab-allocator = { version = "0.1.0", features = ["log"] } +buddy-slab-allocator = "0.1.0" ``` -启用后可使用标准 `log` crate 的宏记录分配事件： +可直接使用标准 `log` crate 的宏记录分配事件： ```rust log::info!("分配内存于 {:x}", addr); ``` -未启用时，日志调用会被编译为空操作，零运行时开销。 - ### 内存追踪 通过 `tracking` feature 启用详细的内存使用追踪： @@ -144,9 +137,6 @@ buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # 运行所有测试 cargo test --package buddy-slab-allocator -# 启用日志运行测试 -cargo test --package buddy-slab-allocator --features log - # 启用追踪运行测试 cargo test --package buddy-slab-allocator --features tracking ``` diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs index 819cd43..a5bfe65 100644 --- a/src/buddy/buddy_allocator.rs +++ b/src/buddy/buddy_allocator.rs @@ -5,7 +5,6 @@ use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator}; -#[cfg(feature = "log")] use log::{debug, error, info, warn}; #[cfg(feature = "tracking")] diff --git a/src/buddy/buddy_set.rs b/src/buddy/buddy_set.rs index 61bfab1..9517c1a 100644 --- a/src/buddy/buddy_set.rs +++ b/src/buddy/buddy_set.rs @@ -5,7 +5,6 @@ use crate::{AllocError, AllocResult}; -#[cfg(feature = "log")] use log::{error, warn}; use super::{ diff --git a/src/buddy/pooled_list.rs b/src/buddy/pooled_list.rs index 546f4ad..aa28634 100644 --- a/src/buddy/pooled_list.rs +++ b/src/buddy/pooled_list.rs @@ -3,7 +3,6 @@ //! Provides linked lists that draw nodes from a shared global pool, //! allowing efficient use of memory across all zones and orders. -#[cfg(feature = "log")] use log::{error, warn}; use super::{buddy_block::BuddyBlock, global_node_pool::GlobalNodePool}; diff --git a/src/buddy/stats.rs b/src/buddy/stats.rs index 1ffe982..6c88cc4 100644 --- a/src/buddy/stats.rs +++ b/src/buddy/stats.rs @@ -65,7 +65,6 @@ impl MemoryStatsReporter { request_align: usize, ) { { - #[cfg(feature = "log")] use log::error; error!("========================================"); error!( diff --git a/src/global_allocator.rs b/src/global_allocator.rs index d5393ee..6dcffce 100644 --- a/src/global_allocator.rs +++ b/src/global_allocator.rs @@ -17,7 +17,6 @@ use super::buddy::BuddyStats; use super::page_allocator::CompositePageAllocator; use super::slab::{PageAllocatorForSlab, SlabByteAllocator}; -#[cfg(feature = "log")] use log::error; const MIN_HEAP_SIZE: usize = 0x8000; // 32KB minimum heap diff --git a/src/lib.rs b/src/lib.rs index 157b6a0..10f66bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ //! - **Slab Byte Allocator**: Fast small object allocation (≤2048 bytes) //! - **Global Allocator**: Automatic selection between page and slab allocation based on size //! - **No_std Compatible**: Fully `#![no_std]` for embedded/kernel use -//! - **Optional Logging**: Conditional compilation with `log` feature +//! - **Logging Support**: Integrates with the `log` crate for allocation diagnostics //! - **Memory Tracking**: Detailed statistics with `tracking` feature //! //! # Quick Start @@ -75,33 +75,6 @@ extern crate alloc; -// Logging support - conditionally import log crate -#[cfg(feature = "log")] -extern crate log; - -// Stub macros when log is disabled - these become no-ops -#[cfg(not(feature = "log"))] -macro_rules! error { - ($($arg:tt)*) => {}; -} -#[cfg(not(feature = "log"))] -macro_rules! warn { - ($($arg:tt)*) => {}; -} -#[cfg(not(feature = "log"))] -macro_rules! info { - ($($arg:tt)*) => {}; -} -#[cfg(not(feature = "log"))] -macro_rules! debug { - ($($arg:tt)*) => {}; -} -#[cfg(not(feature = "log"))] -#[allow(unused_macros)] -macro_rules! trace { - ($($arg:tt)*) => {}; -} - pub use axallocator::{ AllocError, AllocResult, BaseAllocator, ByteAllocator, IdAllocator, PageAllocator, }; diff --git a/src/page_allocator.rs b/src/page_allocator.rs index 6abdff9..2b304ab 100644 --- a/src/page_allocator.rs +++ b/src/page_allocator.rs @@ -3,7 +3,6 @@ use crate::buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER}; use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator}; -#[cfg(feature = "log")] use log::{debug, warn}; /// Maximum number of buddy blocks in a single contiguous allocation diff --git a/src/slab/slab_byte_allocator.rs b/src/slab/slab_byte_allocator.rs index 80d4a17..7c72022 100644 --- a/src/slab/slab_byte_allocator.rs +++ b/src/slab/slab_byte_allocator.rs @@ -6,7 +6,6 @@ use core::alloc::Layout; use core::ptr::NonNull; -#[cfg(feature = "log")] use log::warn; use crate::{AllocError, AllocResult, BaseAllocator, ByteAllocator}; diff --git a/src/slab/slab_cache.rs b/src/slab/slab_cache.rs index e61aaf7..4cea017 100644 --- a/src/slab/slab_cache.rs +++ b/src/slab/slab_cache.rs @@ -3,7 +3,6 @@ //! This module implements SlabCache which manages three lists (empty, partial, full) //! of slab nodes for a specific size class. -#[cfg(feature = "log")] use log::{error, warn}; use super::slab_byte_allocator::{PageAllocatorForSlab as BytePageAllocator, SizeClass}; diff --git a/src/slab/slab_node.rs b/src/slab/slab_node.rs index fabcdfd..00282aa 100644 --- a/src/slab/slab_node.rs +++ b/src/slab/slab_node.rs @@ -3,7 +3,6 @@ //! This module defines the SlabNode structure which manages exactly 512 objects //! using a fixed bitmap. -#[cfg(feature = "log")] use log::error; pub use super::slab_byte_allocator::SizeClass; From bcf744329461fb51506355a4df4b377a2c0cee3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Tue, 7 Apr 2026 20:30:53 +0800 Subject: [PATCH 03/19] feat: refactor allocator interfaces and remove deprecated dependencies; update documentation and examples --- CHANGELOG.md | 18 ++++- Cargo.toml | 1 - README.md | 8 +- README_CN.md | 8 +- benches/buddy_allocator.rs | 4 +- benches/slab_allocator.rs | 4 +- src/buddy/buddy_allocator.rs | 70 +++++++---------- src/error.rs | 15 ++++ src/global_allocator.rs | 134 ++++++++------------------------ src/lib.rs | 7 +- src/page_allocator.rs | 44 +++++------ src/slab/slab_byte_allocator.rs | 35 +++------ tests/integration_test.rs | 7 +- 13 files changed, 141 insertions(+), 214 deletions(-) create mode 100644 src/error.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d7c65..81ffdff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Removed the external allocator-interface dependency and restored crate-local `AllocError` / `AllocResult` +- Stopped exporting the generic allocator traits `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` +- Public allocator APIs now favor concrete allocator methods directly instead of requiring trait imports + +### Removed +- Removed the external allocator-interface crate from `[dependencies]` + +### Migration Notes +- Replace imports of `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` with direct method calls on `BuddyPageAllocator`, `CompositePageAllocator`, `SlabByteAllocator`, and `GlobalAllocator` +- `PageAllocatorForSlab` remains available for wiring `SlabByteAllocator` to a page allocator + ## [0.2.0] - 2026-03-05 ### Added -- Added `axallocator = "0.2"` as a dependency +- Added an external allocator-interface dependency ### Changed -- `AllocError`, `AllocResult`, `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` are now re-exported from `axallocator` instead of being defined locally +- `AllocError`, `AllocResult`, `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` are now re-exported from the external allocator-interface crate instead of being defined locally - Updated Rust toolchain to `nightly-2026-02-25` - Benchmarks no longer require `--features bench`; `criterion` and `rand` moved to `[dev-dependencies]` ### Removed -- Removed locally defined allocator trait and error type definitions (now provided by `axallocator`) +- Removed locally defined allocator trait and error type definitions (now provided by the external allocator-interface crate) - Removed the deprecated `bench` feature flag ## [0.1.1] - 2026-02-06 diff --git a/Cargo.toml b/Cargo.toml index 1a64980..4c9b561 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ default = [] tracking = [] [dependencies] -axallocator = "0.2" cfg-if = "1.0" log = "0.4" diff --git a/README.md b/README.md index f3205d8..a7889c3 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,7 @@ page_alloc.dealloc_pages(addr, 4); ```rust use buddy_slab_allocator::SlabByteAllocator; -use buddy_slab_allocator::page_allocator::PageAllocatorForSlab; -use buddy_slab_allocator::CompositePageAllocator; +use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; @@ -87,13 +86,14 @@ let mut page_alloc = CompositePageAllocator::::new(); page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); let mut slab_alloc = SlabByteAllocator::::new(); +slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); // Small allocations are fast let layout = Layout::from_size_align(64, 8).unwrap(); -let ptr = slab_alloc.alloc(&mut page_alloc, layout).unwrap(); +let ptr = slab_alloc.alloc(layout).unwrap(); // Free memory -slab_alloc.dealloc(&mut page_alloc, ptr, layout); +slab_alloc.dealloc(ptr, layout); ``` ## Features Details diff --git a/README_CN.md b/README_CN.md index 7464cb9..20a92c6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -78,8 +78,7 @@ page_alloc.dealloc_pages(addr, 4); ```rust use buddy_slab_allocator::SlabByteAllocator; -use buddy_slab_allocator::page_allocator::PageAllocatorForSlab; -use buddy_slab_allocator::CompositePageAllocator; +use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; @@ -87,13 +86,14 @@ let mut page_alloc = CompositePageAllocator::::new(); page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); let mut slab_alloc = SlabByteAllocator::::new(); +slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); // 小对象分配快速 let layout = Layout::from_size_align(64, 8).unwrap(); -let ptr = slab_alloc.alloc(&mut page_alloc, layout).unwrap(); +let ptr = slab_alloc.alloc(layout).unwrap(); // 释放内存 -slab_alloc.dealloc(&mut page_alloc, ptr, layout); +slab_alloc.dealloc(ptr, layout); ``` ## 特性详解 diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index 947a4ff..3b6ddf5 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -3,9 +3,7 @@ //! This benchmark suite tests the performance and stability of the Buddy allocator //! which handles page-level allocations with automatic merging. -use buddy_slab_allocator::{ - BaseAllocator, BuddyPageAllocator, CompositePageAllocator, PageAllocator, -}; +use buddy_slab_allocator::{BuddyPageAllocator, CompositePageAllocator}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use rand::{RngExt, SeedableRng}; use std::alloc::Layout; diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index 8b2b445..6a12935 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -3,9 +3,7 @@ //! This benchmark suite tests the performance and stability of the Slab allocator //! which is optimized for small object allocations (≤2048 bytes). -use buddy_slab_allocator::{ - BaseAllocator, ByteAllocator, CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator, -}; +use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use rand::{RngExt, SeedableRng}; use std::alloc::Layout; diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs index a5bfe65..c5c9d08 100644 --- a/src/buddy/buddy_allocator.rs +++ b/src/buddy/buddy_allocator.rs @@ -3,7 +3,7 @@ //! Provides buddy allocator with support for multiple memory zones and //! a single shared global node pool for all zones and orders. -use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator}; +use crate::{AllocError, AllocResult}; use log::{debug, error, info, warn}; @@ -288,6 +288,13 @@ impl BuddyPageAllocator { Ok(()) } + pub fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { + self.add_memory_region(start, size)?; + #[cfg(feature = "tracking")] + self.update_stats(); + Ok(()) + } + /// Find the zone that contains the given address pub fn find_zone_for_addr(&self, addr: usize) -> Option { (0..self.num_zones).find(|&i| self.zones[i].addr_in_zone(addr)) @@ -377,41 +384,8 @@ impl BuddyPageAllocator { pub fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) { // No-op when tracking is disabled } -} - -impl crate::slab::PageAllocatorForSlab for BuddyPageAllocator { - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - ::alloc_pages(self, num_pages, alignment) - } - - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - ::dealloc_pages(self, pos, num_pages); - } -} - -impl Default for BuddyPageAllocator { - fn default() -> Self { - Self::new() - } -} - -impl BaseAllocator for BuddyPageAllocator { - fn init(&mut self, start: usize, size: usize) { - self.bootstrap(start, size); - } - fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { - self.add_memory_region(start, size)?; - #[cfg(feature = "tracking")] - self.update_stats(); - Ok(()) - } -} - -impl PageAllocator for BuddyPageAllocator { - const PAGE_SIZE: usize = PAGE_SIZE; - - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { + pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { // Try to expand node pool if we are close to exhaustion self.maybe_expand_node_pool(); @@ -458,7 +432,7 @@ impl PageAllocator for BuddyPageAllocator { Err(AllocError::NoMemory) } - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { + pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { self.maybe_expand_node_pool(); if let Some(zone_idx) = self.find_zone_for_addr(pos) { self.zones[zone_idx].dealloc_pages(&mut self.global_node_pool, pos, num_pages); @@ -469,7 +443,7 @@ impl PageAllocator for BuddyPageAllocator { } } - fn alloc_pages_at( + pub fn alloc_pages_at( &mut self, base: usize, num_pages: usize, @@ -498,21 +472,21 @@ impl PageAllocator for BuddyPageAllocator { } } - fn total_pages(&self) -> usize { + pub fn total_pages(&self) -> usize { #[cfg(feature = "tracking")] return self.stats.total_pages; #[cfg(not(feature = "tracking"))] return 0; } - fn used_pages(&self) -> usize { + pub fn used_pages(&self) -> usize { #[cfg(feature = "tracking")] return self.stats.used_pages; #[cfg(not(feature = "tracking"))] return 0; } - fn available_pages(&self) -> usize { + pub fn available_pages(&self) -> usize { #[cfg(feature = "tracking")] return self.stats.free_pages; #[cfg(not(feature = "tracking"))] @@ -520,6 +494,22 @@ impl PageAllocator for BuddyPageAllocator { } } +impl crate::slab::PageAllocatorForSlab for BuddyPageAllocator { + fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { + Self::alloc_pages(self, num_pages, alignment) + } + + fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { + Self::dealloc_pages(self, pos, num_pages); + } +} + +impl Default for BuddyPageAllocator { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..e41867d --- /dev/null +++ b/src/error.rs @@ -0,0 +1,15 @@ +/// The error type used for allocation operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AllocError { + /// Invalid size, alignment, or other input parameter. + InvalidParam, + /// Memory added by `add_memory` overlaps with existing memory. + MemoryOverlap, + /// Not enough memory is available to satisfy the request. + NoMemory, + /// Attempted to deallocate memory that was not allocated. + NotAllocated, +} + +/// A [`Result`] alias with [`AllocError`] as the error type. +pub type AllocResult = Result; diff --git a/src/global_allocator.rs b/src/global_allocator.rs index 6dcffce..e42460c 100644 --- a/src/global_allocator.rs +++ b/src/global_allocator.rs @@ -5,7 +5,7 @@ extern crate alloc; -use crate::{AllocError, AllocResult, BaseAllocator, ByteAllocator, PageAllocator}; +use crate::{AllocError, AllocResult}; use core::alloc::Layout; use core::ptr::NonNull; #[cfg(feature = "tracking")] @@ -243,8 +243,9 @@ impl GlobalAllocator { let pages_needed = layout.size().div_ceil(PAGE_SIZE); - let addr = - PageAllocator::alloc_pages(&mut self.page_allocator, pages_needed, layout.align())?; + let addr = self + .page_allocator + .alloc_pages(pages_needed, layout.align())?; let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; #[cfg(feature = "tracking")] @@ -268,7 +269,7 @@ impl GlobalAllocator { /// # Examples /// /// ```no_run - /// use buddy_slab_allocator::{GlobalAllocator, PageAllocator}; + /// use buddy_slab_allocator::GlobalAllocator; /// /// const PAGE_SIZE: usize = 0x1000; /// let mut allocator = GlobalAllocator::::new(); @@ -282,7 +283,7 @@ impl GlobalAllocator { return Err(AllocError::NoMemory); } - let addr = PageAllocator::alloc_pages(&mut self.page_allocator, num_pages, alignment)?; + let addr = self.page_allocator.alloc_pages(num_pages, alignment)?; // Update statistics #[cfg(feature = "tracking")] @@ -318,11 +319,8 @@ impl GlobalAllocator { // This memory was allocated by page allocator let pages_needed = layout.size().div_ceil(PAGE_SIZE); - PageAllocator::dealloc_pages( - &mut self.page_allocator, - ptr.as_ptr() as usize, - pages_needed, - ); + self.page_allocator + .dealloc_pages(ptr.as_ptr() as usize, pages_needed); #[cfg(feature = "tracking")] { saturating_sub_atomic(&self.stats.used_pages, pages_needed); @@ -339,7 +337,7 @@ impl GlobalAllocator { return; } - PageAllocator::dealloc_pages(&mut self.page_allocator, pos, num_pages); + self.page_allocator.dealloc_pages(pos, num_pages); // Update statistics #[cfg(feature = "tracking")] @@ -400,85 +398,7 @@ impl GlobalAllocator { } impl GlobalAllocator { - /// Get memory statistics - #[cfg(feature = "tracking")] - pub fn get_stats(&self) -> UsageStats { - self.stats.snapshot() - } - - /// Get buddy allocator statistics - #[cfg(feature = "tracking")] - pub fn get_buddy_stats(&self) -> BuddyStats { - self.page_allocator.get_buddy_stats() - } -} - -impl Default for GlobalAllocator { - fn default() -> Self { - Self::new() - } -} - -impl BaseAllocator for GlobalAllocator { - fn init(&mut self, start: usize, size: usize) { - self.page_allocator.init(start, size); - } - - fn add_memory(&mut self, start: usize, size: usize) -> AllocResult { - self.page_allocator.add_memory(start, size) - } -} - -impl PageAllocator for GlobalAllocator { - const PAGE_SIZE: usize = PAGE_SIZE; - - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - if !self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::NoMemory); - } - - let addr = as PageAllocator>::alloc_pages( - &mut self.page_allocator, - num_pages, - alignment, - )?; - - // Update statistics - #[cfg(feature = "tracking")] - { - self.stats - .used_pages - .fetch_add(num_pages, Ordering::Relaxed); - self.stats - .free_pages - .fetch_sub(num_pages, Ordering::Relaxed); - } - - Ok(addr) - } - - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - if !self.initialized.load(Ordering::SeqCst) { - return; - } - - as PageAllocator>::dealloc_pages( - &mut self.page_allocator, - pos, - num_pages, - ); - - // Update statistics - #[cfg(feature = "tracking")] - { - saturating_sub_atomic(&self.stats.used_pages, num_pages); - self.stats - .free_pages - .fetch_add(num_pages, Ordering::Relaxed); - } - } - - fn alloc_pages_at( + pub fn alloc_pages_at( &mut self, base: usize, num_pages: usize, @@ -488,14 +408,10 @@ impl PageAllocator for GlobalAllocator { return Err(AllocError::NoMemory); } - let addr = as PageAllocator>::alloc_pages_at( - &mut self.page_allocator, - base, - num_pages, - alignment, - )?; + let addr = self + .page_allocator + .alloc_pages_at(base, num_pages, alignment)?; - // Update statistics #[cfg(feature = "tracking")] { self.stats @@ -509,15 +425,33 @@ impl PageAllocator for GlobalAllocator { Ok(addr) } - fn total_pages(&self) -> usize { + pub fn total_pages(&self) -> usize { self.page_allocator.total_pages() } - fn used_pages(&self) -> usize { + pub fn used_pages(&self) -> usize { self.page_allocator.used_pages() } - fn available_pages(&self) -> usize { + pub fn available_pages(&self) -> usize { self.page_allocator.available_pages() } + + /// Get memory statistics + #[cfg(feature = "tracking")] + pub fn get_stats(&self) -> UsageStats { + self.stats.snapshot() + } + + /// Get buddy allocator statistics + #[cfg(feature = "tracking")] + pub fn get_buddy_stats(&self) -> BuddyStats { + self.page_allocator.get_buddy_stats() + } +} + +impl Default for GlobalAllocator { + fn default() -> Self { + Self::new() + } } diff --git a/src/lib.rs b/src/lib.rs index 10f66bb..98d091b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ //! # Quick Start //! //! ```no_run -//! use buddy_slab_allocator::{GlobalAllocator, PageAllocator}; +//! use buddy_slab_allocator::GlobalAllocator; //! use core::alloc::Layout; //! //! const PAGE_SIZE: usize = 0x1000; @@ -75,9 +75,8 @@ extern crate alloc; -pub use axallocator::{ - AllocError, AllocResult, BaseAllocator, ByteAllocator, IdAllocator, PageAllocator, -}; +mod error; +pub use error::{AllocError, AllocResult}; /// Default page size for backward compatibility (4KB) pub const DEFAULT_PAGE_SIZE: usize = 0x1000; diff --git a/src/page_allocator.rs b/src/page_allocator.rs index 2b304ab..03ff99e 100644 --- a/src/page_allocator.rs +++ b/src/page_allocator.rs @@ -1,7 +1,7 @@ //! Page allocator with contiguous block combination support. use crate::buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER}; -use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator}; +use crate::{AllocError, AllocResult}; use log::{debug, warn}; @@ -120,6 +120,16 @@ impl CompositePageAllocator { self.buddy.alloc_pages_lowmem(num_pages, alignment) } + /// Initialize the allocator with a free memory region. + pub fn init(&mut self, start: usize, size: usize) { + self.buddy.init(start, size); + } + + /// Add a free memory region to the allocator. + pub fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { + self.buddy.add_memory(start, size) + } + /// Try to find and allocate contiguous small blocks from buddy free lists. /// /// This method searches buddy free lists for contiguous blocks that can satisfy @@ -285,12 +295,8 @@ impl CompositePageAllocator { fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) { // No-op when tracking is disabled } -} -impl PageAllocator for CompositePageAllocator { - const PAGE_SIZE: usize = PAGE_SIZE; - - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { + pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { if num_pages == 0 { return Err(AllocError::InvalidParam); } @@ -320,7 +326,7 @@ impl PageAllocator for CompositePageAllocator Ok(base_addr) } - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { + pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { if num_pages == 0 { return; } @@ -355,7 +361,7 @@ impl PageAllocator for CompositePageAllocator /// Allocate contiguous memory pages at a specific address. /// /// Delegates to buddy allocator. - fn alloc_pages_at( + pub fn alloc_pages_at( &mut self, base: usize, num_pages: usize, @@ -365,17 +371,17 @@ impl PageAllocator for CompositePageAllocator } /// Return total number of memory pages. - fn total_pages(&self) -> usize { + pub fn total_pages(&self) -> usize { self.buddy.total_pages() } /// Return number of allocated memory pages. - fn used_pages(&self) -> usize { + pub fn used_pages(&self) -> usize { self.buddy.used_pages() } /// Return number of available memory pages. - fn available_pages(&self) -> usize { + pub fn available_pages(&self) -> usize { self.buddy.available_pages() } } @@ -388,28 +394,16 @@ impl CompositePageAllocator { } } -impl BaseAllocator for CompositePageAllocator { - /// Initialize the allocator with a free memory region. - fn init(&mut self, start: usize, size: usize) { - self.buddy.init(start, size); - } - - /// Add a free memory region to the allocator. - fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { - self.buddy.add_memory(start, size) - } -} - // Implement PageAllocatorForSlab for CompositePageAllocator impl crate::slab::PageAllocatorForSlab for CompositePageAllocator { fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - ::alloc_pages(self, num_pages, alignment) + Self::alloc_pages(self, num_pages, alignment) } fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - ::dealloc_pages(self, pos, num_pages) + Self::dealloc_pages(self, pos, num_pages) } } diff --git a/src/slab/slab_byte_allocator.rs b/src/slab/slab_byte_allocator.rs index 7c72022..a8ae295 100644 --- a/src/slab/slab_byte_allocator.rs +++ b/src/slab/slab_byte_allocator.rs @@ -8,7 +8,7 @@ use core::ptr::NonNull; use log::warn; -use crate::{AllocError, AllocResult, BaseAllocator, ByteAllocator}; +use crate::{AllocError, AllocResult}; // Re-export public types from sibling modules pub use super::slab_cache::SlabCache; @@ -142,25 +142,8 @@ impl SlabByteAllocator { pub fn set_page_allocator(&mut self, page_allocator: *mut dyn PageAllocatorForSlab) { self.page_allocator = Some(page_allocator); } -} -impl Default for SlabByteAllocator { - fn default() -> Self { - Self::new() - } -} - -// This implementation is never called, so no-op implementations are fine -impl BaseAllocator for SlabByteAllocator { - fn init(&mut self, _start: usize, _size: usize) {} - - fn add_memory(&mut self, _start: usize, _size: usize) -> AllocResult { - Ok(()) - } -} - -impl ByteAllocator for SlabByteAllocator { - fn alloc(&mut self, layout: Layout) -> AllocResult> { + pub fn alloc(&mut self, layout: Layout) -> AllocResult> { let size_class = SizeClass::from_layout(layout).ok_or(AllocError::InvalidParam)?; let Some(page_allocator_ptr) = self.page_allocator else { @@ -177,7 +160,7 @@ impl ByteAllocator for SlabByteAllocator { Ok(unsafe { NonNull::new_unchecked(obj_addr as *mut u8) }) } - fn dealloc(&mut self, ptr: NonNull, layout: Layout) { + pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) { let size_class = SizeClass::from_layout(layout).expect("Invalid layout"); let obj_addr = ptr.as_ptr() as usize; @@ -200,19 +183,25 @@ impl ByteAllocator for SlabByteAllocator { self.total_bytes = self.total_bytes.saturating_sub(freed_bytes); } - fn total_bytes(&self) -> usize { + pub fn total_bytes(&self) -> usize { self.total_bytes } - fn used_bytes(&self) -> usize { + pub fn used_bytes(&self) -> usize { self.allocated_bytes } - fn available_bytes(&self) -> usize { + pub fn available_bytes(&self) -> usize { self.total_bytes.saturating_sub(self.allocated_bytes) } } +impl Default for SlabByteAllocator { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/integration_test.rs b/tests/integration_test.rs index a48b633..c2f5675 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -10,8 +10,7 @@ extern crate buddy_slab_allocator; use alloc::vec::Vec; use buddy_slab_allocator::{ - AllocError, BaseAllocator, ByteAllocator, CompositePageAllocator, GlobalAllocator, - PageAllocator, SlabByteAllocator, + AllocError, CompositePageAllocator, GlobalAllocator, PageAllocatorForSlab, SlabByteAllocator, }; use core::alloc::Layout; @@ -166,7 +165,7 @@ fn test_slab_allocator_basic() { slab_allocator.init(); let page_alloc_ptr = &mut page_allocator as *mut CompositePageAllocator - as *mut dyn buddy_slab_allocator::slab::PageAllocatorForSlab; + as *mut dyn PageAllocatorForSlab; slab_allocator.set_page_allocator(page_alloc_ptr); // Test various small allocations @@ -199,7 +198,7 @@ fn test_slab_allocator_many_objects() { slab_allocator.init(); let page_alloc_ptr = &mut page_allocator as *mut CompositePageAllocator - as *mut dyn buddy_slab_allocator::slab::PageAllocatorForSlab; + as *mut dyn PageAllocatorForSlab; slab_allocator.set_page_allocator(page_alloc_ptr); // Allocate many small objects From f4739c90ce86fcc50ff5b0a13a6f89ee3af6fcb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Tue, 7 Apr 2026 21:11:31 +0800 Subject: [PATCH 04/19] Refactor benchmarks and remove stability tests - Consolidated slab allocator benchmarks into a single file, removing redundant code and improving organization. - Removed the stability.rs file, which contained stress tests that are now integrated into the stress_test.rs file. - Introduced a new stress_test.rs file to handle various stress scenarios, including random mixed allocations, exhaustion recovery, and fragmentation recovery. - Updated README.md to reflect changes in the test structure and added instructions for running stress tests. - Removed the test_composite_page_allocator_alloc_at function from integration_test.rs as it was not providing meaningful validation. --- .github/workflows/check.yml | 24 +- .github/workflows/test.yml | 2 +- Cargo.toml | 4 - benches/README_CN.md | 274 ++++------------------ benches/buddy_allocator.rs | 395 +++++++++++++------------------- benches/common.rs | 46 ++++ benches/global_allocator.rs | 290 +++++++---------------- benches/slab_allocator.rs | 407 +++++++++------------------------ benches/stability.rs | 442 ------------------------------------ tests/README.md | 171 +++----------- tests/integration_test.rs | 21 -- tests/stress_test.rs | 174 ++++++++++++++ 12 files changed, 660 insertions(+), 1590 deletions(-) create mode 100644 benches/common.rs delete mode 100644 benches/stability.rs create mode 100644 tests/stress_test.rs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e827f11..af80fd1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -37,7 +37,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run clippy - run: cargo clippy --features log,tracking -- -D warnings + run: cargo clippy --features tracking -- -D warnings build: name: Build @@ -55,7 +55,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build library - run: cargo build --features log,tracking + run: cargo build --features tracking doc: name: Documentation @@ -73,4 +73,22 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build documentation - run: cargo doc --no-deps --features log,tracking + run: cargo doc --no-deps --features tracking + + bench-check: + name: Bench Compile Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Check benchmarks + run: cargo check --benches diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51ad5d7..0ad077b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,4 +20,4 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run all tests - run: cargo test --features log,tracking + run: cargo test --features tracking diff --git a/Cargo.toml b/Cargo.toml index 4c9b561..6b099bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,3 @@ harness = false [[bench]] name = "slab_allocator" harness = false - -[[bench]] -name = "stability" -harness = false diff --git a/benches/README_CN.md b/benches/README_CN.md index fe6b470..c75de09 100644 --- a/benches/README_CN.md +++ b/benches/README_CN.md @@ -1,263 +1,65 @@ # Benchmark 使用说明 -本目录包含 buddy-slab-allocator 项目的性能测试和稳定性测试。 +本目录包含 buddy-slab-allocator 的性能 benchmark，统一基于 `criterion`。 +稳定性、耗尽恢复和长时间随机 workload 已迁移到 `tests/stress_test.rs`，不再作为 timing benchmark 运行。 -## 运行 Benchmark +## 当前 benchmark 套件 -```bash -cargo bench -``` +- `global_allocator.rs` + 全局分配器真实混合 workload。 +- `buddy_allocator.rs` + 页分配、对齐、碎片化恢复、统计查询。 +- `slab_allocator.rs` + 小对象 size class、热路径复用、steady-state 回收。 -### 运行特定 benchmark suite +## 运行方式 ```bash -# 测试全局分配器 -cargo bench --bench global_allocator +# 运行全部 benchmark +cargo bench -# 测试 Buddy 页分配器 +# 运行单个 benchmark suite cargo bench --bench buddy_allocator - -# 测试 Slab 字节分配器 cargo bench --bench slab_allocator +cargo bench --bench global_allocator -# 测试稳定性 -cargo bench --bench stability -``` - -### 高级选项 - -```bash -# 启用 memory tracking 功能运行 benchmark -cargo bench --features tracking +# 仅检查 benchmark 是否可编译 +cargo check --benches -# 保存基线用于后续对比 +# 保存基线 cargo bench -- --save-baseline main # 与基线对比 cargo bench -- --baseline main - -# 只运行特定 benchmark -cargo bench global_alloc_small -``` - -## Benchmark Suite 说明 - -### 1. Global Allocator Benchmarks (`global_allocator.rs`) - -测试全局分配器的统一接口，自动在 Slab 和 Buddy 分配器之间路由。 - -**测试内容**： -- **小对象分配** (8-1024 字节)：使用 Slab 分配器 -- **大对象分配** (>2048 字节)：使用 Buddy 分配器 -- **分配/释放循环**：模拟真实使用场景 -- **随机分配**：不同大小和模式的测试 -- **混合模式**：交替进行小对象和大对象分配 -- **页分配**：直接进行页级分配 - -**测试目标**： -- 验证自动路由机制的效率 -- 评估不同大小分配的性能 -- 测试实际使用场景下的表现 - -### 2. Buddy Allocator Benchmarks (`buddy_allocator.rs`) - -专门测试 Buddy 页分配器的页级内存分配功能。 - -**测试内容**： -- **单页/多页分配**：不同分配大小的测试 -- **带合并的释放**：自动块合并的效率 -- **对齐要求**：不同对齐约束的测试 -- **碎片化抗性**：长期碎片化行为测试 -- **随机模式**：随机分配/释放的压力测试 -- **复合分配器**：连续块组合测试 -- **统计查询**：统计信息查询的性能 - -**测试目标**： -- 验证 Buddy 算法的性能特征 -- 测试自动合并机制的有效性 -- 评估碎片化程度 - -### 3. Slab Allocator Benchmarks (`slab_allocator.rs`) - -专门测试 Slab 字节分配器的小对象分配优化。 - -**测试内容**： -- **大小类别**：所有支持的分配大小（8-2048 字节） -- **释放性能**：小对象的释放速度 -- **对齐支持**：各种对齐要求的测试 -- **随机分配**：不同大小模式的测试 -- **对象池**：重用效率测试 -- **混合大小**：多个大小类别的交互测试 -- **压力测试**：高容量分配压力测试 -- **内存压力**：接近耗尽时的行为测试 - -**测试目标**： -- 验证小对象分配的 O(1) 时间复杂度 -- 测试不同大小类别的性能 -- 评估对象池重用的效率 - -### 4. Stability Benchmarks (`stability.rs`) - -全面的稳定性和压力测试，验证分配器在极端条件下的表现。 - -**测试内容**： -- **随机模式稳定性**：长期运行的随机分配/释放 -- **耗尽处理**：内存限制下的优雅失败 -- **碎片化抗性**：碎片化下的持续使用 -- **交替模式**：特定的分配/释放序列 -- **长时间运行**：扩展持续时间测试 -- **快速混合大小**：不同大小的压力测试 -- **页分配压力**：页级分配的压力测试 -- **内存泄漏检测**：验证没有内存泄漏 -- **边界情况**：最小/最大大小、奇数对齐 - -**测试目标**： -- 验证长时间运行的稳定性 -- 检测内存泄漏 -- 测试极限条件下的行为 -- 确保正确处理边界情况 - -## Benchmark 结果解读 - -### 结果位置 - -运行 benchmark 后，Criterion 会生成详细的 HTML 格式报告： - -- **位置**：`target/criterion/report/index.html` -- **内容**：性能图表、统计分析、对比基线 - -### 报告内容 - -Criterion 生成的报告包含以下信息： - -- **平均时间**：执行的平均耗时 -- **标准差**：性能波动程度 -- **中位数**：典型性能（排除异常值） -- **最小值/最大值**：性能范围 -- **样本大小**：迭代次数 -- **性能图表**：随时间变化的可视化图表 -- **对比分析**：与基线或之前运行的对比 - -### 关键性能指标 - -**需要关注的性能指标**： - -1. **分配速度**： - - 不同大小类别的分配时间 - - 最佳和最差情况 - - 性能一致性 - -2. **释放开销**： - - 释放操作的时间 - - Buddy 系统的合并开销 - - Slab 系统的重用效率 - -3. **碎片化影响**： - - 长期运行后的性能变化 - - 内存利用率 - - 大块分配成功率 - -4. **可扩展性**： - - 增加负载时的性能变化 - - 并发性能（如果测试） - - 极限容量测试 - -**稳定性指标**： - -1. **一致性**： - - 多次运行结果的一致性 - - 标准差大小 - - 无明显性能下降 - -2. **内存泄漏**： - - 使用 `tracking` feature 验证 - - 长时间运行后内存占用稳定 - - 释放后内存完全回收 - -3. **错误处理**： - - 内存耗尽时的优雅失败 - - 边界情况的正确处理 - - 无崩溃或 panic - -## 自定义 Benchmark - -### 修改测试参数 - -可以在 `benches/` 目录下的源文件中修改 benchmark 参数： - -- `HEAP_SIZE`：测试堆大小（默认：16MB） -- `PAGE_SIZE`：页大小（默认：4096 字节） -- 迭代次数：根据硬件调整 - -### 添加新测试 - -1. 在对应的 benchmark 文件中添加新的测试函数 -2. 使用 `criterion` 提供的宏和类型 -3. 在 `criterion_group!` 中注册新测试 -4. 在 `criterion_main!` 中包含新测试组 - -### 示例：添加新的 benchmark - -```rust -use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId}; - -fn bench_my_test(c: &mut Criterion) { - let sizes = [64, 128, 256]; - - let mut group = c.benchmark_group("my_test"); - - for size in sizes.iter() { - group.bench_with_input( - BenchmarkId::from_parameter(size), - size, - |b, _| { - b.iter(|| { - // 你的测试代码 - black_box(size) - }) - } - ); - } - - group.finish(); -} - -criterion_group!(benches, bench_my_test); -criterion_main!(benches); ``` -## 性能优化建议 - -根据 benchmark 结果，可以： - -1. **优化热点路径**：找到最耗时的操作进行优化 -2. **减少碎片化**：调整分配策略或增加内存池 -3. **改进缓存友好性**：优化数据结构和访问模式 -4. **并行化**：如果合适，添加并发支持 +## 设计原则 -## 常见问题 +- 使用 `std::hint::black_box`，不再使用已废弃的 `criterion::black_box` +- 统一 `Criterion` 配置： + - `warm_up_time = 1s` + - `measurement_time = 5s` + - `sample_size = 50` +- 优先 benchmark 成对的 alloc/free，避免单轮把堆推向不同状态 +- 对 workload 型场景使用固定批次操作和固定随机种子 +- 通过 `Throughput::{Elements, Bytes}` 明确每类 benchmark 的吞吐量口径 -### Q: Benchmark 编译失败怎么办？ +## 报告解读 -A: 最常见的原因是 Rust 版本不兼容。请确保使用 **Rust 1.93.0 或更高版本**. +运行 benchmark 后，Criterion 会生成 HTML 报告： -如果仍然失败： -1. 清理并重新构建：`cargo clean && cargo build` -2. 检查网络连接和依赖下载 -3. 更新 rustup：`rustup update` +- 路径：`target/criterion/report/index.html` +- 重点关注： + - 平均时间与置信区间 + - `Bytes/s` 或 `Elements/s` 吞吐量 + - 与 baseline 的回归或改进幅度 -### Q: Benchmark 运行时间太长怎么办？ +## 与压力测试的区别 -A: 可以： -1. 减少迭代次数 -2. 只运行特定的 benchmark -3. 调整 `HEAP_SIZE` 减小测试范围 - -### Q: 如何比较不同版本的性能？ - -A: 使用 Criterion 的基线功能： -```bash -# 第一次运行，保存基线 +- `benches/` + 只负责性能测量，不包含耗尽恢复、泄漏检查等强断言逻辑。 +- `tests/stress_test.rs` + 负责长时间随机、碎片化恢复、耗尽恢复与 tracking 不变量检查。 cargo bench -- --save-baseline v1 # 修改代码后，对比基线 diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index 3b6ddf5..5d39356 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -1,286 +1,205 @@ -//! Benchmarks for BuddyPageAllocator - page-level memory allocation -//! -//! This benchmark suite tests the performance and stability of the Buddy allocator -//! which handles page-level allocations with automatic merging. +//! Benchmarks for buddy-based page allocation paths. -use buddy_slab_allocator::{BuddyPageAllocator, CompositePageAllocator}; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{RngExt, SeedableRng}; -use std::alloc::Layout; -use std::alloc::{alloc, dealloc}; - -const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB - -/// Allocate a test heap from the system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); - (ptr, layout) -} +mod common; -/// Deallocate the test heap -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; -} +use buddy_slab_allocator::{BuddyPageAllocator, CompositePageAllocator}; +use common::{criterion_config, seeded_rng, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; +use rand::RngExt; +use std::hint::black_box; -/// Benchmark single page allocation -fn bench_single_page_alloc(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); +fn bench_page_alloc_free(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/page_alloc_free"); - c.bench_function("buddy_single_page_alloc", |b| { + for &num_pages in &[1usize, 2, 4, 16, 64] { + group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); + let heap = TestHeap::new(HEAP_SIZE); let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); - - b.iter(|| { - let addr = allocator.alloc_pages(black_box(1), PAGE_SIZE); - black_box(addr) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark multiple page allocations -fn bench_multi_page_alloc(c: &mut Criterion) { - let page_counts = [1, 2, 4, 8, 16, 32, 64]; - - let mut group = c.benchmark_group("buddy_multi_page_alloc"); - - for num_pages in page_counts.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(num_pages), num_pages, |b, _| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); - - b.iter(|| { - let addr = allocator.alloc_pages(black_box(*num_pages), PAGE_SIZE); - black_box(addr) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); + allocator.init(heap.addr(), HEAP_SIZE); + + group.bench_with_input( + BenchmarkId::from_parameter(num_pages), + &num_pages, + |b, &pages| { + b.iter(|| { + let addr = allocator.alloc_pages(black_box(pages), PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, pages); + black_box(addr); + }); + }, + ); } group.finish(); } -/// Benchmark deallocation with automatic merging -fn bench_dealloc_with_merge(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); +fn bench_alignment_alloc_free(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/alignment_alloc_free"); - c.bench_function("buddy_dealloc_with_merge", |b| { + for &alignment in &[PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4] { + group.throughput(Throughput::Bytes((4 * PAGE_SIZE) as u64)); + let heap = TestHeap::new(HEAP_SIZE); let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); - - // Pre-allocate multiple buddy blocks that will merge - let mut addrs = Vec::new(); - for _ in 0..64 { - let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - addrs.push(addr); - } - - b.iter(|| { - // Deallocate and allocate to test merging efficiency - let addr = addrs.pop().unwrap(); - allocator.dealloc_pages(black_box(addr), 1); - // Allocate again to reuse merged blocks - let new_addr = allocator.alloc_pages(1, PAGE_SIZE); - addrs.push(new_addr.unwrap()); - }); - - // Cleanup remaining allocations - for addr in addrs { - allocator.dealloc_pages(addr, 1); - } - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark allocation with different alignments -fn bench_alloc_with_alignment(c: &mut Criterion) { - let alignments = [PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4, PAGE_SIZE * 8]; - - let mut group = c.benchmark_group("buddy_alloc_with_alignment"); - - for alignment in alignments.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(alignment), alignment, |b, _| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); - - b.iter(|| { - let addr = allocator.alloc_pages(black_box(4), black_box(*alignment)); - black_box(addr) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); + allocator.init(heap.addr(), HEAP_SIZE); + + group.bench_with_input( + BenchmarkId::from_parameter(alignment), + &alignment, + |b, &align| { + b.iter(|| { + let addr = allocator + .alloc_pages(black_box(4), black_box(align)) + .unwrap(); + allocator.dealloc_pages(addr, 4); + black_box(addr); + }); + }, + ); } group.finish(); } -/// Benchmark fragmentation resistance -fn bench_fragmentation(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); +fn bench_fragmentation_recovery(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/fragmentation_recovery"); + group.throughput(Throughput::Bytes((64 * PAGE_SIZE) as u64)); - c.bench_function("buddy_fragmentation", |b| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); + group.bench_function("alloc_large_after_fragmentation", |b| { + b.iter_batched_ref( + || { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = BuddyPageAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE); - b.iter(|| { - // Allocate many small blocks, then free half - let mut addrs = Vec::new(); - for _ in 0..512 { - let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - addrs.push(addr); - } - - // Free every other allocation - for i in (0..512).step_by(2) { - allocator.dealloc_pages(addrs[i], 1); - } - - // Try to allocate a large block - should succeed if fragmentation is low - let large_layout = Layout::from_size_align(PAGE_SIZE * 64, PAGE_SIZE).unwrap(); - let large_addr = allocator.alloc_pages(64, PAGE_SIZE); - - // Cleanup - if let Ok(addr) = large_addr { - allocator.dealloc_pages(addr, 64); - } - for i in (1..512).step_by(2) { - allocator.dealloc_pages(addrs[i], 1); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} + let mut addrs = Vec::with_capacity(512); + for _ in 0..512 { + addrs.push(allocator.alloc_pages(1, PAGE_SIZE).unwrap()); + } -/// Benchmark random allocation pattern -fn bench_random_pattern(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); + (heap, allocator, addrs) + }, + |(_, allocator, addrs)| { + for i in (0..addrs.len()).step_by(2) { + allocator.dealloc_pages(addrs[i], 1); + } - c.bench_function("buddy_random_pattern", |b| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); + let large = allocator.alloc_pages(64, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(large, 64); - b.iter(|| { - let mut allocated = Vec::new(); - - // Random alloc/dealloc pattern - for _ in 0..1000 { - if allocated.is_empty() || rng.random_bool(0.7) { - // Allocate - let pages = 1 << rng.random_range(0..5); // 1, 2, 4, 8, 16 pages - match allocator.alloc_pages(pages, PAGE_SIZE) { - Ok(addr) => allocated.push((addr, pages)), - Err(_) => break, // Out of memory - } - } else { - // Deallocate random item - let idx = rng.random_range(0..allocated.len()); - let (addr, pages) = allocated.swap_remove(idx); - allocator.dealloc_pages(addr, pages); + for i in (1..addrs.len()).step_by(2) { + allocator.dealloc_pages(addrs[i], 1); } - } - // Cleanup - for (addr, pages) in allocated { - allocator.dealloc_pages(addr, pages); - } - }); + black_box(large); + }, + BatchSize::SmallInput, + ); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark CompositePageAllocator with contiguous block combination -fn bench_composite_allocator(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("composite_allocator", |b| { - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); +fn bench_random_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/random_workload"); + group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); + + group.bench_function("mixed_alloc_free_batch", |b| { + b.iter_batched_ref( + || { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = BuddyPageAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE); + (heap, allocator) + }, + |(_, allocator)| { + let mut rng = seeded_rng(); + let mut allocated = Vec::new(); + + for _ in 0..OPERATIONS_PER_BATCH { + if allocated.is_empty() || rng.random_bool(0.65) { + let pages = 1usize << rng.random_range(0..=4); + if let Ok(addr) = allocator.alloc_pages(pages, PAGE_SIZE) { + allocated.push((addr, pages)); + } + } else { + let idx = rng.random_range(0..allocated.len()); + let (addr, pages) = allocated.swap_remove(idx); + allocator.dealloc_pages(addr, pages); + } + } - b.iter(|| { - // Allocate and deallocate pages - for _ in 0..100 { - let addr = allocator.alloc_pages(1, PAGE_SIZE); - if let Ok(a) = addr { - allocator.dealloc_pages(a, 1); + for (addr, pages) in allocated { + allocator.dealloc_pages(addr, pages); } - } - }); + }, + BatchSize::SmallInput, + ); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark allocate at specific address -fn bench_alloc_at(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("buddy_alloc_at", |b| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); +fn bench_composite_non_power_of_two(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/composite_non_power_of_two"); - // Pre-allocate to create space for specific allocation - let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr1, 1); - - b.iter(|| { - // Try to allocate at a specific address - let target = addr1; - let addr = allocator.alloc_pages_at(black_box(target), 1, PAGE_SIZE); - black_box(addr) - }); - }); + for &num_pages in &[3usize, 5, 9] { + group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = CompositePageAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE); + + group.bench_with_input( + BenchmarkId::from_parameter(num_pages), + &num_pages, + |b, &pages| { + b.iter(|| { + let addr = allocator.alloc_pages(pages, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, pages); + black_box(addr); + }); + }, + ); + } - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark statistics retrieval -fn bench_statistics(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); +fn bench_statistics_query(c: &mut Criterion) { + let mut group = c.benchmark_group("buddy/statistics_query"); + group.throughput(Throughput::Elements(3)); - c.bench_function("buddy_statistics", |b| { - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE); + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = BuddyPageAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE); - // Make some allocations - for _ in 0..10 { - let _ = allocator.alloc_pages(1, PAGE_SIZE); - } + for _ in 0..64 { + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 1); + } + group.bench_function("page_counters", |b| { b.iter(|| { - let total = allocator.total_pages(); - let used = allocator.used_pages(); - let available = allocator.available_pages(); - black_box((total, used, available)) + let snapshot = ( + allocator.total_pages(), + allocator.used_pages(), + allocator.available_pages(), + ); + black_box(snapshot); }); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -criterion_group!( - benches, - bench_single_page_alloc, - bench_multi_page_alloc, - bench_dealloc_with_merge, - bench_alloc_with_alignment, - bench_fragmentation, - bench_random_pattern, - bench_composite_allocator, - bench_alloc_at, - bench_statistics -); +criterion_group! { + name = benches; + config = criterion_config(); + targets = + bench_page_alloc_free, + bench_alignment_alloc_free, + bench_fragmentation_recovery, + bench_random_workload, + bench_composite_non_power_of_two, + bench_statistics_query +} criterion_main!(benches); diff --git a/benches/common.rs b/benches/common.rs new file mode 100644 index 0000000..671fa2a --- /dev/null +++ b/benches/common.rs @@ -0,0 +1,46 @@ +use criterion::Criterion; +use rand::{rngs::StdRng, SeedableRng}; +use std::alloc::{alloc, dealloc, Layout}; +use std::time::Duration; + +pub const PAGE_SIZE: usize = 0x1000; +pub const HEAP_SIZE: usize = 64 * 1024 * 1024; +pub const OPERATIONS_PER_BATCH: u64 = 256; +const HEAP_ALIGN: usize = 64 * 1024; + +pub struct TestHeap { + ptr: *mut u8, + layout: Layout, +} + +impl TestHeap { + pub fn new(size: usize) -> Self { + let layout = Layout::from_size_align(size, HEAP_ALIGN).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate test heap"); + Self { ptr, layout } + } + + pub fn addr(&self) -> usize { + self.ptr as usize + } +} + +impl Drop for TestHeap { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +pub fn criterion_config() -> Criterion { + Criterion::default() + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(5)) + .sample_size(50) + .noise_threshold(0.05) +} + +#[allow(dead_code)] +pub fn seeded_rng() -> StdRng { + StdRng::from_seed([0; 32]) +} diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index 792abd4..5310787 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -1,247 +1,129 @@ -//! Benchmarks for GlobalAllocator - the unified allocator interface -//! -//! This benchmark suite tests the performance and stability of the GlobalAllocator -//! which automatically routes small allocations to Slab and large allocations to Buddy. +//! Benchmarks for the unified global allocator API. + +mod common; use buddy_slab_allocator::GlobalAllocator; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{RngExt, SeedableRng}; +use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use std::alloc::Layout; -use std::alloc::{alloc, dealloc}; - -const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB +use std::hint::black_box; -/// Allocate a test heap from the system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); - (ptr, layout) +struct GlobalHarness { + _heap: TestHeap, + allocator: Box>, } -/// Deallocate the test heap -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; +impl GlobalHarness { + fn new() -> Self { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = Box::new(GlobalAllocator::::new()); + allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + Self { + _heap: heap, + allocator, + } + } } -/// Benchmark small allocations (≤2048 bytes) - uses Slab allocator -fn bench_small_alloc(c: &mut Criterion) { - let sizes: [usize; 6] = [8, 16, 64, 256, 512, 1024]; +fn bench_small_alloc_free(c: &mut Criterion) { + let mut group = c.benchmark_group("global/small_alloc_free"); - let mut group = c.benchmark_group("global_alloc_small"); - - for size in sizes.iter() { - let layout = Layout::from_size_align(*size, 8).unwrap(); - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); + for &size in &[8usize, 64, 512, 2048] { + group.throughput(Throughput::Bytes(size as u64)); + let mut harness = GlobalHarness::new(); + let layout = Layout::from_size_align(size, 8).unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = allocator.alloc(black_box(layout)); - black_box(ptr) + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + harness.allocator.dealloc(ptr, layout); + black_box(ptr); }); }); - - dealloc_test_heap(heap_ptr, heap_layout); } group.finish(); } -/// Benchmark large allocations (>2048 bytes) - uses Buddy allocator -fn bench_large_alloc(c: &mut Criterion) { - let sizes: [usize; 5] = [ - PAGE_SIZE, - PAGE_SIZE * 2, - PAGE_SIZE * 4, - PAGE_SIZE * 8, - PAGE_SIZE * 16, - ]; +fn bench_large_alloc_free(c: &mut Criterion) { + let mut group = c.benchmark_group("global/large_alloc_free"); - let mut group = c.benchmark_group("global_alloc_large"); - - for size in sizes.iter() { - let layout = Layout::from_size_align(*size, PAGE_SIZE).unwrap(); - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); + for &size in &[PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16] { + group.throughput(Throughput::Bytes(size as u64)); + let mut harness = GlobalHarness::new(); + let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = allocator.alloc(black_box(layout)); - black_box(ptr) + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + harness.allocator.dealloc(ptr, layout); + black_box(ptr); }); }); - - dealloc_test_heap(heap_ptr, heap_layout); } group.finish(); } -/// Benchmark deallocation for small allocations -fn bench_small_dealloc(c: &mut Criterion) { - let layout = Layout::from_size_align(64, 8).unwrap(); - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("global_dealloc_small", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - // Pre-allocate a batch of pointers - let mut pointers = Vec::new(); - for _ in 0..1000 { - let ptr = allocator.alloc(layout).unwrap(); - pointers.push(ptr); - } - - b.iter(|| { - // Release and acquire to keep benchmark going - let ptr = pointers.pop().unwrap(); - allocator.dealloc(black_box(ptr), layout); - // Allocate again to maintain pool - let new_ptr = allocator.alloc(layout).unwrap(); - pointers.push(new_ptr); - }); - - // Cleanup - for ptr in pointers { - allocator.dealloc(ptr, layout); - } - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark allocation/deallocation cycle -fn bench_alloc_dealloc_cycle(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); +fn bench_mixed_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("global/mixed_workload"); + group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - c.bench_function("global_alloc_dealloc_cycle", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - // Simulate realistic usage: allocate and immediately dealloc - for _ in 0..100 { - let layout = Layout::from_size_align(64, 8).unwrap(); - let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark random size allocations -fn bench_random_allocations(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("global_random_allocations", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); + let mut harness = GlobalHarness::new(); + let layouts = [ + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(256, 8).unwrap(), + Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(), + Layout::from_size_align(PAGE_SIZE * 4, PAGE_SIZE).unwrap(), + ]; + group.bench_function("alternating_small_and_large", |b| { b.iter(|| { - // 100 random allocations - for _ in 0..100 { - let size: usize = rng.random_range(8..2048); - let size = size.next_power_of_two(); - let layout = Layout::from_size_align(size, 8).unwrap(); - let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); + for idx in 0..OPERATIONS_PER_BATCH as usize { + let layout = layouts[idx % layouts.len()]; + let ptr = harness.allocator.alloc(layout).unwrap(); + harness.allocator.dealloc(ptr, layout); + black_box(ptr); } }); }); - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark mixed allocation pattern - simulates real-world usage -fn bench_mixed_pattern(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("global_mixed_pattern", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - // Pre-allocate a pool of objects - let mut small_ptrs = Vec::new(); - let mut large_ptrs = Vec::new(); - - for _ in 0..100 { - let small_layout = Layout::from_size_align(64, 8).unwrap(); - small_ptrs.push(allocator.alloc(small_layout).unwrap()); - } - - for _ in 0..10 { - let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); - large_ptrs.push(allocator.alloc(large_layout).unwrap()); - } - - b.iter(|| { - // Alternate between small and large allocations - let small_layout = Layout::from_size_align(64, 8).unwrap(); - let ptr = allocator.alloc(small_layout).unwrap(); - allocator.dealloc(ptr, small_layout); - - let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); - let ptr2 = allocator.alloc(large_layout).unwrap(); - allocator.dealloc(ptr2, large_layout); - }); - - // Cleanup - let small_layout = Layout::from_size_align(64, 8).unwrap(); - for ptr in small_ptrs { - allocator.dealloc(ptr, small_layout); - } - let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); - for ptr in large_ptrs { - allocator.dealloc(ptr, large_layout); - } - }); - - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark page allocation through GlobalAllocator -fn bench_page_allocation(c: &mut Criterion) { - let page_counts = [1, 2, 4, 8, 16]; - - let mut group = c.benchmark_group("global_alloc_pages"); - - for num_pages in page_counts.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(num_pages), num_pages, |b, _| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - let addr = allocator.alloc_pages(black_box(*num_pages), PAGE_SIZE); - black_box(addr) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); +fn bench_page_interface(c: &mut Criterion) { + let mut group = c.benchmark_group("global/page_interface"); + + for &num_pages in &[1usize, 4, 16, 64] { + group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); + let mut harness = GlobalHarness::new(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_pages), + &num_pages, + |b, &pages| { + b.iter(|| { + let addr = harness + .allocator + .alloc_pages(black_box(pages), PAGE_SIZE) + .unwrap(); + harness.allocator.dealloc_pages(addr, pages); + black_box(addr); + }); + }, + ); } group.finish(); } -criterion_group!( - benches, - bench_small_alloc, - bench_large_alloc, - bench_small_dealloc, - bench_alloc_dealloc_cycle, - bench_random_allocations, - bench_mixed_pattern, - bench_page_allocation -); +criterion_group! { + name = benches; + config = criterion_config(); + targets = + bench_small_alloc_free, + bench_large_alloc_free, + bench_mixed_workload, + bench_page_interface +} criterion_main!(benches); diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index 6a12935..fe19925 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -1,351 +1,166 @@ -//! Benchmarks for SlabByteAllocator - small object allocation -//! -//! This benchmark suite tests the performance and stability of the Slab allocator -//! which is optimized for small object allocations (≤2048 bytes). +//! Benchmarks for slab-based small-object allocation paths. + +mod common; use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator}; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{RngExt, SeedableRng}; +use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use std::alloc::Layout; -use std::alloc::{alloc, dealloc}; - -const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB - -/// Allocate a test heap from the system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); - (ptr, layout) -} - -/// Deallocate the test heap -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; -} - -/// Create initialized slab allocator with page allocator -fn create_slab_allocator( - heap_ptr: *mut u8, -) -> ( - CompositePageAllocator, - SlabByteAllocator, -) { - let mut page_alloc = CompositePageAllocator::::new(); - page_alloc.init(heap_ptr as usize, HEAP_SIZE); - - let mut slab_alloc = SlabByteAllocator::::new(); - slab_alloc.init(); - - (page_alloc, slab_alloc) -} - -/// Benchmark allocations for different size classes -fn bench_size_classes(c: &mut Criterion) { - let size_classes: [usize; 6] = [8, 64, 512, 1024, 1536, 2048]; - - let mut group = c.benchmark_group("slab_size_classes"); - - for size in size_classes.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| { - let layout = Layout::from_size_align(*size, 8).unwrap(); - - b.iter(|| { - let ptr = slab_alloc.alloc(black_box(layout)).unwrap(); - slab_alloc.dealloc(ptr, layout); - black_box(ptr) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); - } +use std::hint::black_box; - group.finish(); +struct SlabHarness { + _heap: TestHeap, + _page_allocator: Box>, + slab_allocator: SlabByteAllocator, } -/// Benchmark deallocation -fn bench_dealloc(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - c.bench_function("slab_dealloc", |b| { - let layout = Layout::from_size_align(64, 8).unwrap(); - - // Pre-allocate pointers - let mut pointers = Vec::new(); - for _ in 0..1000 { - let ptr = slab_alloc.alloc(layout).unwrap(); - pointers.push(ptr); - } - - b.iter(|| { - let ptr = pointers.pop().unwrap(); - slab_alloc.dealloc(black_box(ptr), layout); - // Allocate again to maintain pool - let new_ptr = slab_alloc.alloc(layout).unwrap(); - pointers.push(new_ptr); - }); - - // Cleanup - for ptr in pointers { - slab_alloc.dealloc(ptr, layout); +impl SlabHarness { + fn new() -> Self { + let heap = TestHeap::new(HEAP_SIZE); + let mut page_allocator = Box::new(CompositePageAllocator::::new()); + page_allocator.init(heap.addr(), HEAP_SIZE); + + let mut slab_allocator = SlabByteAllocator::::new(); + slab_allocator.init(); + let page_allocator_ptr = (&mut *page_allocator) as *mut dyn PageAllocatorForSlab; + slab_allocator.set_page_allocator(page_allocator_ptr); + + Self { + _heap: heap, + _page_allocator: page_allocator, + slab_allocator, } - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark allocation/deallocation cycle -fn bench_alloc_dealloc_cycle(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - c.bench_function("slab_alloc_dealloc_cycle", |b| { - b.iter(|| { - for _ in 0..100 { - let layout = Layout::from_size_align(64, 8).unwrap(); - let ptr = slab_alloc.alloc(layout).unwrap(); - slab_alloc.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); + } } -/// Benchmark allocations with different alignments -fn bench_alignment(c: &mut Criterion) { - let alignments = [8, 1024, 2048]; - - let mut group = c.benchmark_group("slab_alignment"); +fn bench_size_class_alloc_free(c: &mut Criterion) { + let mut group = c.benchmark_group("slab/size_class_alloc_free"); - for alignment in alignments.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - group.bench_with_input(BenchmarkId::from_parameter(alignment), alignment, |b, _| { - let layout = Layout::from_size_align(64, *alignment).unwrap(); + for &size in &[8usize, 64, 256, 512, 1024, 2048] { + group.throughput(Throughput::Bytes(size as u64)); + let mut harness = SlabHarness::new(); + let layout = Layout::from_size_align(size, 8).unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = slab_alloc.alloc(black_box(layout)).unwrap(); - slab_alloc.dealloc(ptr, layout); - black_box(ptr) + let ptr = harness.slab_allocator.alloc(black_box(layout)).unwrap(); + harness.slab_allocator.dealloc(ptr, layout); + black_box(ptr); }); }); - - dealloc_test_heap(heap_ptr, heap_layout); } group.finish(); } -/// Benchmark random size allocations -fn bench_random_allocations(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - c.bench_function("slab_random_allocations", |b| { - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - b.iter(|| { - for _ in 0..100 { - let size: usize = rng.random_range(8..2049); - let size = if size.is_power_of_two() { - size - } else { - size.next_power_of_two() - }; - let layout = Layout::from_size_align(size, 8).unwrap(); - let ptr = slab_alloc.alloc(layout).unwrap(); - slab_alloc.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Benchmark object pooling - allocate, use, dealloc repeatedly -fn bench_object_pooling(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); +fn bench_hot_reuse(c: &mut Criterion) { + let mut group = c.benchmark_group("slab/hot_reuse"); + group.throughput(Throughput::Elements(1)); - c.bench_function("slab_object_pooling", |b| { - let layout = Layout::from_size_align(128, 8).unwrap(); + let mut harness = SlabHarness::new(); + let layout = Layout::from_size_align(128, 8).unwrap(); - // Simulate object pool with fixed size - let mut pool = Vec::new(); - for _ in 0..100 { - let ptr = slab_alloc.alloc(layout).unwrap(); - pool.push(ptr); - } + let ptr = harness.slab_allocator.alloc(layout).unwrap(); + harness.slab_allocator.dealloc(ptr, layout); + group.bench_function("alloc_free_hot_cache", |b| { b.iter(|| { - // Get an object from pool, use it, put it back - let ptr = pool.pop().unwrap(); - // Simulate usage + let ptr = harness.slab_allocator.alloc(layout).unwrap(); + harness.slab_allocator.dealloc(ptr, layout); black_box(ptr); - pool.push(ptr); }); - - // Cleanup - for ptr in pool { - slab_alloc.dealloc(ptr, layout); - } }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark mixed size allocation pattern -fn bench_mixed_sizes(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - c.bench_function("slab_mixed_sizes", |b| { - let sizes = [8, 64, 256, 512, 1024, 2048]; - let layouts: Vec = sizes - .iter() - .map(|&s| Layout::from_size_align(s, 8).unwrap()) - .collect(); - +fn bench_mixed_size_batch(c: &mut Criterion) { + let mut group = c.benchmark_group("slab/mixed_size_batch"); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(256, 8).unwrap(), + Layout::from_size_align(512, 8).unwrap(), + Layout::from_size_align(1024, 8).unwrap(), + Layout::from_size_align(2048, 8).unwrap(), + ]; + let total_bytes: usize = layouts.iter().map(Layout::size).sum(); + group.throughput(Throughput::Bytes(total_bytes as u64)); + + let mut harness = SlabHarness::new(); + + group.bench_function("alloc_free_layout_batch", |b| { b.iter(|| { - for layout in &layouts { - let ptr = slab_alloc.alloc(*layout).unwrap(); - slab_alloc.dealloc(ptr, *layout); + for layout in layouts { + let ptr = harness.slab_allocator.alloc(layout).unwrap(); + harness.slab_allocator.dealloc(ptr, layout); + black_box(ptr); } }); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark rapid allocation and deallocation stress test -fn bench_stress_test(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); +fn bench_steady_state(c: &mut Criterion) { + let mut group = c.benchmark_group("slab/steady_state"); + group.throughput(Throughput::Elements(1)); - c.bench_function("slab_stress_test", |b| { - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - b.iter(|| { - let mut allocated = Vec::new(); + let mut harness = SlabHarness::new(); + let layout = Layout::from_size_align(64, 8).unwrap(); + let mut active = Vec::with_capacity(256); - // Stress test: rapid alloc/dealloc with varying sizes - for _ in 0..1000 { - if allocated.is_empty() || rng.random_bool(0.7) { - // Allocate - let size = rng.random_range(8..2049); - let layout = Layout::from_size_align(size, 8).unwrap(); - if let Ok(ptr) = slab_alloc.alloc(layout) { - allocated.push((ptr, layout)); - } - } else { - // Deallocate random - let idx = rng.random_range(0..allocated.len()); - let (ptr, layout) = allocated.swap_remove(idx); - slab_alloc.dealloc(ptr, layout); - } - } + for _ in 0..256 { + active.push(harness.slab_allocator.alloc(layout).unwrap()); + } - // Cleanup - allocated.clear(); - let ptrs_to_free: Vec<_> = std::mem::take(&mut allocated); - for (ptr, layout) in ptrs_to_free { - slab_alloc.dealloc(ptr, layout); - } + group.bench_function("recycle_live_object", |b| { + b.iter(|| { + let ptr = active.pop().unwrap(); + harness.slab_allocator.dealloc(ptr, layout); + let new_ptr = harness.slab_allocator.alloc(layout).unwrap(); + active.push(new_ptr); + black_box(new_ptr); }); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark performance under memory pressure -fn bench_memory_pressure(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); +fn bench_batch_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("slab/batch_workload"); + group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - c.bench_function("slab_memory_pressure", |b| { - let layout = Layout::from_size_align(64, 8).unwrap(); + let mut harness = SlabHarness::new(); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(256, 8).unwrap(), + Layout::from_size_align(1024, 8).unwrap(), + ]; + group.bench_function("round_robin_layouts", |b| { b.iter(|| { - // Allocate until near capacity - let mut allocated = Vec::new(); - loop { - match slab_alloc.alloc(layout) { - Ok(ptr) => allocated.push(ptr), - Err(_) => break, - } - } - - // Free half and reallocate - for i in (0..allocated.len()).step_by(2) { - slab_alloc.dealloc(allocated[i], layout); - } - - // Try to allocate more - for _ in 0..10 { - let _ = slab_alloc.alloc(layout); - } - - // Cleanup - for ptr in allocated { - slab_alloc.dealloc(ptr, layout); + for idx in 0..OPERATIONS_PER_BATCH as usize { + let layout = layouts[idx % layouts.len()]; + let ptr = harness.slab_allocator.alloc(layout).unwrap(); + harness.slab_allocator.dealloc(ptr, layout); + black_box(ptr); } }); }); - dealloc_test_heap(heap_ptr, heap_layout); + group.finish(); } -/// Benchmark statistics retrieval -fn bench_statistics(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr); - slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - - // Make some allocations - for _ in 0..10 { - let layout = Layout::from_size_align(64, 8).unwrap(); - let _ = slab_alloc.alloc(layout); - } - - c.bench_function("slab_statistics", |b| { - b.iter(|| { - let total = slab_alloc.total_bytes(); - let used = slab_alloc.used_bytes(); - let available = slab_alloc.available_bytes(); - black_box((total, used, available)) - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); +criterion_group! { + name = benches; + config = criterion_config(); + targets = + bench_size_class_alloc_free, + bench_hot_reuse, + bench_mixed_size_batch, + bench_steady_state, + bench_batch_workload } - -criterion_group!( - benches, - bench_size_classes, - bench_dealloc, - bench_alloc_dealloc_cycle, - bench_alignment, - bench_random_allocations, - bench_object_pooling, - bench_mixed_sizes, - bench_stress_test, - bench_memory_pressure, - bench_statistics -); criterion_main!(benches); diff --git a/benches/stability.rs b/benches/stability.rs deleted file mode 100644 index 6de7cf0..0000000 --- a/benches/stability.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! Stability and stress testing benchmarks -//! -//! This benchmark suite focuses on stability testing under various stress conditions: -//! - Memory exhaustion handling -//! - Long-running operation stability -//! - Random allocation/deallocation patterns -//! - Fragmentation resistance -//! - Memory leak detection - -use buddy_slab_allocator::GlobalAllocator; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rand::{RngExt, SeedableRng}; -use std::alloc::Layout; -use std::alloc::{alloc, dealloc}; - -const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB - -/// Allocate a test heap from the system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); - (ptr, layout) -} - -/// Deallocate the test heap -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; -} - -/// Stability test: Random allocation/deallocation pattern over many iterations -fn bench_random_pattern_stability(c: &mut Criterion) { - let iterations = [1000, 5000, 10000]; - - let mut group = c.benchmark_group("stability_random_pattern"); - - for &iter in iterations.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(iter), &iter, |b, _| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - b.iter(|| { - let mut allocated = Vec::new(); - - for _ in 0..iter { - if allocated.is_empty() || rng.random_bool(0.6) { - // 60% allocate - let size = rng.random_range(8..8193); - let size = if size <= 2048 { - size - } else { - // Align to page size for large allocations - ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE - }; - let layout = Layout::from_size_align(size, 8).unwrap(); - if let Ok(ptr) = allocator.alloc(layout) { - allocated.push((ptr, layout)); - } - } else { - // 40% deallocate - let idx = rng.random_range(0..allocated.len()); - let (ptr, layout) = allocated.swap_remove(idx); - allocator.dealloc(ptr, layout); - } - } - - // Cleanup - verify no leaks by checking stats - #[cfg(feature = "tracking")] - { - let before = allocator.get_stats(); - } - - for (ptr, layout) in allocated { - allocator.dealloc(ptr, layout); - } - - #[cfg(feature = "tracking")] - { - let after = allocator.get_stats(); - // Verify we're back to initial state - assert_eq!(after.used_pages, 0, "Memory leak detected!"); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); - } - - group.finish(); -} - -/// Stability test: Allocate to exhaustion and handle gracefully -fn bench_exhaustion_handling(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stability_exhaustion", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - let mut allocated = Vec::new(); - let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); - - // Allocate until exhaustion - loop { - match allocator.alloc(layout) { - Ok(ptr) => allocated.push(ptr), - Err(_) => break, // Expected - out of memory - } - } - - // Verify we can still deallocate - for ptr in allocated.iter().take(10) { - allocator.dealloc(*ptr, layout); - } - - // Verify we can allocate again after freeing - let new_ptr = allocator.alloc(layout); - assert!(new_ptr.is_ok(), "Failed to allocate after freeing memory"); - - // Cleanup - if let Ok(ptr) = new_ptr { - allocator.dealloc(ptr, layout); - } - for ptr in allocated { - allocator.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Stability test: Fragmentation resistance -fn bench_fragmentation_resistance(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stability_fragmentation", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - let mut small_ptrs = Vec::new(); - let small_layout = Layout::from_size_align(64, 8).unwrap(); - - // Allocate many small objects - for _ in 0..1000 { - if let Ok(ptr) = allocator.alloc(small_layout) { - small_ptrs.push(ptr); - } - } - - // Free every other one to create fragmentation - for i in (0..small_ptrs.len()).step_by(2) { - allocator.dealloc(small_ptrs[i], small_layout); - } - - // Try to allocate a large contiguous block - let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap(); - let large_ptr = allocator.alloc(large_layout); - - // Cleanup - for ptr in small_ptrs { - allocator.dealloc(ptr, small_layout); - } - if let Ok(ptr) = large_ptr { - allocator.dealloc(ptr, large_layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Stability test: Alternating allocation/deallocation pattern -fn bench_alternating_pattern(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stability_alternating", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - let mut ptrs = Vec::new(); - - // Alternating allocate/dealloc pattern - for i in 0..500 { - if i % 2 == 0 { - let layout = Layout::from_size_align(64, 8).unwrap(); - if let Ok(ptr) = allocator.alloc(layout) { - ptrs.push((ptr, layout)); - } - } else if !ptrs.is_empty() { - let (ptr, layout) = ptrs.remove(0); - allocator.dealloc(ptr, layout); - } - } - - // Cleanup - for (ptr, layout) in ptrs { - allocator.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Stability test: Long-running operation with periodic checks -fn bench_long_running(c: &mut Criterion) { - let durations = [1000, 5000, 10000]; - - let mut group = c.benchmark_group("stability_long_running"); - - for &duration in durations.iter() { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - group.bench_with_input(BenchmarkId::from_parameter(duration), &duration, |b, _| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - b.iter(|| { - for _ in 0..duration { - let size = rng.random_range(8..2049); - let layout = Layout::from_size_align(size, 8).unwrap(); - let ptr = allocator.alloc(layout); - - if let Ok(p) = ptr { - allocator.dealloc(p, layout); - } - } - - #[cfg(feature = "tracking")] - { - let stats = allocator.get_stats(); - // Verify stats are consistent - assert!(stats.used_pages <= stats.total_pages); - assert!(stats.free_pages <= stats.total_pages); - assert_eq!(stats.used_pages + stats.free_pages, stats.total_pages); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); - } - - group.finish(); -} - -/// Stress test: Rapid allocation/deallocation of mixed sizes -fn bench_rapid_mixed_sizes(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stress_rapid_mixed", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - b.iter(|| { - let mut allocated = Vec::new(); - - // Rapid mixed-size allocations - for _ in 0..5000 { - let size = rng.random_range(8..16385); - let aligned_size = if size <= 2048 { - size - } else { - ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE - }; - let layout = Layout::from_size_align(aligned_size, 8).unwrap(); - - if let Ok(ptr) = allocator.alloc(layout) { - if rng.random_bool(0.5) { - // Immediately free 50% of allocations - allocator.dealloc(ptr, layout); - } else { - allocated.push((ptr, layout)); - } - } - } - - // Cleanup - for (ptr, layout) in allocated { - allocator.dealloc(ptr, layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Stress test: Page allocation pressure -fn bench_page_allocation_pressure(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stress_page_pressure", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - let mut page_addrs = Vec::new(); - - // Allocate pages until near exhaustion - loop { - match allocator.alloc_pages(1, PAGE_SIZE) { - Ok(addr) => page_addrs.push(addr), - Err(_) => break, - } - } - - // Free and reallocate in different patterns - for i in (0..page_addrs.len()).step_by(2) { - allocator.dealloc_pages(page_addrs[i], 1); - } - - // Try to allocate more - let mut new_addrs = Vec::new(); - for _ in 0..5 { - if let Ok(addr) = allocator.alloc_pages(1, PAGE_SIZE) { - new_addrs.push(addr); - } - } - - // Cleanup - for addr in page_addrs { - allocator.dealloc_pages(addr, 1); - } - for addr in new_addrs { - allocator.dealloc_pages(addr, 1); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Memory leak detection test -fn bench_memory_leak_detection(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stability_leak_detection", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - #[cfg(feature = "tracking")] - { - let initial_stats = allocator.get_stats(); - } - - b.iter(|| { - let mut ptrs = Vec::new(); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - - // Perform random allocations - for _ in 0..1000 { - let size = rng.random_range(8..2049); - let layout = Layout::from_size_align(size, 8).unwrap(); - if let Ok(ptr) = allocator.alloc(layout) { - ptrs.push((ptr, layout)); - } - } - - // Free all allocations - for (ptr, layout) in &ptrs { - allocator.dealloc(*ptr, *layout); - } - - #[cfg(feature = "tracking")] - { - let stats = allocator.get_stats(); - // Verify no memory leaks - assert_eq!(stats.used_pages, 0, "Memory leak detected!"); - } - - ptrs.clear(); - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -/// Stability test: Edge cases -fn bench_edge_cases(c: &mut Criterion) { - let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE); - - c.bench_function("stability_edge_cases", |b| { - let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap(); - - b.iter(|| { - // Test minimum size allocation - let min_layout = Layout::from_size_align(1, 1).unwrap(); - let min_ptr = allocator.alloc(min_layout); - if let Ok(ptr) = min_ptr { - allocator.dealloc(ptr, min_layout); - } - - // Test maximum small size allocation - let max_small_layout = Layout::from_size_align(2048, 2048).unwrap(); - let max_small_ptr = allocator.alloc(max_small_layout); - if let Ok(ptr) = max_small_ptr { - allocator.dealloc(ptr, max_small_layout); - } - - // Test page-aligned allocation - let page_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); - let page_ptr = allocator.alloc(page_layout); - if let Ok(ptr) = page_ptr { - allocator.dealloc(ptr, page_layout); - } - - // Test odd alignment - alignment must be power of two - let odd_align = 8; // Changed from 7 to 8 (power of 2) - let odd_layout = Layout::from_size_align(64, odd_align).unwrap(); - let odd_ptr = allocator.alloc(odd_layout); - if let Ok(ptr) = odd_ptr { - allocator.dealloc(ptr, odd_layout); - } - }); - }); - - dealloc_test_heap(heap_ptr, heap_layout); -} - -criterion_group!( - benches, - bench_random_pattern_stability, - bench_exhaustion_handling, - bench_fragmentation_resistance, - bench_alternating_pattern, - bench_long_running, - bench_rapid_mixed_sizes, - bench_page_allocation_pressure, - bench_memory_leak_detection, - bench_edge_cases -); -criterion_main!(benches); diff --git a/tests/README.md b/tests/README.md index d562924..dae056b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,165 +1,46 @@ # Allocator Test Suite -本目录包含allocator模块的集成测试。单元测试位于各模块源文件中，文档测试位于公共API函数注释中。 +本目录包含 allocator 的集成测试与压力测试。 -## 测试组织结构 +## 结构 -### 单元测试 (src/**/*.rs) -位于各模块源文件的`#[cfg(test)]`模块中，测试单个模块的功能： -- `src/buddy/buddy_allocator.rs` - Buddy分配器单元测试 -- `src/buddy/global_node_pool.rs` - 全局节点池单元测试 -- `src/buddy/pooled_list.rs` - 池化链表单元测试 -- `src/page_allocator.rs` - 组合页分配器单元测试 -- `src/slab/slab_byte_allocator.rs` - Slab字节分配器单元测试 -- `src/slab/slab_cache.rs` - Slab缓存单元测试 -- `src/slab/slab_node.rs` - Slab节点单元测试 +- `integration_test.rs` + 常规集成测试，覆盖全局分配器、Buddy、Slab 以及多模块协同行为。 +- `dma32_pages_test.rs` + 低地址页分配相关测试。 +- `stress_test.rs` + 长时间随机、耗尽恢复、碎片化恢复等压力测试。 + 这些测试默认使用 `#[ignore]`，不会进入常规 `cargo test` 路径。 -### 集成测试 (tests/) -位于tests目录，测试多个模块协同工作： -- `integration_test.rs` - 完整系统集成测试 +单元测试仍位于 `src/**/*.rs` 的 `#[cfg(test)]` 模块中，文档测试位于公共 API 注释中。 -### 文档测试 -位于公共API函数文档注释中的代码示例： -- `GlobalAllocator::init()` - 初始化示例 -- `GlobalAllocator::alloc()` - 分配示例 -- `GlobalAllocator::alloc_pages()` - 页分配示例 +## 常用命令 -## 运行测试 - -### 运行所有测试 ```bash -cd allocator +# 常规测试 cargo test -``` - -### 运行单元测试 -```bash -cargo test --lib -``` - -### 运行集成测试 -```bash -cargo test --test integration_test -``` - -### 运行文档测试 -```bash -cargo test --doc -``` - -### 运行特定模块的单元测试 -```bash -cargo test --lib buddy_allocator -cargo test --lib slab_byte_allocator -``` -### 启用统计跟踪功能 -```bash +# 带 tracking 的常规测试 cargo test --features tracking -``` - -### 显示测试输出 -```bash -cargo test -- --nocapture -``` - -### 并行运行测试 -```bash -cargo test -- --test-threads=4 -``` - -## 测试覆盖 - -### 功能覆盖 -- ✅ 页级分配（Buddy系统） -- ✅ 字节级分配（Slab系统） -- ✅ 全局分配器协调 -- ✅ 多区域支持 -- ✅ 对齐要求 -- ✅ 内存统计 -- ✅ 错误处理 -- ✅ 碎片化处理 -- ✅ 合并优化 - -### 场景覆盖 -- ✅ 基本分配/释放 -- ✅ 大量小对象 -- ✅ 大块内存 -- ✅ 混合大小 -- ✅ 交错操作 -- ✅ 边界条件 -- ✅ 压力测试 -- ✅ 性能基准 - -### 错误情况覆盖 -- ✅ 无效参数 -- ✅ 内存不足 -- ✅ 重复释放 -- ✅ 对齐错误 -- ✅ 超大分配 - -## 测试统计 -- 单元测试：26个 (位于src/**/*.rs) -- 集成测试：15个 (位于tests/integration_test.rs) -- 文档测试：3个 (位于函数文档注释) +# 串行执行，便于排查测试交互 +cargo test -- --test-threads=1 -总计：44+个测试 +# 仅运行压力测试 +cargo test --test stress_test -- --ignored --nocapture -## 持续集成 - -所有测试应在提交前通过： -```bash -cargo test --all-features -cargo test --no-default-features -cargo test --features tracking +# 带 tracking 运行压力测试 +cargo test --test stress_test --features tracking -- --ignored --nocapture ``` -## 性能指标 +## 设计原则 -基准测试提供以下性能指标： -- 页分配吞吐量 -- Slab分配延迟 -- 碎片化恢复能力 -- 并发分配性能 -- 重新分配开销 +- 常规测试应保持快速、稳定，可直接进入 CI。 +- 压力测试用于长时间 workload、耗尽恢复、碎片化恢复与统计不变量检查。 +- 性能测量移至 `benches/`，不在测试中混入 benchmark 逻辑。 ## 注意事项 -1. 所有测试使用系统分配器分配测试堆内存 -2. 测试后会自动清理分配的内存 -3. 某些测试可能需要较大内存（64MB） -4. 基准测试默认被忽略，需显式运行 -5. 统计跟踪测试需要启用`tracking`特性 - -## 添加新测试 - -### 添加单元测试 -在相应模块源文件的`#[cfg(test)]`模块中添加： -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_new_feature() { - // 测试代码 - } -} -``` - -### 添加集成测试 -在`tests/integration_test.rs`中添加新测试函数。 - -### 添加文档测试 -在公共API函数文档中添加示例： -```rust -/// 函数说明 -/// -/// # Examples -/// -/// ```no_run -/// // 示例代码 -/// ``` -pub fn my_function() {} -``` +1. 所有测试都使用宿主分配器申请一块测试堆。 +2. 压力测试默认被忽略，需要显式运行。 +3. `tracking` 相关断言仅在启用 `tracking` feature 时编译。 diff --git a/tests/integration_test.rs b/tests/integration_test.rs index c2f5675..b6d21ab 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -132,27 +132,6 @@ fn test_composite_page_allocator_fragmentation() { dealloc_test_heap(heap_ptr, heap_layout); } -#[test] -fn test_composite_page_allocator_alloc_at() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - // Note: alloc_pages_at may not always succeed due to buddy system structure - // We just test that it doesn't crash - let target_addr = heap_addr + PAGE_SIZE * 100; - let _result = allocator.alloc_pages_at(target_addr, 4, PAGE_SIZE); - - // If successful, clean up - // if result.is_ok() { - // allocator.dealloc_pages(target_addr, 4); - // } - - dealloc_test_heap(heap_ptr, heap_layout); -} - #[test] fn test_slab_allocator_basic() { let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); diff --git a/tests/stress_test.rs b/tests/stress_test.rs new file mode 100644 index 0000000..d668c0e --- /dev/null +++ b/tests/stress_test.rs @@ -0,0 +1,174 @@ +//! Ignored stress tests for allocator stability and long-running behavior. + +use buddy_slab_allocator::GlobalAllocator; +use rand::{RngExt, SeedableRng}; +use std::alloc::{alloc, dealloc, Layout}; + +const PAGE_SIZE: usize = 0x1000; +const HEAP_SIZE: usize = 64 * 1024 * 1024; + +struct TestHeap { + ptr: *mut u8, + layout: Layout, +} + +impl TestHeap { + fn new(size: usize) -> Self { + let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate test heap"); + Self { ptr, layout } + } + + fn addr(&self) -> usize { + self.ptr as usize + } +} + +impl Drop for TestHeap { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +#[test] +#[ignore = "stress test"] +fn stress_random_mixed_alloc_free() { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = GlobalAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let mut rng = rand::rngs::StdRng::from_seed([0; 32]); + let mut allocated = Vec::new(); + + for _ in 0..10_000 { + if allocated.is_empty() || rng.random_bool(0.65) { + let size: usize = rng.random_range(8..8193); + let layout = if size <= 2048 { + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let aligned = size.div_ceil(PAGE_SIZE) * PAGE_SIZE; + Layout::from_size_align(aligned, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + allocated.push((ptr, layout)); + } + } else { + let idx = rng.random_range(0..allocated.len()); + let (ptr, layout) = allocated.swap_remove(idx); + allocator.dealloc(ptr, layout); + } + } + + for (ptr, layout) in allocated { + allocator.dealloc(ptr, layout); + } +} + +#[test] +#[ignore = "stress test"] +fn stress_exhaustion_recovery() { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = GlobalAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); + let mut allocated = Vec::new(); + + while let Ok(ptr) = allocator.alloc(layout) { + allocated.push(ptr); + } + + assert!( + !allocated.is_empty(), + "expected to exhaust heap after allocations" + ); + + for ptr in allocated.drain(..allocated.len() / 4) { + allocator.dealloc(ptr, layout); + } + + let recovered = allocator.alloc(layout); + assert!( + recovered.is_ok(), + "allocator should recover after freeing pages" + ); + + if let Ok(ptr) = recovered { + allocator.dealloc(ptr, layout); + } + + for ptr in allocated { + allocator.dealloc(ptr, layout); + } +} + +#[test] +#[ignore = "stress test"] +fn stress_fragmentation_recovery() { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = GlobalAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let small_layout = Layout::from_size_align(64, 8).unwrap(); + let mut small_ptrs = Vec::new(); + + for _ in 0..4000 { + if let Ok(ptr) = allocator.alloc(small_layout) { + small_ptrs.push(ptr); + } + } + + for i in (0..small_ptrs.len()).step_by(2) { + allocator.dealloc(small_ptrs[i], small_layout); + } + + let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap(); + let large = allocator.alloc(large_layout); + + for ptr in small_ptrs.into_iter().skip(1).step_by(2) { + allocator.dealloc(ptr, small_layout); + } + + if let Ok(ptr) = large { + allocator.dealloc(ptr, large_layout); + } +} + +#[cfg(feature = "tracking")] +#[test] +#[ignore = "stress test"] +fn stress_tracking_invariants() { + let heap = TestHeap::new(HEAP_SIZE); + let mut allocator = GlobalAllocator::::new(); + allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let mut rng = rand::rngs::StdRng::from_seed([1; 32]); + let mut allocated = Vec::new(); + + for _ in 0..5000 { + if allocated.is_empty() || rng.random_bool(0.6) { + let size: usize = rng.random_range(8..4097); + let layout = if size <= 2048 { + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let aligned = size.div_ceil(PAGE_SIZE) * PAGE_SIZE; + Layout::from_size_align(aligned, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + allocated.push((ptr, layout)); + } + } else { + let idx = rng.random_range(0..allocated.len()); + let (ptr, layout) = allocated.swap_remove(idx); + allocator.dealloc(ptr, layout); + } + } + + for (ptr, layout) in allocated { + allocator.dealloc(ptr, layout); + } + + let stats = allocator.get_stats(); + assert!(stats.used_pages <= stats.total_pages); + assert!(stats.free_pages <= stats.total_pages); + assert_eq!(stats.used_pages + stats.free_pages, stats.total_pages); +} From 632013f4a1496a9a454eddbeb5a94273bf09299d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Tue, 7 Apr 2026 21:37:40 +0800 Subject: [PATCH 05/19] feat: remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic --- CHANGELOG.md | 1 + src/buddy/buddy_allocator.rs | 29 ------ src/buddy/buddy_set.rs | 169 ----------------------------------- src/global_allocator.rs | 27 ------ src/page_allocator.rs | 14 +-- 5 files changed, 2 insertions(+), 238 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ffdff..7b0b51d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the external allocator-interface dependency and restored crate-local `AllocError` / `AllocResult` - Stopped exporting the generic allocator traits `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` - Public allocator APIs now favor concrete allocator methods directly instead of requiring trait imports +- Removed `alloc_pages_at` because the current buddy/slab architecture does not stably support fixed-address allocation ### Removed - Removed the external allocator-interface crate from `[dependencies]` diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs index c5c9d08..78d4601 100644 --- a/src/buddy/buddy_allocator.rs +++ b/src/buddy/buddy_allocator.rs @@ -443,35 +443,6 @@ impl BuddyPageAllocator { } } - pub fn alloc_pages_at( - &mut self, - base: usize, - num_pages: usize, - alignment: usize, - ) -> AllocResult { - // Try to expand node pool if we are close to exhaustion - self.maybe_expand_node_pool(); - - if let Some(zone_idx) = self.find_zone_for_addr(base) { - match self.zones[zone_idx].alloc_pages_at( - &mut self.global_node_pool, - base, - num_pages, - alignment, - ) { - Ok(addr) => { - #[cfg(feature = "tracking")] - self.update_stats(); - Ok(addr) - } - Err(e) => Err(e), - } - } else { - warn!("buddy allocator: alloc_pages_at: address {base:#x} not in any zone"); - Err(AllocError::InvalidParam) - } - } - pub fn total_pages(&self) -> usize { #[cfg(feature = "tracking")] return self.stats.total_pages; diff --git a/src/buddy/buddy_set.rs b/src/buddy/buddy_set.rs index 9517c1a..5018bbd 100644 --- a/src/buddy/buddy_set.rs +++ b/src/buddy/buddy_set.rs @@ -433,175 +433,6 @@ impl BuddySet { 0 } } - - /// Allocate pages at a specific address - /// - /// This method allocates memory at a specific address. If the address range - /// is completely free, it will be allocated. If part of a larger free block, - /// the block will be split appropriately. - pub fn alloc_pages_at( - &mut self, - pool: &mut GlobalNodePool, - base: usize, - num_pages: usize, - alignment: usize, - ) -> AllocResult { - if num_pages == 0 { - return Err(AllocError::InvalidParam); - } - - // Check if address belongs to this zone - if !self.addr_in_zone(base) { - error!( - "zone {}: Address {:#x} not in zone [{:#x}, {:#x})", - self.zone_id, base, self.base_addr, self.end_addr - ); - return Err(AllocError::InvalidParam); - } - - // Check page alignment - if base & (PAGE_SIZE - 1) != 0 { - error!( - "zone {}: Address {:#x} is not page-aligned", - self.zone_id, base - ); - return Err(AllocError::InvalidParam); - } - - // Check alignment requirement - if !base.is_multiple_of(alignment) { - error!( - "zone {}: Address {:#x} is not aligned to {:#x}", - self.zone_id, base, alignment - ); - return Err(AllocError::InvalidParam); - } - - // Check if range fits in zone - let size = num_pages * PAGE_SIZE; - if base + size > self.end_addr { - error!( - "zone {}: Allocation range [{:#x}, {:#x}) exceeds zone end {:#x}", - self.zone_id, - base, - base + size, - self.end_addr - ); - return Err(AllocError::InvalidParam); - } - - // Calculate required order (round up to next power of 2 if needed) - let required_order = if num_pages.is_power_of_two() { - num_pages.trailing_zeros() as usize - } else { - num_pages.next_power_of_two().trailing_zeros() as usize - }; - - // Calculate the order for the block that contains this address - // The block must be aligned to its size - let pfn = base / PAGE_SIZE; - let aligned_pfn = pfn & !((1 << required_order) - 1); - - // Check if the requested base is properly aligned for its size - if aligned_pfn != pfn { - error!( - "zone {}: Base address {:#x} (PFN {}) is not aligned for {} pages", - self.zone_id, - base, - pfn, - 1 << required_order - ); - return Err(AllocError::InvalidParam); - } - - // Try to find a free block that contains this address - // Start from the required order and go up to larger blocks - for order in required_order..=self.max_order() { - let block_pfn = pfn & !((1 << order) - 1); - let block_addr = block_pfn * PAGE_SIZE; - - // Check if this order can contain the request - if let Some((node_idx, prev_idx)) = self.find_block_in_order(pool, order, block_addr) { - // Verify the block is indeed in the free list and capture its data - let node_data = { - let node = pool.get_node(node_idx).unwrap(); - if node.data.order != order || node.data.addr != block_addr { - continue; - } - node.data - }; - - // Remove this block from free list using prev_idx for O(1) deletion - self.free_lists[order].remove_with_prev(pool, node_idx, prev_idx); - - // Now we have a larger block, need to split it - // to keep only the part that covers [base, base + size) - let mut current_block = BuddyBlock { - order, - addr: block_addr, - }; - - // Split down to required order, keeping the requested region - while current_block.order > required_order { - current_block.order -= 1; - let split_size = (1 << current_block.order) * PAGE_SIZE; - - // Calculate buddy address - let buddy_addr = current_block.addr + split_size; - - // Check if buddy is part of the requested region - let request_end = base + size; - - if buddy_addr < base || buddy_addr >= request_end { - // Buddy is outside the requested region, add it back to free list - let success = self.add_block_to_order( - pool, - current_block.order, - BuddyBlock { - order: current_block.order, - addr: buddy_addr, - }, - ); - if !success { - error!( - "zone {}: Failed to return buddy block during split", - self.zone_id - ); - // Put the original block back and fail - self.add_block_to_order(pool, order, node_data); - return Err(AllocError::NoMemory); - } - } - // If buddy is inside the requested region, keep it (don't add back) - } - - // Verify we ended up with the correct block - assert!( - current_block.addr == base, - "zone {}: Final block address {:#x} doesn't match requested {:#x}", - self.zone_id, - current_block.addr, - base - ); - assert!( - current_block.order == required_order, - "zone {}: Final block order {} doesn't match required {}", - self.zone_id, - current_block.order, - required_order - ); - - return Ok(base); - } - } - - // No free block found that contains the requested address - error!( - "zone {}: No free block found for address {:#x} ({} pages)", - self.zone_id, base, num_pages - ); - Err(AllocError::NoMemory) - } } impl Default for BuddySet { diff --git a/src/global_allocator.rs b/src/global_allocator.rs index e42460c..3cdd648 100644 --- a/src/global_allocator.rs +++ b/src/global_allocator.rs @@ -398,33 +398,6 @@ impl GlobalAllocator { } impl GlobalAllocator { - pub fn alloc_pages_at( - &mut self, - base: usize, - num_pages: usize, - alignment: usize, - ) -> AllocResult { - if !self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::NoMemory); - } - - let addr = self - .page_allocator - .alloc_pages_at(base, num_pages, alignment)?; - - #[cfg(feature = "tracking")] - { - self.stats - .used_pages - .fetch_add(num_pages, Ordering::Relaxed); - self.stats - .free_pages - .fetch_sub(num_pages, Ordering::Relaxed); - } - - Ok(addr) - } - pub fn total_pages(&self) -> usize { self.page_allocator.total_pages() } diff --git a/src/page_allocator.rs b/src/page_allocator.rs index 03ff99e..4d5a564 100644 --- a/src/page_allocator.rs +++ b/src/page_allocator.rs @@ -234,7 +234,7 @@ impl CompositePageAllocator { ); // Allocate this specific block - if let Err(_e) = self.buddy.alloc_pages_at(addr, block_pages, alignment) { + if let Err(_e) = self.buddy.alloc_pages(block_pages, alignment) { // Allocation failed, rollback warn!("Contiguous block allocation failed at {i}, rolling back"); #[allow(clippy::needless_range_loop)] @@ -358,18 +358,6 @@ impl CompositePageAllocator { } } - /// Allocate contiguous memory pages at a specific address. - /// - /// Delegates to buddy allocator. - pub fn alloc_pages_at( - &mut self, - base: usize, - num_pages: usize, - alignment: usize, - ) -> AllocResult { - self.buddy.alloc_pages_at(base, num_pages, alignment) - } - /// Return total number of memory pages. pub fn total_pages(&self) -> usize { self.buddy.total_pages() From 14653ce8d51853cac57922981f4df47e79c93bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Wed, 8 Apr 2026 00:27:26 +0800 Subject: [PATCH 06/19] Refactor integration and stress tests for allocator - Introduced a MockOs struct to simulate CPU context switching in tests. - Updated integration tests to use the new MockOs for cross-CPU allocation and deallocation scenarios. - Refactored memory allocation and deallocation functions for better clarity and reuse. - Enhanced stress tests to utilize the MockOs for simulating multi-CPU environments. - Improved error handling and assertions in tests to ensure robustness. - Removed redundant code and comments for cleaner test files. --- Cargo.toml | 1 + README.md | 199 ++++------- README_CN.md | 197 ++++------- benches/README_CN.md | 72 +--- benches/global_allocator.rs | 103 +++++- benches/slab_allocator.rs | 77 +++-- src/buddy/buddy_allocator.rs | 10 - src/global_allocator.rs | 468 ++++++++++++++++---------- src/lib.rs | 41 ++- src/os.rs | 24 ++ src/page_allocator.rs | 13 - src/slab/mod.rs | 4 +- src/slab/slab_byte_allocator.rs | 174 ++++++---- src/slab/slab_cache.rs | 211 ++++++------ src/slab/slab_node.rs | 12 +- tests/dma32_pages_test.rs | 570 ++++++-------------------------- tests/integration_test.rs | 441 ++++++++++++++---------- tests/stress_test.rs | 92 +++++- 18 files changed, 1316 insertions(+), 1393 deletions(-) create mode 100644 src/os.rs diff --git a/Cargo.toml b/Cargo.toml index 6b099bd..3f062d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ tracking = [] [dependencies] cfg-if = "1.0" log = "0.4" +spin = "0.10" [dev-dependencies] criterion = { version = "0.8", features = ["html_reports"] } diff --git a/README.md b/README.md index a7889c3..86500a1 100644 --- a/README.md +++ b/README.md @@ -1,185 +1,114 @@ # buddy-slab-allocator -A high-performance page-level and byte-level memory allocator designed for embedded/kernel environments. +A `no_std` buddy + slab allocator for kernel and embedded environments. ## Features -- **Buddy Page Allocator**: Page-level memory allocation -- **Slab Byte Allocator**: Small object allocation -- **Composite Page Allocator**: Unified multi-region page allocation interface -- **Global Allocator**: Coordinates page and byte allocators with unified allocation interface -- **Zero `std` Dependency**: Fully `#![no_std]`, suitable for embedded and kernel environments -- **Logging Support**: Built on the `log` crate and available by default -- **Memory Tracking**: Support `tracking` feature for detailed statistics +- Buddy page allocator as the shared page backend +- Slab byte allocator as the small-object frontend +- Multi-core `GlobalAllocator` facade with per-CPU slab fronts +- Global OS provider for CPU-local routing +- Optional `tracking` statistics +- Built-in `log` support ## Quick Start ### Add Dependency -Add to your `Cargo.toml`: - ```toml [dependencies] buddy-slab-allocator = "0.1.0" -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # Enable tracking +buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } ``` -### Basic Usage - -#### Using Global Allocator +### Using `GlobalAllocator` ```rust -use buddy_slab_allocator::GlobalAllocator; +use buddy_slab_allocator::{GlobalAllocator, Os}; use core::alloc::Layout; -// Create global allocator -let mut global = GlobalAllocator::new(); - -// Initialize the global allocator with memory region -let heap_start = 0x8000_0000; -let heap_size = 16 * 1024 * 1024; // 16MB -global.init(heap_start, heap_size).unwrap(); - -// Or add multiple memory pools -global.add_memory(0x80000000, 0x1000000).unwrap(); -global.add_memory(0x81000000, 0x1000000).unwrap(); - -// Small object allocation (automatically uses Slab allocator) -let small_layout = Layout::from_size_align(64, 8).unwrap(); -let small_ptr = global.alloc(small_layout).unwrap(); - -// Large object allocation (automatically uses page allocator) -let large_layout = Layout::from_size_align(0x1000, 0x1000).unwrap(); -let large_ptr = global.alloc(large_layout).unwrap(); +const PAGE_SIZE: usize = 0x1000; -// Free memory -global.dealloc(small_ptr, small_layout); -global.dealloc(large_ptr, large_layout); -``` +struct DemoOs; +impl Os for DemoOs { + fn current_cpu_idx(&self) -> usize { + 0 + } +} -#### Using Page Allocator Directly +static OS: DemoOs = DemoOs; -```rust -use buddy_slab_allocator::CompositePageAllocator; +let mut allocator = GlobalAllocator::::new(); -const PAGE_SIZE: usize = 0x1000; -let mut page_alloc = CompositePageAllocator::::new(); +let meta_start = 0x8100_0000; +let meta_size = GlobalAllocator::::required_metadata_size(1); +let heap_start = 0x8000_0000; +let heap_size = 16 * 1024 * 1024; -// Initialize with memory region -page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +unsafe { + allocator + .init(meta_start, meta_size, heap_start, heap_size, 1, &OS) + .unwrap(); +} -// Allocate pages -let addr = page_alloc.alloc_pages(4, PAGE_SIZE).unwrap(); -// Use the allocated memory... -page_alloc.dealloc_pages(addr, 4); +let layout = Layout::from_size_align(64, 8).unwrap(); +let ptr = allocator.alloc(layout).unwrap(); +allocator.dealloc(ptr, layout); ``` -#### Using Slab Allocator Directly +### Using Buddy and Slab Separately ```rust -use buddy_slab_allocator::SlabByteAllocator; -use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab}; +use buddy_slab_allocator::{ + CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, +}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; let mut page_alloc = CompositePageAllocator::::new(); -page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +page_alloc.init(0x8000_0000, 16 * 1024 * 1024); let mut slab_alloc = SlabByteAllocator::::new(); -slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - -// Small allocations are fast let layout = Layout::from_size_align(64, 8).unwrap(); -let ptr = slab_alloc.alloc(layout).unwrap(); - -// Free memory -slab_alloc.dealloc(ptr, layout); -``` - -## Features Details - -### Logging - -```toml -buddy-slab-allocator = "0.1.0" -``` - -You can use standard `log` crate macros to log allocation events: - -```rust -log::info!("Allocated memory at {:x}", addr); -``` - -### Memory Tracking -Enable detailed memory usage tracking via `tracking` feature: - -```toml -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } +let ptr = loop { + match slab_alloc.alloc(layout).unwrap() { + SlabAllocDecision::Allocated(ptr, _) => break ptr, + SlabAllocDecision::NeedsRefill { + size_class, + page_count, + slab_bytes, + } => { + let slab_base = page_alloc.alloc_pages(page_count, slab_bytes).unwrap(); + slab_alloc + .provide_slab(size_class, 0, slab_base, slab_bytes) + .unwrap(); + } + } +}; + +if let SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + .. +} = slab_alloc.dealloc(ptr, layout) +{ + page_alloc.dealloc_pages(slab_base, page_count); +} ``` -After enabling, you can: -- Collect statistics of bytes allocated for each memory usage -- Record backtrace information for each allocation -- Track allocation generation changes - -## Performance - -- **Fast Allocation**: Small object allocation has O(1) time complexity -- **Memory Efficiency**: Buddy algorithm effectively reduces external fragmentation -- **Auto Merge**: Freed pages automatically merge to reduce fragmentation - ## Testing -Run the test suite: - ```bash -# Run all tests -cargo test --package buddy-slab-allocator - -# Run tests with tracking enabled -cargo test --package buddy-slab-allocator --features tracking +cargo test +cargo test --features tracking ``` ## Benchmarking -This project includes comprehensive benchmarks to evaluate performance and stability under various conditions. For detailed instructions, see `benches/README.md`. - ```bash -# Run all benchmarks +cargo check --benches cargo bench ``` -For detailed usage instructions, refer to `benches/README.md`. - -## Documentation - -API documentation is available on [docs.rs](https://docs.rs/buddy-slab-allocator). - -To build and view documentation locally: - -```bash -cargo doc --no-deps --open -``` - -## License - -This project is licensed under: - -- **GPL-3.0-or-later** OR -- **Apache-2.0** OR -- **MIT** - -You may choose any of these licenses for your use. - -## Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. - -## Repository - -[https://github.com/arceos-hypervisor/buddy-slab-allocator](https://github.com/arceos-hypervisor/buddy-slab-allocator) - -## Chinese Documentation - -中文文档请查看 [README_CN.md](README_CN.md) +Detailed benchmark notes are in `benches/README_CN.md`. diff --git a/README_CN.md b/README_CN.md index 20a92c6..9c9cd62 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,181 +1,114 @@ # buddy-slab-allocator 内存分配器 -一个高效的页级和字节级内存分配器，为嵌入式/内核环境设计。 +一个面向内核和嵌入式环境的 `no_std` Buddy + Slab 分配器。 ## 特性 -- **Buddy 页分配器**：页级内存分配 -- **Slab 字节分配器**：小对象分配 -- **复合页分配器**：统一的多区域页分配接口 -- **全局分配器**：协调页分配器和字节分配器，提供统一的分配接口 -- **零 `std` 依赖**：完全 `#![no_std]`，适合嵌入式和内核环境 -- **日志支持**：基于 `log` crate，默认可用 -- **内存追踪**：支持 `tracking` feature 收集详细统计信息 +- Buddy 作为共享页分配后端 +- Slab 作为小对象分配前端 +- 多核 `GlobalAllocator` 门面：共享 Buddy + 每 CPU 一个 Slab +- 全局 OS provider，用于获取当前 CPU +- 可选 `tracking` 统计 +- 内置 `log` 支持 ## 快速开始 ### 添加依赖 -在 `Cargo.toml` 中添加： - ```toml [dependencies] buddy-slab-allocator = "0.1.0" -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # 启用追踪 +buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } ``` -### 基本使用 - -#### 使用全局分配器 +### 使用 `GlobalAllocator` ```rust -use buddy_slab_allocator::GlobalAllocator; +use buddy_slab_allocator::{GlobalAllocator, Os}; use core::alloc::Layout; -// 创建全局分配器 -let mut global = GlobalAllocator::new(); - -// 使用内存区域初始化全局分配器 -let heap_start = 0x8000_0000; -let heap_size = 16 * 1024 * 1024; // 16MB -global.init(heap_start, heap_size).unwrap(); - -// 或添加多个内存池 -global.add_memory(0x80000000, 0x1000000).unwrap(); -global.add_memory(0x81000000, 0x1000000).unwrap(); - -// 小于 2048 byte 的小对象分配（自动使用 Slab） -let small_layout = Layout::from_size_align(64, 8).unwrap(); -let small_ptr = global.alloc(small_layout).unwrap(); - -// 大对象分配（自动使用页分配器） -let large_layout = Layout::from_size_align(0x1000, 0x1000).unwrap(); -let large_ptr = global.alloc(large_layout).unwrap(); +const PAGE_SIZE: usize = 0x1000; -// 释放内存 -global.dealloc(small_ptr, small_layout); -global.dealloc(large_ptr, large_layout); -``` +struct DemoOs; +impl Os for DemoOs { + fn current_cpu_idx(&self) -> usize { + 0 + } +} -#### 直接使用页分配器 +static OS: DemoOs = DemoOs; -```rust -use buddy_slab_allocator::CompositePageAllocator; +let mut allocator = GlobalAllocator::::new(); -const PAGE_SIZE: usize = 0x1000; -let mut page_alloc = CompositePageAllocator::::new(); +let meta_start = 0x8100_0000; +let meta_size = GlobalAllocator::::required_metadata_size(1); +let heap_start = 0x8000_0000; +let heap_size = 16 * 1024 * 1024; -// 使用内存区域初始化 -page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +unsafe { + allocator + .init(meta_start, meta_size, heap_start, heap_size, 1, &OS) + .unwrap(); +} -// 分配页 -let addr = page_alloc.alloc_pages(4, PAGE_SIZE).unwrap(); -// 使用分配的内存... -page_alloc.dealloc_pages(addr, 4); +let layout = Layout::from_size_align(64, 8).unwrap(); +let ptr = allocator.alloc(layout).unwrap(); +allocator.dealloc(ptr, layout); ``` -#### 直接使用 Slab 分配器 +### 分别使用 Buddy 与 Slab ```rust -use buddy_slab_allocator::SlabByteAllocator; -use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab}; +use buddy_slab_allocator::{ + CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, +}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; let mut page_alloc = CompositePageAllocator::::new(); -page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +page_alloc.init(0x8000_0000, 16 * 1024 * 1024); let mut slab_alloc = SlabByteAllocator::::new(); -slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab); - -// 小对象分配快速 let layout = Layout::from_size_align(64, 8).unwrap(); -let ptr = slab_alloc.alloc(layout).unwrap(); - -// 释放内存 -slab_alloc.dealloc(ptr, layout); -``` - -## 特性详解 - -### 日志 - -```toml -buddy-slab-allocator = "0.1.0" -``` - -可直接使用标准 `log` crate 的宏记录分配事件： - -```rust -log::info!("分配内存于 {:x}", addr); -``` - -### 内存追踪 - -通过 `tracking` feature 启用详细的内存使用追踪： -```toml -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } +let ptr = loop { + match slab_alloc.alloc(layout).unwrap() { + SlabAllocDecision::Allocated(ptr, _) => break ptr, + SlabAllocDecision::NeedsRefill { + size_class, + page_count, + slab_bytes, + } => { + let slab_base = page_alloc.alloc_pages(page_count, slab_bytes).unwrap(); + slab_alloc + .provide_slab(size_class, 0, slab_base, slab_bytes) + .unwrap(); + } + } +}; + +if let SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + .. +} = slab_alloc.dealloc(ptr, layout) +{ + page_alloc.dealloc_pages(slab_base, page_count); +} ``` -启用后可以： -- 收集每种内存用途的字节数统计 -- 记录每次分配的回溯信息 -- 跟踪分配代际变化 - -## 性能特性 - -- **快速分配**：小对象分配 O(1) 时间复杂度 -- **内存效率**：Buddy 算法有效减少外部碎片 -- **自动合并**：释放的页面自动合并，减少碎片 - ## 测试 -运行测试套件： - ```bash -# 运行所有测试 -cargo test --package buddy-slab-allocator - -# 启用追踪运行测试 -cargo test --package buddy-slab-allocator --features tracking +cargo test +cargo test --features tracking ``` -## 性能测试 - -此项目包含全面的性能测试，用于在各种条件下评估分配器的性能和稳定性。详细的中文说明请查看 `benches/README_CN.md`。 +## Benchmark ```bash -# 运行所有 benchmark +cargo check --benches cargo bench ``` -详细使用方法请参考 `benches/README_CN.md`。 - -## 文档 - -API 文档可在 [docs.rs](https://docs.rs/buddy-slab-allocator) 上查看。 - -在本地构建和查看文档： - -```bash -cargo doc --no-deps --open -``` - -## 许可证 - -本项目采用以下许可证： - -- **GPL-3.0-or-later** 或 -- **Apache-2.0** 或 -- **MIT** - -你可以选择其中任何一种许可证使用。 - -## 贡献 - -欢迎贡献！请随时提交 Pull Request。 - -## 仓库 - -[https://github.com/arceos-hypervisor/buddy-slab-allocator](https://github.com/arceos-hypervisor/buddy-slab-allocator) +Benchmark 说明见 `benches/README_CN.md`。 diff --git a/benches/README_CN.md b/benches/README_CN.md index c75de09..8ea39a4 100644 --- a/benches/README_CN.md +++ b/benches/README_CN.md @@ -1,86 +1,42 @@ # Benchmark 使用说明 本目录包含 buddy-slab-allocator 的性能 benchmark，统一基于 `criterion`。 -稳定性、耗尽恢复和长时间随机 workload 已迁移到 `tests/stress_test.rs`，不再作为 timing benchmark 运行。 ## 当前 benchmark 套件 - `global_allocator.rs` - 全局分配器真实混合 workload。 + 多核 `GlobalAllocator` 的混合 workload - `buddy_allocator.rs` - 页分配、对齐、碎片化恢复、统计查询。 + Buddy 页分配、对齐、碎片化恢复、统计查询 - `slab_allocator.rs` - 小对象 size class、热路径复用、steady-state 回收。 + Slab 前端 size class、steady-state、热路径复用 ## 运行方式 ```bash +# 仅检查 benchmark 是否可编译 +cargo check --benches + # 运行全部 benchmark cargo bench -# 运行单个 benchmark suite +# 单独运行某个 suite cargo bench --bench buddy_allocator cargo bench --bench slab_allocator cargo bench --bench global_allocator - -# 仅检查 benchmark 是否可编译 -cargo check --benches - -# 保存基线 -cargo bench -- --save-baseline main - -# 与基线对比 -cargo bench -- --baseline main ``` ## 设计原则 -- 使用 `std::hint::black_box`，不再使用已废弃的 `criterion::black_box` -- 统一 `Criterion` 配置： - - `warm_up_time = 1s` - - `measurement_time = 5s` - - `sample_size = 50` -- 优先 benchmark 成对的 alloc/free，避免单轮把堆推向不同状态 -- 对 workload 型场景使用固定批次操作和固定随机种子 -- 通过 `Throughput::{Elements, Bytes}` 明确每类 benchmark 的吞吐量口径 - -## 报告解读 - -运行 benchmark 后，Criterion 会生成 HTML 报告： - -- 路径：`target/criterion/report/index.html` -- 重点关注： - - 平均时间与置信区间 - - `Bytes/s` 或 `Elements/s` 吞吐量 - - 与 baseline 的回归或改进幅度 +- 使用 `std::hint::black_box` +- 统一 `criterion` 配置 +- 尽量 benchmark 成对的 alloc/free +- 固定 workload 批次与随机种子 +- 使用 `Throughput::{Elements, Bytes}` 表达吞吐量 ## 与压力测试的区别 - `benches/` - 只负责性能测量，不包含耗尽恢复、泄漏检查等强断言逻辑。 + 只做性能测量 - `tests/stress_test.rs` - 负责长时间随机、碎片化恢复、耗尽恢复与 tracking 不变量检查。 -cargo bench -- --save-baseline v1 - -# 修改代码后，对比基线 -cargo bench -- --baseline v1 -``` - -### Q: Benchmark 结果不稳定怎么办？ - -A: 可能原因： -1. 系统负载过高 -2. 频率调整或电源管理 -3. 后台程序干扰 - -建议： -1. 关闭不必要的程序 -2. 多次运行取平均值 -3. 使用固定 CPU 频率 - -## 参考文献 - -- [Criterion.rs 文档](https://bheisler.github.io/criterion.rs/book/) -- [Rust 性能优化指南](https://nnethercote.github.io/perf-book/) -- [Buddy 算法原理](https://en.wikipedia.org/wiki/Buddy_memory_allocation) -- [Slab 分配器设计](https://en.wikipedia.org/wiki/Slab_allocation) + 负责长时间随机、碎片化恢复、耗尽恢复与 tracking 不变量 diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index 5310787..0ce98f9 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -2,24 +2,85 @@ mod common; -use buddy_slab_allocator::GlobalAllocator; +use buddy_slab_allocator::{GlobalAllocator, Os}; use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use std::alloc::Layout; +use spin::Mutex; +use std::alloc::{alloc, dealloc, Layout}; use std::hint::black_box; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } + + fn set_cpu(&self, cpu: usize) { + self.cpu.store(cpu, Ordering::Relaxed); + } +} + +impl Os for MockOs { + fn current_cpu_idx(&self) -> usize { + self.cpu.load(Ordering::Relaxed) + } +} + +static MOCK_OS: MockOs = MockOs::new(); + +struct MetadataBuf { + ptr: *mut u8, + layout: Layout, +} + +impl MetadataBuf { + fn new(cpu_count: usize) -> Self { + let size = GlobalAllocator::::required_metadata_size(cpu_count); + let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate metadata"); + Self { ptr, layout } + } +} + +impl Drop for MetadataBuf { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} struct GlobalHarness { _heap: TestHeap, - allocator: Box>, + _meta: MetadataBuf, + allocator: GlobalAllocator, } impl GlobalHarness { - fn new() -> Self { + fn new(cpu_count: usize) -> Self { let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = Box::new(GlobalAllocator::::new()); - allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let meta = MetadataBuf::new(cpu_count); + let mut allocator = GlobalAllocator::::new(); + unsafe { + allocator + .init( + meta.ptr as usize, + meta.layout.size(), + heap.addr(), + HEAP_SIZE, + cpu_count, + &MOCK_OS, + ) + .unwrap(); + } Self { _heap: heap, + _meta: meta, allocator, } } @@ -30,13 +91,15 @@ fn bench_small_alloc_free(c: &mut Criterion) { for &size in &[8usize, 64, 512, 2048] { group.throughput(Throughput::Bytes(size as u64)); - let mut harness = GlobalHarness::new(); + let harness = Mutex::new(GlobalHarness::new(2)); let layout = Layout::from_size_align(size, 8).unwrap(); group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); - harness.allocator.dealloc(ptr, layout); + MOCK_OS.set_cpu(0); + let guard = harness.lock(); + let ptr = guard.allocator.alloc(black_box(layout)).unwrap(); + guard.allocator.dealloc(ptr, layout); black_box(ptr); }); }); @@ -50,13 +113,14 @@ fn bench_large_alloc_free(c: &mut Criterion) { for &size in &[PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16] { group.throughput(Throughput::Bytes(size as u64)); - let mut harness = GlobalHarness::new(); + let harness = Mutex::new(GlobalHarness::new(2)); let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); - harness.allocator.dealloc(ptr, layout); + let guard = harness.lock(); + let ptr = guard.allocator.alloc(black_box(layout)).unwrap(); + guard.allocator.dealloc(ptr, layout); black_box(ptr); }); }); @@ -69,7 +133,7 @@ fn bench_mixed_workload(c: &mut Criterion) { let mut group = c.benchmark_group("global/mixed_workload"); group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - let mut harness = GlobalHarness::new(); + let harness = Mutex::new(GlobalHarness::new(2)); let layouts = [ Layout::from_size_align(64, 8).unwrap(), Layout::from_size_align(256, 8).unwrap(), @@ -80,9 +144,11 @@ fn bench_mixed_workload(c: &mut Criterion) { group.bench_function("alternating_small_and_large", |b| { b.iter(|| { for idx in 0..OPERATIONS_PER_BATCH as usize { + MOCK_OS.set_cpu(idx % 2); let layout = layouts[idx % layouts.len()]; - let ptr = harness.allocator.alloc(layout).unwrap(); - harness.allocator.dealloc(ptr, layout); + let guard = harness.lock(); + let ptr = guard.allocator.alloc(layout).unwrap(); + guard.allocator.dealloc(ptr, layout); black_box(ptr); } }); @@ -96,18 +162,19 @@ fn bench_page_interface(c: &mut Criterion) { for &num_pages in &[1usize, 4, 16, 64] { group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); - let mut harness = GlobalHarness::new(); + let harness = Mutex::new(GlobalHarness::new(2)); group.bench_with_input( BenchmarkId::from_parameter(num_pages), &num_pages, |b, &pages| { b.iter(|| { - let addr = harness + let guard = harness.lock(); + let addr = guard .allocator .alloc_pages(black_box(pages), PAGE_SIZE) .unwrap(); - harness.allocator.dealloc_pages(addr, pages); + guard.allocator.dealloc_pages(addr, pages); black_box(addr); }); }, diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index fe19925..38c7f95 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -2,7 +2,9 @@ mod common; -use buddy_slab_allocator::{CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator}; +use buddy_slab_allocator::{ + CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, +}; use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use std::alloc::Layout; @@ -10,25 +12,52 @@ use std::hint::black_box; struct SlabHarness { _heap: TestHeap, - _page_allocator: Box>, + page_allocator: CompositePageAllocator, slab_allocator: SlabByteAllocator, } impl SlabHarness { fn new() -> Self { let heap = TestHeap::new(HEAP_SIZE); - let mut page_allocator = Box::new(CompositePageAllocator::::new()); + let mut page_allocator = CompositePageAllocator::::new(); page_allocator.init(heap.addr(), HEAP_SIZE); - let mut slab_allocator = SlabByteAllocator::::new(); - slab_allocator.init(); - let page_allocator_ptr = (&mut *page_allocator) as *mut dyn PageAllocatorForSlab; - slab_allocator.set_page_allocator(page_allocator_ptr); - Self { _heap: heap, - _page_allocator: page_allocator, - slab_allocator, + page_allocator, + slab_allocator: SlabByteAllocator::new(), + } + } + + fn alloc_small(&mut self, layout: Layout) -> core::ptr::NonNull { + loop { + match self.slab_allocator.alloc(layout).unwrap() { + SlabAllocDecision::Allocated(ptr, _) => return ptr, + SlabAllocDecision::NeedsRefill { + size_class, + page_count, + slab_bytes, + } => { + let slab_base = self + .page_allocator + .alloc_pages(page_count, slab_bytes) + .unwrap(); + self.slab_allocator + .provide_slab(size_class, 0, slab_base, slab_bytes) + .unwrap(); + } + } + } + } + + fn dealloc_small(&mut self, ptr: core::ptr::NonNull, layout: Layout) { + if let SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + .. + } = self.slab_allocator.dealloc(ptr, layout) + { + self.page_allocator.dealloc_pages(slab_base, page_count); } } } @@ -43,8 +72,8 @@ fn bench_size_class_alloc_free(c: &mut Criterion) { group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { b.iter(|| { - let ptr = harness.slab_allocator.alloc(black_box(layout)).unwrap(); - harness.slab_allocator.dealloc(ptr, layout); + let ptr = harness.alloc_small(black_box(layout)); + harness.dealloc_small(ptr, layout); black_box(ptr); }); }); @@ -59,14 +88,13 @@ fn bench_hot_reuse(c: &mut Criterion) { let mut harness = SlabHarness::new(); let layout = Layout::from_size_align(128, 8).unwrap(); - - let ptr = harness.slab_allocator.alloc(layout).unwrap(); - harness.slab_allocator.dealloc(ptr, layout); + let ptr = harness.alloc_small(layout); + harness.dealloc_small(ptr, layout); group.bench_function("alloc_free_hot_cache", |b| { b.iter(|| { - let ptr = harness.slab_allocator.alloc(layout).unwrap(); - harness.slab_allocator.dealloc(ptr, layout); + let ptr = harness.alloc_small(layout); + harness.dealloc_small(ptr, layout); black_box(ptr); }); }); @@ -92,8 +120,8 @@ fn bench_mixed_size_batch(c: &mut Criterion) { group.bench_function("alloc_free_layout_batch", |b| { b.iter(|| { for layout in layouts { - let ptr = harness.slab_allocator.alloc(layout).unwrap(); - harness.slab_allocator.dealloc(ptr, layout); + let ptr = harness.alloc_small(layout); + harness.dealloc_small(ptr, layout); black_box(ptr); } }); @@ -109,16 +137,15 @@ fn bench_steady_state(c: &mut Criterion) { let mut harness = SlabHarness::new(); let layout = Layout::from_size_align(64, 8).unwrap(); let mut active = Vec::with_capacity(256); - for _ in 0..256 { - active.push(harness.slab_allocator.alloc(layout).unwrap()); + active.push(harness.alloc_small(layout)); } group.bench_function("recycle_live_object", |b| { b.iter(|| { let ptr = active.pop().unwrap(); - harness.slab_allocator.dealloc(ptr, layout); - let new_ptr = harness.slab_allocator.alloc(layout).unwrap(); + harness.dealloc_small(ptr, layout); + let new_ptr = harness.alloc_small(layout); active.push(new_ptr); black_box(new_ptr); }); @@ -143,8 +170,8 @@ fn bench_batch_workload(c: &mut Criterion) { b.iter(|| { for idx in 0..OPERATIONS_PER_BATCH as usize { let layout = layouts[idx % layouts.len()]; - let ptr = harness.slab_allocator.alloc(layout).unwrap(); - harness.slab_allocator.dealloc(ptr, layout); + let ptr = harness.alloc_small(layout); + harness.dealloc_small(ptr, layout); black_box(ptr); } }); diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs index 78d4601..d099a37 100644 --- a/src/buddy/buddy_allocator.rs +++ b/src/buddy/buddy_allocator.rs @@ -465,16 +465,6 @@ impl BuddyPageAllocator { } } -impl crate::slab::PageAllocatorForSlab for BuddyPageAllocator { - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - Self::alloc_pages(self, num_pages, alignment) - } - - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - Self::dealloc_pages(self, pos, num_pages); - } -} - impl Default for BuddyPageAllocator { fn default() -> Self { Self::new() diff --git a/src/global_allocator.rs b/src/global_allocator.rs index 3cdd648..dd39c57 100644 --- a/src/global_allocator.rs +++ b/src/global_allocator.rs @@ -1,27 +1,31 @@ //! Global allocator implementation. //! -//! This module implements a global allocator that coordinates between -//! buddy page allocator and slab byte allocator for optimal performance. +//! This module implements a multi-core global allocator that coordinates +//! between a shared page allocator backend and per-CPU slab frontends. extern crate alloc; +use crate::os::set_os_provider; +use crate::slab::slab_node::SlabNode; +use crate::slab::{SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision}; use crate::{AllocError, AllocResult}; use core::alloc::Layout; +use core::mem::{align_of, size_of}; use core::ptr::NonNull; +use core::slice; #[cfg(feature = "tracking")] use core::sync::atomic::AtomicUsize; use core::sync::atomic::{AtomicBool, Ordering}; +use log::error; +use spin::Mutex; #[cfg(feature = "tracking")] use super::buddy::BuddyStats; use super::page_allocator::CompositePageAllocator; -use super::slab::{PageAllocatorForSlab, SlabByteAllocator}; - -use log::error; -const MIN_HEAP_SIZE: usize = 0x8000; // 32KB minimum heap +const MIN_HEAP_SIZE: usize = 0x8000; -/// Memory usage statistics +/// Memory usage statistics. #[cfg(feature = "tracking")] #[derive(Debug, Clone, Copy, Default)] pub struct UsageStats { @@ -32,7 +36,6 @@ pub struct UsageStats { pub heap_bytes: usize, } -/// Internal atomic representation of usage statistics #[cfg(feature = "tracking")] struct UsageStatsAtomic { total_pages: AtomicUsize, @@ -78,10 +81,31 @@ fn saturating_sub_atomic(counter: &AtomicUsize, value: usize) { } } -/// Global allocator that coordinates composite and slab allocators +#[derive(Clone, Copy, Default)] +struct MetadataRegionInfo { + start: usize, + size: usize, +} + +struct PerCpuSlabSlot { + slab: Mutex>, +} + +impl PerCpuSlabSlot { + const fn new() -> Self { + Self { + slab: Mutex::new(SlabByteAllocator::new()), + } + } +} + +/// Multi-core global allocator facade. pub struct GlobalAllocator { - page_allocator: CompositePageAllocator, - slab_allocator: SlabByteAllocator, + buddy: Mutex>, + slab_slots_ptr: Option>>, + cpu_count: usize, + metadata_region: MetadataRegionInfo, + os: &'static dyn crate::Os, #[cfg(feature = "tracking")] stats: UsageStatsAtomic, initialized: AtomicBool, @@ -90,33 +114,79 @@ pub struct GlobalAllocator GlobalAllocator { pub const fn new() -> Self { Self { - page_allocator: CompositePageAllocator::::new(), - slab_allocator: SlabByteAllocator::::new(), + buddy: Mutex::new(CompositePageAllocator::::new()), + slab_slots_ptr: None, + cpu_count: 0, + metadata_region: MetadataRegionInfo { start: 0, size: 0 }, + os: &crate::NoImplOs, #[cfg(feature = "tracking")] stats: UsageStatsAtomic::new(), initialized: AtomicBool::new(false), } } + pub fn required_metadata_size(cpu_count: usize) -> usize { + if cpu_count == 0 { + return 0; + } + let slot_size = size_of::>(); + let slot_align = align_of::>(); + let size = slot_size.saturating_mul(cpu_count); + crate::align_up(size, slot_align) + } + + pub fn required_metadata_align() -> usize { + align_of::>() + } + /// Set the address translator so that the underlying page allocator can /// reason about physical address ranges (e.g. low-memory regions below 4GiB). - pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) { - self.page_allocator.set_addr_translator(translator); + pub fn set_addr_translator(&self, translator: &'static dyn crate::AddrTranslator) { + self.buddy.lock().set_addr_translator(translator); + } + + fn slab_slots(&self) -> &[PerCpuSlabSlot] { + let ptr = self + .slab_slots_ptr + .expect("global allocator slab slots accessed before init"); + unsafe { slice::from_raw_parts(ptr.as_ptr(), self.cpu_count) } } - /// Allocate low-memory pages (physical address < 4GiB). - /// This is a thin wrapper over the composite allocator's lowmem API. - pub fn alloc_dma32_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { + fn slab_slot(&self, cpu_id: usize) -> AllocResult<&PerCpuSlabSlot> { if !self.initialized.load(Ordering::SeqCst) { - error!("global allocator: Allocator not initialized"); return Err(AllocError::NoMemory); } + if cpu_id >= self.cpu_count { + return Err(AllocError::InvalidParam); + } + Ok(&self.slab_slots()[cpu_id]) + } - let addr = self - .page_allocator - .alloc_pages_lowmem(num_pages, alignment)?; + fn current_cpu_idx_checked(&self) -> AllocResult { + let cpu_id = self.os.current_cpu_idx(); + if cpu_id >= self.cpu_count { + error!( + "global allocator: OS reported cpu_id {} >= cpu_count {}", + cpu_id, self.cpu_count + ); + return Err(AllocError::InvalidParam); + } + Ok(cpu_id) + } + + fn owner_cpu_from_ptr(&self, ptr: NonNull, layout: Layout) -> Option { + let size_class = SizeClass::from_layout(layout)?; + let slab_bytes = size_class.slab_bytes(PAGE_SIZE); + let slab_base = (ptr.as_ptr() as usize / slab_bytes) * slab_bytes; + let node = SlabNode::new(slab_base, size_class); + if node.is_valid_for_size_class() { + Some(node.owner_cpu()) + } else { + None + } + } - // Update statistics + fn update_page_stats_after_alloc(&self, num_pages: usize) { #[cfg(feature = "tracking")] { self.stats @@ -126,232 +196,264 @@ impl GlobalAllocator { .free_pages .fetch_sub(num_pages, Ordering::Relaxed); } + #[cfg(not(feature = "tracking"))] + let _ = num_pages; + } - Ok(addr) + fn update_page_stats_after_free(&self, num_pages: usize) { + #[cfg(feature = "tracking")] + { + saturating_sub_atomic(&self.stats.used_pages, num_pages); + self.stats + .free_pages + .fetch_add(num_pages, Ordering::Relaxed); + } + #[cfg(not(feature = "tracking"))] + let _ = num_pages; } - /// Initialize allocator with given memory region + /// Initialize the allocator with metadata storage, memory region, CPU topology and OS hooks. /// - /// # Examples + /// # Safety /// - /// ```no_run - /// use buddy_slab_allocator::GlobalAllocator; - /// - /// const PAGE_SIZE: usize = 0x1000; - /// let mut allocator = GlobalAllocator::::new(); - /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); - /// ``` - pub fn init(&mut self, start_vaddr: usize, size: usize) -> AllocResult<()> { - if size <= MIN_HEAP_SIZE { + /// The caller must ensure that `[meta_start, meta_start + meta_size)` is valid writable memory + /// for the lifetime of the allocator and is not used for any other purpose. + pub unsafe fn init( + &mut self, + meta_start: usize, + meta_size: usize, + heap_start: usize, + heap_size: usize, + cpu_count: usize, + os: &'static dyn crate::Os, + ) -> AllocResult<()> { + if self.initialized.load(Ordering::SeqCst) { + return Err(AllocError::InvalidParam); + } + if cpu_count == 0 || heap_size <= MIN_HEAP_SIZE { return Err(AllocError::InvalidParam); } + if !meta_start.is_multiple_of(Self::required_metadata_align()) { + return Err(AllocError::InvalidParam); + } + if meta_size < Self::required_metadata_size(cpu_count) { + return Err(AllocError::NoMemory); + } - self.page_allocator.init(start_vaddr, size); + let slots_ptr = meta_start as *mut PerCpuSlabSlot; + for idx in 0..cpu_count { + slots_ptr.add(idx).write(PerCpuSlabSlot::new()); + } - self.slab_allocator.init(); + self.slab_slots_ptr = Some(NonNull::new(slots_ptr).ok_or(AllocError::InvalidParam)?); + self.cpu_count = cpu_count; + self.metadata_region = MetadataRegionInfo { + start: meta_start, + size: meta_size, + }; + self.os = os; - { - let page_alloc_ptr = &mut self.page_allocator as *mut CompositePageAllocator; - self.slab_allocator - .set_page_allocator(page_alloc_ptr as *mut dyn PageAllocatorForSlab); - } + self.buddy.get_mut().init(heap_start, heap_size); - // Update statistics #[cfg(feature = "tracking")] { + let buddy = self.buddy.get_mut(); self.stats .total_pages - .store(self.page_allocator.total_pages(), Ordering::Relaxed); + .store(buddy.total_pages(), Ordering::Relaxed); self.stats .used_pages - .store(self.page_allocator.used_pages(), Ordering::Relaxed); + .store(buddy.used_pages(), Ordering::Relaxed); self.stats .free_pages - .store(self.page_allocator.available_pages(), Ordering::Relaxed); + .store(buddy.available_pages(), Ordering::Relaxed); } + set_os_provider(os); self.initialized.store(true, Ordering::SeqCst); Ok(()) } - /// Dynamically add memory region to allocator - pub fn add_memory(&mut self, start_vaddr: usize, size: usize) -> AllocResult<()> { - self.page_allocator.add_memory(start_vaddr, size)?; + pub fn add_memory(&self, start_vaddr: usize, size: usize) -> AllocResult<()> { + if !self.initialized.load(Ordering::SeqCst) { + return Err(AllocError::NoMemory); + } + + let mut buddy = self.buddy.lock(); + buddy.add_memory(start_vaddr, size)?; - // Update statistics #[cfg(feature = "tracking")] { self.stats .total_pages - .store(self.page_allocator.total_pages(), Ordering::Relaxed); + .store(buddy.total_pages(), Ordering::Relaxed); self.stats .free_pages - .store(self.page_allocator.available_pages(), Ordering::Relaxed); + .store(buddy.available_pages(), Ordering::Relaxed); } Ok(()) } - /// Smart allocation based on size - /// - /// Small allocations (≤2048 bytes) use slab allocator, - /// larger allocations use page allocator. - /// - /// # Examples - /// - /// ```no_run - /// use buddy_slab_allocator::GlobalAllocator; - /// use core::alloc::Layout; - /// - /// const PAGE_SIZE: usize = 0x1000; - /// let mut allocator = GlobalAllocator::::new(); - /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); - /// - /// let layout = Layout::from_size_align(64, 8).unwrap(); - /// let ptr = allocator.alloc(layout).unwrap(); - /// allocator.dealloc(ptr, layout); - /// ``` - pub fn alloc(&mut self, layout: Layout) -> AllocResult> { + pub fn alloc_dma32_pages(&self, num_pages: usize, alignment: usize) -> AllocResult { + if !self.initialized.load(Ordering::SeqCst) { + error!("global allocator: Allocator not initialized"); + return Err(AllocError::NoMemory); + } + + let mut buddy = self.buddy.lock(); + let addr = buddy.alloc_pages_lowmem(num_pages, alignment)?; + self.update_page_stats_after_alloc(num_pages); + Ok(addr) + } + + pub fn alloc(&self, layout: Layout) -> AllocResult> { if !self.initialized.load(Ordering::SeqCst) { error!("global allocator: Allocator not initialized"); return Err(AllocError::NoMemory); } if layout.size() <= 2048 && layout.align() <= 2048 { - // Try slab allocator first - match self.slab_allocator.alloc(layout) { - Ok(ptr) => { - #[cfg(feature = "tracking")] - { - self.stats - .slab_bytes - .fetch_add(layout.size(), Ordering::Relaxed); + let cpu_id = self.current_cpu_idx_checked()?; + + loop { + let decision = { + let slot = self.slab_slot(cpu_id)?; + let mut slab = slot.slab.lock(); + slab.alloc(layout)? + }; + + match decision { + SlabAllocDecision::Allocated(ptr, _) => { + #[cfg(feature = "tracking")] + { + self.stats + .slab_bytes + .fetch_add(layout.size(), Ordering::Relaxed); + } + return Ok(ptr); + } + SlabAllocDecision::NeedsRefill { + size_class, + page_count, + slab_bytes, + } => { + let slab_base = { + let mut buddy = self.buddy.lock(); + let addr = buddy.alloc_pages(page_count, slab_bytes)?; + self.update_page_stats_after_alloc(page_count); + addr + }; + + let slot = self.slab_slot(cpu_id)?; + let mut slab = slot.slab.lock(); + slab.provide_slab(size_class, cpu_id, slab_base, slab_bytes)?; } - return Ok(ptr); - } - Err(e) => { - // Slab allocator should handle all requests that satisfy constraints - // If it fails, it's a real error (e.g., out of memory) - // Log for debugging - error!( - "global allocator: Slab allocator failed for layout {layout:?}, error: {e:?}, falling back to page allocator" - ); - return Err(e); } } } let pages_needed = layout.size().div_ceil(PAGE_SIZE); - - let addr = self - .page_allocator - .alloc_pages(pages_needed, layout.align())?; - let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; - + let addr = { + let mut buddy = self.buddy.lock(); + buddy.alloc_pages(pages_needed, layout.align())? + }; + self.update_page_stats_after_alloc(pages_needed); #[cfg(feature = "tracking")] { - self.stats - .used_pages - .fetch_add(pages_needed, Ordering::Relaxed); - self.stats - .free_pages - .fetch_sub(pages_needed, Ordering::Relaxed); self.stats .heap_bytes .fetch_add(layout.size(), Ordering::Relaxed); } - - Ok(ptr) + Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) }) } - /// Allocate pages - /// - /// # Examples - /// - /// ```no_run - /// use buddy_slab_allocator::GlobalAllocator; - /// - /// const PAGE_SIZE: usize = 0x1000; - /// let mut allocator = GlobalAllocator::::new(); - /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); - /// - /// let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); - /// allocator.dealloc_pages(addr, 4); - /// ``` - pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { + pub fn alloc_pages(&self, num_pages: usize, alignment: usize) -> AllocResult { if !self.initialized.load(Ordering::SeqCst) { return Err(AllocError::NoMemory); } - let addr = self.page_allocator.alloc_pages(num_pages, alignment)?; - - // Update statistics - #[cfg(feature = "tracking")] - { - self.stats - .used_pages - .fetch_add(num_pages, Ordering::Relaxed); - self.stats - .free_pages - .fetch_sub(num_pages, Ordering::Relaxed); - } - + let addr = { + let mut buddy = self.buddy.lock(); + buddy.alloc_pages(num_pages, alignment)? + }; + self.update_page_stats_after_alloc(num_pages); Ok(addr) } - /// Deallocate memory - pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) { + pub fn dealloc(&self, ptr: NonNull, layout: Layout) { if !self.initialized.load(Ordering::SeqCst) { error!("global allocator: Deallocating memory before initializing"); return; } if layout.size() <= 2048 && layout.align() <= 2048 { - // This memory must have been allocated by slab allocator - // If dealloc fails (not found in slab), it's a critical error - self.slab_allocator.dealloc(ptr, layout); - #[cfg(feature = "tracking")] - { - saturating_sub_atomic(&self.stats.slab_bytes, layout.size()); + let owner_cpu = self + .owner_cpu_from_ptr(ptr, layout) + .unwrap_or_else(|| self.os.current_cpu_idx()); + if let Ok(slot) = self.slab_slot(owner_cpu) { + let decision = { + let mut slab = slot.slab.lock(); + let decision = slab.dealloc(ptr, layout); + #[cfg(feature = "tracking")] + { + let actually_deallocated = match &decision { + SlabDeallocDecision::Done { + actually_deallocated, + .. + } + | SlabDeallocDecision::ReleaseSlab { + actually_deallocated, + .. + } => *actually_deallocated, + }; + if actually_deallocated { + saturating_sub_atomic(&self.stats.slab_bytes, layout.size()); + } + } + decision + }; + + if let SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + .. + } = decision + { + let mut buddy = self.buddy.lock(); + buddy.dealloc_pages(slab_base, page_count); + self.update_page_stats_after_free(page_count); + } } return; } - // This memory was allocated by page allocator let pages_needed = layout.size().div_ceil(PAGE_SIZE); - self.page_allocator - .dealloc_pages(ptr.as_ptr() as usize, pages_needed); + { + let mut buddy = self.buddy.lock(); + buddy.dealloc_pages(ptr.as_ptr() as usize, pages_needed); + } + self.update_page_stats_after_free(pages_needed); #[cfg(feature = "tracking")] { - saturating_sub_atomic(&self.stats.used_pages, pages_needed); - self.stats - .free_pages - .fetch_add(pages_needed, Ordering::Relaxed); saturating_sub_atomic(&self.stats.heap_bytes, layout.size()); } } - /// Deallocate pages - pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { + pub fn dealloc_pages(&self, pos: usize, num_pages: usize) { if !self.initialized.load(Ordering::SeqCst) { return; } - self.page_allocator.dealloc_pages(pos, num_pages); - - // Update statistics - #[cfg(feature = "tracking")] { - saturating_sub_atomic(&self.stats.used_pages, num_pages); - self.stats - .free_pages - .fetch_add(num_pages, Ordering::Relaxed); + let mut buddy = self.buddy.lock(); + buddy.dealloc_pages(pos, num_pages); } + self.update_page_stats_after_free(num_pages); } - /// Reallocate memory #[allow(clippy::not_unsafe_ptr_arg_deref)] - pub fn realloc(&mut self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + pub fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { if new_size == 0 { if let Some(ptr) = NonNull::new(ptr) { self.dealloc(ptr, layout); @@ -368,15 +470,12 @@ impl GlobalAllocator { }; } - let new_layout = Layout::from_size_align(new_size, layout.align()) - .unwrap_or_else(|_| Layout::new::()); - - // If new size fits in old allocation, return old pointer if new_size <= layout.size() { return ptr; } - // Allocate new memory and copy + let new_layout = Layout::from_size_align(new_size, layout.align()) + .unwrap_or_else(|_| Layout::new::()); match self.alloc(new_layout) { Ok(new_ptr) => { let new_ptr = new_ptr.as_ptr(); @@ -395,31 +494,56 @@ impl GlobalAllocator { Err(_) => core::ptr::null_mut(), } } -} -impl GlobalAllocator { pub fn total_pages(&self) -> usize { - self.page_allocator.total_pages() + #[cfg(feature = "tracking")] + { + return self.stats.total_pages.load(Ordering::Relaxed); + } + #[cfg(not(feature = "tracking"))] + { + self.buddy.lock().total_pages() + } } pub fn used_pages(&self) -> usize { - self.page_allocator.used_pages() + #[cfg(feature = "tracking")] + { + return self.stats.used_pages.load(Ordering::Relaxed); + } + #[cfg(not(feature = "tracking"))] + { + self.buddy.lock().used_pages() + } } pub fn available_pages(&self) -> usize { - self.page_allocator.available_pages() + #[cfg(feature = "tracking")] + { + return self.stats.free_pages.load(Ordering::Relaxed); + } + #[cfg(not(feature = "tracking"))] + { + self.buddy.lock().available_pages() + } + } + + pub fn cpu_count(&self) -> usize { + self.cpu_count + } + + pub fn metadata_region(&self) -> (usize, usize) { + (self.metadata_region.start, self.metadata_region.size) } - /// Get memory statistics #[cfg(feature = "tracking")] pub fn get_stats(&self) -> UsageStats { self.stats.snapshot() } - /// Get buddy allocator statistics #[cfg(feature = "tracking")] pub fn get_buddy_stats(&self) -> BuddyStats { - self.page_allocator.get_buddy_stats() + self.buddy.lock().get_buddy_stats() } } diff --git a/src/lib.rs b/src/lib.rs index 98d091b..25a3469 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,11 +24,18 @@ //! //! const PAGE_SIZE: usize = 0x1000; //! let mut allocator = GlobalAllocator::::new(); +//! struct DemoOs; +//! impl buddy_slab_allocator::Os for DemoOs { +//! fn current_cpu_idx(&self) -> usize { 0 } +//! } +//! static OS: DemoOs = DemoOs; //! //! // Initialize with memory region //! let heap_start = 0x8000_0000; //! let heap_size = 16 * 1024 * 1024; // 16MB -//! allocator.init(heap_start, heap_size).unwrap(); +//! let meta_start = 0x8100_0000; +//! let meta_size = GlobalAllocator::::required_metadata_size(1); +//! unsafe { allocator.init(meta_start, meta_size, heap_start, heap_size, 1, &OS).unwrap(); } //! //! // Allocate pages //! let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); @@ -44,7 +51,18 @@ //! //! const PAGE_SIZE: usize = 0x1000; //! let mut allocator = GlobalAllocator::::new(); -//! allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +//! struct DemoOs; +//! impl buddy_slab_allocator::Os for DemoOs { +//! fn current_cpu_idx(&self) -> usize { 0 } +//! } +//! static OS: DemoOs = DemoOs; +//! let meta_start = 0x8100_0000; +//! let meta_size = GlobalAllocator::::required_metadata_size(1); +//! unsafe { +//! allocator +//! .init(meta_start, meta_size, 0x8000_0000, 16 * 1024 * 1024, 1, &OS) +//! .unwrap(); +//! } //! //! // Small allocations go through slab allocator //! let layout = Layout::from_size_align(64, 8).unwrap(); @@ -62,7 +80,18 @@ //! //! const PAGE_SIZE: usize = 0x1000; //! let mut allocator = GlobalAllocator::::new(); -//! allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap(); +//! struct DemoOs; +//! impl buddy_slab_allocator::Os for DemoOs { +//! fn current_cpu_idx(&self) -> usize { 0 } +//! } +//! static OS: DemoOs = DemoOs; +//! let meta_start = 0x8100_0000; +//! let meta_size = GlobalAllocator::::required_metadata_size(1); +//! unsafe { +//! allocator +//! .init(meta_start, meta_size, 0x8000_0000, 16 * 1024 * 1024, 1, &OS) +//! .unwrap(); +//! } //! //! let stats = allocator.get_stats(); //! println!("Total pages: {}", stats.total_pages); @@ -77,6 +106,8 @@ extern crate alloc; mod error; pub use error::{AllocError, AllocResult}; +mod os; +pub use os::{current_cpu_idx, NoImplOs, Os}; /// Default page size for backward compatibility (4KB) pub const DEFAULT_PAGE_SIZE: usize = 0x1000; @@ -138,7 +169,9 @@ pub mod page_allocator; pub use page_allocator::CompositePageAllocator; pub mod slab; -pub use slab::slab_byte_allocator::{PageAllocatorForSlab, SizeClass, SlabByteAllocator}; +pub use slab::slab_byte_allocator::{ + SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, +}; pub mod global_allocator; pub use global_allocator::GlobalAllocator; diff --git a/src/os.rs b/src/os.rs new file mode 100644 index 0000000..14b73a9 --- /dev/null +++ b/src/os.rs @@ -0,0 +1,24 @@ +use spin::RwLock; + +pub trait Os: Sync { + fn current_cpu_idx(&self) -> usize; +} + +pub struct NoImplOs; + +impl Os for NoImplOs { + fn current_cpu_idx(&self) -> usize { + panic!("buddy-slab-allocator: Os provider is not initialized") + } +} + +static NO_IMPL_OS: NoImplOs = NoImplOs; +static OS_PROVIDER: RwLock<&'static dyn Os> = RwLock::new(&NO_IMPL_OS); + +pub(crate) fn set_os_provider(os: &'static dyn Os) { + *OS_PROVIDER.write() = os; +} + +pub fn current_cpu_idx() -> usize { + OS_PROVIDER.read().current_cpu_idx() +} diff --git a/src/page_allocator.rs b/src/page_allocator.rs index 4d5a564..e064e9a 100644 --- a/src/page_allocator.rs +++ b/src/page_allocator.rs @@ -382,19 +382,6 @@ impl CompositePageAllocator { } } -// Implement PageAllocatorForSlab for CompositePageAllocator -impl crate::slab::PageAllocatorForSlab - for CompositePageAllocator -{ - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - Self::alloc_pages(self, num_pages, alignment) - } - - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - Self::dealloc_pages(self, pos, num_pages) - } -} - impl Default for CompositePageAllocator { fn default() -> Self { Self::new() diff --git a/src/slab/mod.rs b/src/slab/mod.rs index 2067c63..3c4ae9d 100644 --- a/src/slab/mod.rs +++ b/src/slab/mod.rs @@ -8,4 +8,6 @@ pub mod slab_cache; pub mod slab_node; // Re-export public types -pub use slab_byte_allocator::{PageAllocatorForSlab, SizeClass, SlabByteAllocator}; +pub use slab_byte_allocator::{ + SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, +}; diff --git a/src/slab/slab_byte_allocator.rs b/src/slab/slab_byte_allocator.rs index a8ae295..3733b20 100644 --- a/src/slab/slab_byte_allocator.rs +++ b/src/slab/slab_byte_allocator.rs @@ -1,4 +1,4 @@ -//! Slab byte allocator implementation for Axvisor. +//! Slab byte allocator implementation. //! //! This module implements an improved slab allocator for small object allocation //! with pooled linked lists, inspired by asterinas design. @@ -10,11 +10,11 @@ use log::warn; use crate::{AllocError, AllocResult}; -// Re-export public types from sibling modules +use super::slab_cache::CacheDeallocDecision; pub use super::slab_cache::SlabCache; pub use super::slab_node::SlabNode; -/// Size classes for slab allocation +/// Size classes for slab allocation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(usize)] pub enum SizeClass { @@ -33,7 +33,6 @@ impl SizeClass { pub const COUNT: usize = 9; const MAX_OBJ_SIZE: usize = 2048; - /// Select size class from memory layout pub fn from_layout(layout: Layout) -> Option { let required_size = layout.size().max(layout.align()); @@ -82,37 +81,44 @@ impl SizeClass { } } - pub fn from_index(index: usize) -> Option { - match index { - 0 => Some(SizeClass::Bytes8), - 1 => Some(SizeClass::Bytes16), - 2 => Some(SizeClass::Bytes32), - 3 => Some(SizeClass::Bytes64), - 4 => Some(SizeClass::Bytes128), - 5 => Some(SizeClass::Bytes256), - 6 => Some(SizeClass::Bytes512), - 7 => Some(SizeClass::Bytes1024), - 8 => Some(SizeClass::Bytes2048), - _ => None, - } + pub fn page_count(&self, page_size: usize) -> usize { + SlabNode::new(0, *self).page_count(page_size) + } + + pub fn slab_bytes(&self, page_size: usize) -> usize { + self.page_count(page_size) * page_size } } -/// Page allocator trait for slab allocator -pub trait PageAllocatorForSlab { - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult; - fn dealloc_pages(&mut self, pos: usize, num_pages: usize); +pub enum SlabAllocDecision { + Allocated(NonNull, usize), + NeedsRefill { + size_class: SizeClass, + page_count: usize, + slab_bytes: usize, + }, } -/// Slab byte allocator with pooled linked lists +pub enum SlabDeallocDecision { + Done { + bytes_released: usize, + actually_deallocated: bool, + }, + ReleaseSlab { + slab_base: usize, + page_count: usize, + slab_bytes: usize, + actually_deallocated: bool, + }, +} + +/// Slab byte allocator with pooled linked lists. pub struct SlabByteAllocator { caches: [SlabCache; SizeClass::COUNT], - page_allocator: Option<*mut dyn PageAllocatorForSlab>, total_bytes: usize, allocated_bytes: usize, } -// SAFETY: SlabByteAllocator is used behind SpinNoIrq locks unsafe impl Send for SlabByteAllocator {} unsafe impl Sync for SlabByteAllocator {} @@ -130,57 +136,105 @@ impl SlabByteAllocator { SlabCache::new(SizeClass::Bytes1024), SlabCache::new(SizeClass::Bytes2048), ], - page_allocator: None, total_bytes: 0, allocated_bytes: 0, } } - /// Initialize the allocator - pub fn init(&mut self) {} - - pub fn set_page_allocator(&mut self, page_allocator: *mut dyn PageAllocatorForSlab) { - self.page_allocator = Some(page_allocator); - } - - pub fn alloc(&mut self, layout: Layout) -> AllocResult> { + pub fn alloc(&mut self, layout: Layout) -> AllocResult { let size_class = SizeClass::from_layout(layout).ok_or(AllocError::InvalidParam)?; - - let Some(page_allocator_ptr) = self.page_allocator else { - return Err(AllocError::NoMemory); - }; - - let page_allocator = unsafe { &mut *page_allocator_ptr }; let cache = &mut self.caches[size_class.to_index()]; - let (obj_addr, page_bytes) = cache.alloc_object(page_allocator, PAGE_SIZE)?; - self.allocated_bytes += layout.size().max(layout.align()); - self.total_bytes += page_bytes; + if let Some(obj_addr) = cache.alloc_object()? { + self.allocated_bytes += layout.size().max(layout.align()); + return Ok(SlabAllocDecision::Allocated( + unsafe { NonNull::new_unchecked(obj_addr as *mut u8) }, + 0, + )); + } - Ok(unsafe { NonNull::new_unchecked(obj_addr as *mut u8) }) + Ok(SlabAllocDecision::NeedsRefill { + size_class, + page_count: size_class.page_count(PAGE_SIZE), + slab_bytes: size_class.slab_bytes(PAGE_SIZE), + }) } - pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) { - let size_class = SizeClass::from_layout(layout).expect("Invalid layout"); - let obj_addr = ptr.as_ptr() as usize; + pub fn provide_slab( + &mut self, + size_class: SizeClass, + owner_cpu: usize, + slab_base: usize, + slab_bytes: usize, + ) -> AllocResult<()> { + self.caches[size_class.to_index()].provide_slab(slab_base, slab_bytes, owner_cpu)?; + self.total_bytes += slab_bytes; + Ok(()) + } - let Some(page_allocator_ptr) = self.page_allocator else { - return; + pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) -> SlabDeallocDecision { + let Some(size_class) = SizeClass::from_layout(layout) else { + warn!( + "Invalid layout during slab dealloc: size={}, align={}", + layout.size(), + layout.align() + ); + return SlabDeallocDecision::Done { + bytes_released: 0, + actually_deallocated: false, + }; }; - let page_allocator = unsafe { &mut *page_allocator_ptr }; - let cache = &mut self.caches[size_class.to_index()]; - - let (freed_bytes, actually_freed) = - cache.dealloc_object(obj_addr, page_allocator, PAGE_SIZE); + let decision = + self.caches[size_class.to_index()].dealloc_object(ptr.as_ptr() as usize, PAGE_SIZE); + + match decision { + CacheDeallocDecision::Done { + bytes_released, + actually_deallocated, + } => { + if actually_deallocated { + self.allocated_bytes = self + .allocated_bytes + .saturating_sub(layout.size().max(layout.align())); + } + SlabDeallocDecision::Done { + bytes_released, + actually_deallocated, + } + } + CacheDeallocDecision::ReleaseSlab { + slab_base, + page_count, + slab_bytes, + actually_deallocated, + } => { + if actually_deallocated { + self.allocated_bytes = self + .allocated_bytes + .saturating_sub(layout.size().max(layout.align())); + } + self.total_bytes = self.total_bytes.saturating_sub(slab_bytes); + SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + slab_bytes, + actually_deallocated, + } + } + } + } - // Only update allocated_bytes if this was not a double-free - if actually_freed { - self.allocated_bytes = self - .allocated_bytes - .saturating_sub(layout.size().max(layout.align())); + pub fn owner_cpu_of(&self, ptr: NonNull, layout: Layout) -> Option { + let size_class = SizeClass::from_layout(layout)?; + let slab_bytes = size_class.slab_bytes(PAGE_SIZE); + let slab_base = (ptr.as_ptr() as usize / slab_bytes) * slab_bytes; + let node = SlabNode::new(slab_base, size_class); + if node.is_valid_for_size_class() { + Some(node.owner_cpu()) + } else { + None } - self.total_bytes = self.total_bytes.saturating_sub(freed_bytes); } pub fn total_bytes(&self) -> usize { @@ -228,7 +282,6 @@ mod tests { #[test] fn test_size_class_boundaries() { - // Test all size class boundaries assert_eq!(SizeClass::Bytes8.size(), 8); assert_eq!(SizeClass::Bytes16.size(), 16); assert_eq!(SizeClass::Bytes32.size(), 32); @@ -242,7 +295,6 @@ mod tests { #[test] fn test_size_class_alignment_limits() { - // Alignment too large should return None assert_eq!( SizeClass::from_layout(Layout::from_size_align(64, 4096).unwrap()), None diff --git a/src/slab/slab_cache.rs b/src/slab/slab_cache.rs index 4cea017..3b61496 100644 --- a/src/slab/slab_cache.rs +++ b/src/slab/slab_cache.rs @@ -5,7 +5,7 @@ use log::{error, warn}; -use super::slab_byte_allocator::{PageAllocatorForSlab as BytePageAllocator, SizeClass}; +use super::slab_byte_allocator::SizeClass; use super::slab_node::SlabNode; use crate::{AllocError, AllocResult}; @@ -86,7 +86,21 @@ impl SlabIntrusiveList { } } -/// Slab cache for a specific size class +/// Internal deallocation result for a single size-class cache. +pub(crate) enum CacheDeallocDecision { + Done { + bytes_released: usize, + actually_deallocated: bool, + }, + ReleaseSlab { + slab_base: usize, + page_count: usize, + slab_bytes: usize, + actually_deallocated: bool, + }, +} + +/// Slab cache for a specific size class. pub struct SlabCache { size_class: SizeClass, empty: SlabIntrusiveList, @@ -104,14 +118,11 @@ impl SlabCache { } } - /// Allocate an object from this cache - /// Returns (object_addr, bytes_allocated_from_page_allocator) - pub fn alloc_object( - &mut self, - page_allocator: &mut dyn BytePageAllocator, - page_size: usize, - ) -> AllocResult<(usize, usize)> { - // 1. Try to allocate from partial list + /// Try to allocate an object from this cache. + /// + /// Returns `Ok(Some(addr))` if allocation succeeds, `Ok(None)` if the cache + /// needs the caller to provide a slab page, or an error if metadata is invalid. + pub fn alloc_object(&mut self) -> AllocResult> { if let Some(slab_base) = self.partial.back() { let mut node = SlabNode::new(slab_base, self.size_class); if !node.is_valid_for_size_class() { @@ -123,12 +134,13 @@ impl SlabCache { self.partial.remove(self.size_class, slab_base); self.full.push_back(self.size_class, slab_base); } - return Ok((obj_addr, 0)); + return Ok(Some(obj_addr)); } - panic!("Allocation from partial slab failed despite free_count > 0, bitmap inconsistency detected"); + panic!( + "Allocation from partial slab failed despite free_count > 0, bitmap inconsistency detected" + ); } - // 2. Try to allocate from empty list if let Some(slab_base) = self.empty.pop_back(self.size_class) { let mut node = SlabNode::new(slab_base, self.size_class); if !node.is_valid_for_size_class() { @@ -137,82 +149,38 @@ impl SlabCache { if let Some(obj_idx) = node.alloc_object() { let obj_addr = node.object_addr(obj_idx); self.partial.push_back(self.size_class, slab_base); - - let prealloc_bytes = self.preallocate_empty_slab(page_allocator, page_size); - return Ok((obj_addr, prealloc_bytes)); + return Ok(Some(obj_addr)); } - panic!("Allocation from empty slab failed despite all objects being free, bitmap inconsistency detected"); - } - - // 3. Allocate a new node from page allocator - let (obj_addr, bytes) = self.allocate_new_slab(page_allocator, page_size)?; - Ok((obj_addr, bytes)) - } - - /// Allocate a new slab from page allocator - /// Returns (object_addr, bytes_allocated_from_page_allocator) - fn allocate_new_slab( - &mut self, - page_allocator: &mut dyn BytePageAllocator, - page_size: usize, - ) -> AllocResult<(usize, usize)> { - let object_size = self.size_class.size(); - let bytes_needed = SlabNode::MAX_OBJECTS * object_size; - let page_count = bytes_needed.div_ceil(page_size); - let slab_bytes = page_count * page_size; - - let start_addr = page_allocator.alloc_pages(page_count, slab_bytes)?; - - let mut new_node = SlabNode::new(start_addr, self.size_class); - new_node.init_header(slab_bytes); - - if let Some(obj_idx) = new_node.alloc_object() { - let obj_addr = new_node.object_addr(obj_idx); - self.partial.push_back(self.size_class, start_addr); - - let prealloc_bytes = self.preallocate_empty_slab(page_allocator, page_size); - return Ok((obj_addr, slab_bytes + prealloc_bytes)); + panic!( + "Allocation from empty slab failed despite all objects being free, bitmap inconsistency detected" + ); } - // This should never happen - newly initialized slab must have at least one free object - panic!("Failed to allocate from newly initialized slab: bitmap inconsistency or corruption detected"); + Ok(None) } - /// Pre-allocate an empty slab for future allocations - /// Returns bytes allocated from page allocator (0 if already has empty nodes) - fn preallocate_empty_slab( + /// Insert a freshly provided slab page into this cache. + pub fn provide_slab( &mut self, - page_allocator: &mut dyn BytePageAllocator, - page_size: usize, - ) -> usize { - if self.empty.len() > 0 { - return 0; - } - - let object_size = self.size_class.size(); - let bytes_needed = SlabNode::MAX_OBJECTS * object_size; - let page_count = bytes_needed.div_ceil(page_size); - let slab_bytes = page_count * page_size; - - if let Ok(start_addr) = page_allocator.alloc_pages(page_count, slab_bytes) { - let mut new_node = SlabNode::new(start_addr, self.size_class); - new_node.init_header(slab_bytes); - self.empty.push_back(self.size_class, start_addr); - return slab_bytes; + slab_base: usize, + slab_bytes: usize, + owner_cpu: usize, + ) -> AllocResult<()> { + let mut node = SlabNode::new(slab_base, self.size_class); + node.init_header(slab_bytes, owner_cpu); + if node.free_count() == 0 { + return Err(AllocError::InvalidParam); } - - 0 + self.empty.push_back(self.size_class, slab_base); + Ok(()) } - /// Deallocate an object - /// Returns (bytes_freed_from_page_allocator, actually_deallocated) - /// actually_deallocated is false if this was a double-free - pub fn dealloc_object( + /// Deallocate an object from this cache. + pub(crate) fn dealloc_object( &mut self, obj_addr: usize, - page_allocator: &mut dyn BytePageAllocator, page_size: usize, - ) -> (usize, bool) { + ) -> CacheDeallocDecision { let object_size = self.size_class.size(); let bytes_needed = SlabNode::MAX_OBJECTS * object_size; let page_count = bytes_needed.div_ceil(page_size); @@ -221,56 +189,62 @@ impl SlabCache { let slab_base = align_down_any(obj_addr, slab_bytes); let mut node = SlabNode::new(slab_base, self.size_class); if !node.is_valid_for_size_class() { - // This can happen if the slab was already returned to the page allocator - // and the memory was reused, or if the pointer is completely invalid. - // For robustness, especially in double-free tests, we return false. warn!( "slab allocator: Invalid slab base {:#x} for size class {:?}", slab_base, self.size_class ); - warn!("this can happen if the slab was already returned to the page allocator and the memory was reused, - or if the pointer is completely invalid"); - return (0, false); + warn!( + "this can happen if the slab was already returned to the page allocator and the memory was reused, or if the pointer is completely invalid" + ); + return CacheDeallocDecision::Done { + bytes_released: 0, + actually_deallocated: false, + }; } let was_full = node.is_full(); - let (should_dealloc_slab, actually_freed) = + let (should_release_slab, actually_freed) = if let Some(obj_idx) = node.object_index_from_addr(obj_addr) { - // dealloc_object returns true if object was allocated, false if already free (double-free) let actually_freed = node.dealloc_object(obj_idx); (node.is_empty() && actually_freed, actually_freed) } else { error!("Invalid address {obj_addr:x} in slab at {slab_base:x}: not a valid object"); - return (0, true); // Not a double-free, just invalid address (treat as no-op) + return CacheDeallocDecision::Done { + bytes_released: 0, + actually_deallocated: true, + }; }; - // Only manipulate lists if this was not a double-free if actually_freed { - // Remove slab from its current list before moving or deallocating it if was_full { self.full.remove(self.size_class, slab_base); } else { self.partial.remove(self.size_class, slab_base); } - if should_dealloc_slab { - // Slab became empty - either deallocate or move to empty list + if should_release_slab { if self.empty.len() >= 2 { - page_allocator.dealloc_pages(slab_base, page_count); - return (slab_bytes, true); - } else { - self.empty.push_back(self.size_class, slab_base); - return (0, true); + return CacheDeallocDecision::ReleaseSlab { + slab_base, + page_count, + slab_bytes, + actually_deallocated: true, + }; } + self.empty.push_back(self.size_class, slab_base); + return CacheDeallocDecision::Done { + bytes_released: 0, + actually_deallocated: true, + }; } - // Slab still has objects - if it was full, it's now partial - if was_full { - self.partial.push_back(self.size_class, slab_base); - } + self.partial.push_back(self.size_class, slab_base); } - (0, actually_freed) + CacheDeallocDecision::Done { + bytes_released: 0, + actually_deallocated: actually_freed, + } } } @@ -280,7 +254,6 @@ mod tests { use alloc::alloc::{alloc, dealloc}; use core::alloc::Layout; - // Re-import SizeClass for tests use super::super::slab_byte_allocator::SizeClass; struct MockPageAllocator { @@ -293,9 +266,7 @@ mod tests { allocated: alloc::vec::Vec::new(), } } - } - impl BytePageAllocator for MockPageAllocator { fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { let size = num_pages * 4096; let layout = @@ -308,6 +279,7 @@ mod tests { Ok(addr) } + #[allow(dead_code)] fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { if let Some(idx) = self .allocated @@ -320,33 +292,43 @@ mod tests { } } + fn provide_cache_slab(cache: &mut SlabCache, page_allocator: &mut MockPageAllocator) { + let page_count = SlabNode::new(0, SizeClass::Bytes64).page_count(4096); + let slab_bytes = page_count * 4096; + let slab_base = page_allocator.alloc_pages(page_count, slab_bytes).unwrap(); + cache.provide_slab(slab_base, slab_bytes, 0).unwrap(); + } + #[test] fn test_alloc_dealloc() { let mut cache = SlabCache::new(SizeClass::Bytes64); let mut page_allocator = MockPageAllocator::new(); + provide_cache_slab(&mut cache, &mut page_allocator); - let (obj_addr, _) = cache.alloc_object(&mut page_allocator, 4096).unwrap(); - + let obj_addr = cache.alloc_object().unwrap().unwrap(); assert_ne!(obj_addr, 0); - cache.dealloc_object(obj_addr, &mut page_allocator, 4096); + match cache.dealloc_object(obj_addr, 4096) { + CacheDeallocDecision::Done { .. } | CacheDeallocDecision::ReleaseSlab { .. } => {} + } } #[test] fn test_multiple_allocs() { let mut cache = SlabCache::new(SizeClass::Bytes64); let mut page_allocator = MockPageAllocator::new(); + provide_cache_slab(&mut cache, &mut page_allocator); let mut addrs = alloc::vec::Vec::new(); for _ in 0..10 { - let (addr, _) = cache.alloc_object(&mut page_allocator, 4096).unwrap(); + let addr = cache.alloc_object().unwrap().unwrap(); addrs.push(addr); } assert_eq!(addrs.len(), 10); for addr in addrs { - cache.dealloc_object(addr, &mut page_allocator, 4096); + cache.dealloc_object(addr, 4096); } } @@ -354,15 +336,16 @@ mod tests { fn test_empty_node_management() { let mut cache = SlabCache::new(SizeClass::Bytes64); let mut page_allocator = MockPageAllocator::new(); + provide_cache_slab(&mut cache, &mut page_allocator); - let (addr1, _) = cache.alloc_object(&mut page_allocator, 4096).unwrap(); - cache.dealloc_object(addr1, &mut page_allocator, 4096); + let addr1 = cache.alloc_object().unwrap().unwrap(); + cache.dealloc_object(addr1, 4096); - let (addr2, _) = cache.alloc_object(&mut page_allocator, 4096).unwrap(); - cache.dealloc_object(addr2, &mut page_allocator, 4096); + let addr2 = cache.alloc_object().unwrap().unwrap(); + cache.dealloc_object(addr2, 4096); - let (addr3, _) = cache.alloc_object(&mut page_allocator, 4096).unwrap(); - cache.dealloc_object(addr3, &mut page_allocator, 4096); + let addr3 = cache.alloc_object().unwrap().unwrap(); + cache.dealloc_object(addr3, 4096); assert!(cache.empty.len() <= 2); } diff --git a/src/slab/slab_node.rs b/src/slab/slab_node.rs index 00282aa..49a0286 100644 --- a/src/slab/slab_node.rs +++ b/src/slab/slab_node.rs @@ -14,6 +14,7 @@ pub(crate) struct SlabHeader { object_count: u16, free_count: u16, _reserved: u16, + owner_cpu: u32, slab_bytes: usize, prev: usize, next: usize, @@ -52,7 +53,7 @@ impl SlabNode { unsafe { &mut *(self.addr as *mut SlabHeader) } } - pub fn init_header(&mut self, slab_bytes: usize) { + pub fn init_header(&mut self, slab_bytes: usize, owner_cpu: usize) { let object_size = self.size_class.size(); let header_size_aligned = self.header_size_aligned(); let object_count = if slab_bytes > header_size_aligned { @@ -94,6 +95,7 @@ impl SlabNode { object_count: object_count as u16, free_count: object_count as u16, _reserved: 0, + owner_cpu: owner_cpu as u32, slab_bytes, prev: 0, next: 0, @@ -198,6 +200,10 @@ impl SlabNode { bytes_needed.div_ceil(page_size) } + pub fn owner_cpu(&self) -> usize { + self.header().owner_cpu as usize + } + pub fn prev(&self) -> Option { let prev = self.header().prev; if prev == 0 { @@ -240,7 +246,7 @@ mod tests { let base = unsafe { alloc(layout) } as usize; assert_ne!(base, 0); node.addr = base; - node.init_header(slab_bytes); + node.init_header(slab_bytes, 0); assert!(node.is_empty()); assert!(!node.is_full()); @@ -274,7 +280,7 @@ mod tests { let base = unsafe { alloc(layout) } as usize; assert_ne!(base, 0); node.addr = base; - node.init_header(slab_bytes); + node.init_header(slab_bytes, 0); let obj0 = node.object_addr(0); assert_eq!(node.object_index_from_addr(obj0), Some(0)); diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index 2d4f24f..fc9ab5a 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -1,539 +1,185 @@ -//! Test for GlobalAllocator's alloc_dma32_pages method +//! Tests for GlobalAllocator's alloc_dma32_pages method. #![no_std] extern crate alloc; -use alloc::vec; -use alloc::vec::Vec; -use buddy_slab_allocator::{AddrTranslator, AllocError, GlobalAllocator}; +use buddy_slab_allocator::{AddrTranslator, AllocError, GlobalAllocator, Os}; use core::alloc::Layout; +use core::sync::atomic::{AtomicUsize, Ordering}; -const PAGE_SIZE: usize = 0x1000; // 4KB pages -const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16MB +const PAGE_SIZE: usize = 0x1000; +const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; -/// Mock address translator for testing -/// In a real hypervisor, this would translate virtual addresses to physical addresses struct MockAddrTranslator; impl AddrTranslator for MockAddrTranslator { fn virt_to_phys(&self, va: usize) -> Option { - // For testing purposes, we'll map virtual addresses to physical addresses in the low-memory region - // This ensures that our test memory is considered low-memory (<4GiB) - // We'll simply subtract a large value to get into the low-memory range - Some(va & 0x7fffffff) // Mask to get 31-bit address, which is below 2GiB + Some(va & 0x7fff_ffff) + } +} + +struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } +} + +impl Os for MockOs { + fn current_cpu_idx(&self) -> usize { + self.cpu.load(Ordering::Relaxed) } } -/// Static instance of the mock address translator static MOCK_TRANSLATOR: MockAddrTranslator = MockAddrTranslator; +static MOCK_OS: MockOs = MockOs::new(); -/// Allocate test memory using system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); +fn alloc_region(size: usize, align: usize) -> (*mut u8, Layout) { + let layout = Layout::from_size_align(size, align).unwrap(); let ptr = unsafe { alloc::alloc::alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); + assert!(!ptr.is_null(), "Failed to allocate region"); (ptr, layout) } -/// Deallocate test memory -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { +fn dealloc_region(ptr: *mut u8, layout: Layout) { unsafe { alloc::alloc::dealloc(ptr, layout) }; } -#[test] -fn test_alloc_dma32_pages_uninitialized() { - // Create a new allocator but don't initialize it - let mut allocator = GlobalAllocator::::new(); - - // Try to allocate pages - should fail because allocator is not initialized - let result = allocator.alloc_dma32_pages(1, PAGE_SIZE); - assert!( - matches!(result, Err(AllocError::NoMemory)), - "Expected NoMemory error when allocating from uninitialized allocator" - ); +fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { + alloc_region( + GlobalAllocator::::required_metadata_size(cpu_count), + PAGE_SIZE, + ) } -#[test] -fn test_alloc_dma32_pages_initialized() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - // Create allocator and set address translator - let mut allocator = GlobalAllocator::::new(); +fn init_allocator( + allocator: &mut GlobalAllocator, + heap_addr: usize, + heap_size: usize, +) -> (*mut u8, Layout) { + let (meta_ptr, meta_layout) = alloc_metadata(1); allocator.set_addr_translator(&MOCK_TRANSLATOR); + unsafe { + allocator + .init( + meta_ptr as usize, + meta_layout.size(), + heap_addr, + heap_size, + 1, + &MOCK_OS, + ) + .unwrap(); + } + (meta_ptr, meta_layout) +} - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Test 1: Allocate 1 page with page size alignment - let result1 = allocator.alloc_dma32_pages(1, PAGE_SIZE); - assert!(result1.is_ok(), "Failed to allocate 1 page: {:?}", result1); - let addr1 = result1.unwrap(); - assert!( - addr1 >= heap_addr, - "Allocated address is below memory start" - ); - assert!( - addr1 < heap_addr + TEST_HEAP_SIZE, - "Allocated address is beyond memory end" - ); - assert_eq!( - addr1 % PAGE_SIZE, - 0, - "Allocated address is not page-aligned" - ); - - // Test 2: Allocate multiple pages - let result2 = allocator.alloc_dma32_pages(4, PAGE_SIZE); - assert!(result2.is_ok(), "Failed to allocate 4 pages: {:?}", result2); - let addr2 = result2.unwrap(); - assert!( - addr2 >= heap_addr, - "Allocated address is below memory start" - ); - assert!( - addr2 < heap_addr + TEST_HEAP_SIZE, - "Allocated address is beyond memory end" - ); - assert_eq!( - addr2 % PAGE_SIZE, - 0, - "Allocated address is not page-aligned" - ); - - // Test 3: Allocate with different alignment - let result3 = allocator.alloc_dma32_pages(1, 2 * PAGE_SIZE); // 8KB alignment - assert!( - result3.is_ok(), - "Failed to allocate 1 page with 8KB alignment: {:?}", - result3 - ); - let addr3 = result3.unwrap(); - assert!( - addr3 >= heap_addr, - "Allocated address is below memory start" - ); - assert!( - addr3 < heap_addr + TEST_HEAP_SIZE, - "Allocated address is beyond memory end" - ); - assert_eq!( - addr3 % (2 * PAGE_SIZE), - 0, - "Allocated address is not 8KB-aligned" - ); - - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); +#[test] +fn test_alloc_dma32_pages_uninitialized() { + let allocator = GlobalAllocator::::new(); + let result = allocator.alloc_dma32_pages(1, PAGE_SIZE); + assert!(matches!(result, Err(AllocError::NoMemory))); } #[test] -fn test_alloc_dma32_pages_memory_structure() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn test_alloc_dma32_pages_initialized() { + let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; - // Create allocator and set address translator let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); - - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Test basic memory structure by allocating and deallocating - // This will exercise the internal memory management structures - - // Allocate some DMA32 pages - let alloc_results = vec![ - allocator.alloc_dma32_pages(1, PAGE_SIZE), - allocator.alloc_dma32_pages(2, PAGE_SIZE), - allocator.alloc_dma32_pages(4, PAGE_SIZE), - ]; - - // Verify all allocations succeeded - for (i, result) in alloc_results.iter().enumerate() { - assert!( - result.is_ok(), - "Failed to allocate DMA32 pages (iteration {}): {:?}", - i, - result - ); - } - - // Get allocated addresses - let alloc_addrs: Vec = alloc_results.into_iter().map(|r| r.unwrap()).collect(); - - // Verify all addresses are valid - for addr in &alloc_addrs { - assert!( - addr >= &heap_addr, - "Allocated address is below memory start" - ); - assert!( - addr < &(heap_addr + TEST_HEAP_SIZE), - "Allocated address is beyond memory end" - ); - } + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - // Test memory statistics if tracking feature is enabled - #[cfg(feature = "tracking")] - { - // Get statistics after allocations - let stats_after_alloc = allocator.get_stats(); - assert!( - stats_after_alloc.used_pages > 0, - "Used pages should be greater than 0 after allocations" - ); - - // Get buddy allocator statistics - let buddy_stats = allocator.get_buddy_stats(); - assert!( - buddy_stats.total_pages > 0, - "Buddy total pages should be greater than 0" - ); - } + let addr1 = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); + let addr2 = allocator.alloc_dma32_pages(4, PAGE_SIZE).unwrap(); + let addr3 = allocator.alloc_dma32_pages(1, 2 * PAGE_SIZE).unwrap(); - // Deallocate all pages - for (i, addr) in alloc_addrs.iter().enumerate() { - let num_pages = match i { - 0 => 1, - 1 => 2, - 2 => 4, - _ => 1, - }; - allocator.dealloc_pages(*addr, num_pages); - } + assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); + assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); + assert!(addr3 >= heap_addr && addr3 < heap_addr + TEST_HEAP_SIZE); + assert_eq!(addr1 % PAGE_SIZE, 0); + assert_eq!(addr2 % PAGE_SIZE, 0); + assert_eq!(addr3 % (2 * PAGE_SIZE), 0); - // Test memory statistics after deallocation if tracking feature is enabled - #[cfg(feature = "tracking")] - { - let stats_after_dealloc = allocator.get_stats(); - // Note: Used pages might not be exactly 0 due to internal node pool usage - // but should be significantly reduced - assert!( - stats_after_dealloc.used_pages < 100, - "Used pages should be low after deallocation" - ); - } + allocator.dealloc_pages(addr1, 1); + allocator.dealloc_pages(addr2, 4); + allocator.dealloc_pages(addr3, 1); - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] fn test_alloc_dma32_pages_memory_stats() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); + let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; - // Create allocator and set address translator let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Test memory statistics if tracking feature is enabled #[cfg(feature = "tracking")] { - // Get initial statistics let stats_before = allocator.get_stats(); - assert!( - stats_before.total_pages > 0, - "Total pages should be greater than 0" - ); - assert!( - stats_before.free_pages > 0, - "Free pages should be greater than 0" - ); - assert_eq!( - stats_before.used_pages, 0, - "Used pages should be 0 initially" - ); - - // Allocate some DMA32 pages - let result = allocator.alloc_dma32_pages(2, PAGE_SIZE); - assert!( - result.is_ok(), - "Failed to allocate DMA32 pages: {:?}", - result - ); - let addr = result.unwrap(); - - // Get statistics after allocation - let stats_after = allocator.get_stats(); - assert_eq!( - stats_after.used_pages, - stats_before.used_pages + 2, - "Used pages should increase by 2" - ); - assert_eq!( - stats_after.free_pages, - stats_before.free_pages - 2, - "Free pages should decrease by 2" - ); - - // Deallocate the pages - allocator.dealloc_pages(addr, 2); - - // Get statistics after deallocation - let stats_final = allocator.get_stats(); - assert_eq!( - stats_final.used_pages, stats_before.used_pages, - "Used pages should return to initial value" - ); - assert_eq!( - stats_final.free_pages, stats_before.free_pages, - "Free pages should return to initial value" - ); + assert!(stats_before.total_pages > 0); } - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); -} - -#[test] -fn test_alloc_dma32_pages_multiple_zones() { - // Allocate two separate memory regions for multiple zones - let (heap_ptr1, heap_layout1) = alloc_test_heap(TEST_HEAP_SIZE / 2); - let heap_addr1 = heap_ptr1 as usize; - - let (heap_ptr2, heap_layout2) = alloc_test_heap(TEST_HEAP_SIZE / 2); - let heap_addr2 = heap_ptr2 as usize; + let addr = allocator.alloc_dma32_pages(2, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 2); - // Create allocator and set address translator - let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + #[cfg(feature = "tracking")] + { + let stats_after = allocator.get_stats(); + assert!(stats_after.total_pages > 0); + } - // Initialize allocator with first memory region - let init_result = allocator.init(heap_addr1, TEST_HEAP_SIZE / 2); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Add second memory region as a new zone - let add_result = allocator.add_memory(heap_addr2, TEST_HEAP_SIZE / 2); - assert!( - add_result.is_ok(), - "Failed to add memory region: {:?}", - add_result - ); - - // Test allocating from multiple zones - // First allocation should come from the first zone - let result1 = allocator.alloc_dma32_pages(1, PAGE_SIZE); - assert!( - result1.is_ok(), - "Failed to allocate 1 page from multiple zones: {:?}", - result1 - ); - let addr1 = result1.unwrap(); - assert!( - addr1 >= heap_addr1 || addr1 >= heap_addr2, - "Allocated address is not in any zone" - ); - assert!( - (addr1 < heap_addr1 + TEST_HEAP_SIZE / 2) || (addr1 < heap_addr2 + TEST_HEAP_SIZE / 2), - "Allocated address is beyond memory end" - ); - - // Second allocation should come from either zone - let result2 = allocator.alloc_dma32_pages(2, PAGE_SIZE); - assert!( - result2.is_ok(), - "Failed to allocate 2 pages from multiple zones: {:?}", - result2 - ); - let addr2 = result2.unwrap(); - assert!( - addr2 >= heap_addr1 || addr2 >= heap_addr2, - "Allocated address is not in any zone" - ); - assert!( - (addr2 < heap_addr1 + TEST_HEAP_SIZE / 2) || (addr2 < heap_addr2 + TEST_HEAP_SIZE / 2), - "Allocated address is beyond memory end" - ); - - // Clean up - dealloc_test_heap(heap_ptr1, heap_layout1); - dealloc_test_heap(heap_ptr2, heap_layout2); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] fn test_alloc_dma32_pages_vs_normal_pages() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); + let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; - // Create allocator and set address translator let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Test 1: Allocate DMA32 pages (32-bit memory) - let result_dma32 = allocator.alloc_dma32_pages(1, PAGE_SIZE); - assert!( - result_dma32.is_ok(), - "Failed to allocate DMA32 pages: {:?}", - result_dma32 - ); - let addr_dma32 = result_dma32.unwrap(); - assert!( - addr_dma32 >= heap_addr, - "Allocated DMA32 address is below memory start" - ); - assert!( - addr_dma32 < heap_addr + TEST_HEAP_SIZE, - "Allocated DMA32 address is beyond memory end" - ); - - // Test 2: Allocate normal pages - let result_normal = allocator.alloc_pages(1, PAGE_SIZE); - assert!( - result_normal.is_ok(), - "Failed to allocate normal pages: {:?}", - result_normal - ); - let addr_normal = result_normal.unwrap(); - assert!( - addr_normal >= heap_addr, - "Allocated normal address is below memory start" - ); - assert!( - addr_normal < heap_addr + TEST_HEAP_SIZE, - "Allocated normal address is beyond memory end" - ); - - // Verify both addresses are valid and different - assert_ne!( - addr_dma32, addr_normal, - "DMA32 and normal pages should have different addresses" - ); - - // Test 3: Allocate multiple pages of each type - let result_dma32_multi = allocator.alloc_dma32_pages(4, PAGE_SIZE); - assert!( - result_dma32_multi.is_ok(), - "Failed to allocate multiple DMA32 pages: {:?}", - result_dma32_multi - ); - - let result_normal_multi = allocator.alloc_pages(4, PAGE_SIZE); - assert!( - result_normal_multi.is_ok(), - "Failed to allocate multiple normal pages: {:?}", - result_normal_multi - ); - - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); -} + let addr_dma32 = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); + let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); -#[test] -fn test_alloc_dma32_pages_edge_cases() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; + assert!(addr_dma32 >= heap_addr && addr_dma32 < heap_addr + TEST_HEAP_SIZE); + assert!(addr_normal >= heap_addr && addr_normal < heap_addr + TEST_HEAP_SIZE); - // Create allocator and set address translator - let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + allocator.dealloc_pages(addr_dma32, 1); + allocator.dealloc_pages(addr_normal, 1); - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Test: Allocate 0 pages - let result = allocator.alloc_dma32_pages(0, PAGE_SIZE); - // Note: The behavior for 0 pages may vary - some allocators return 0, others error - // This test assumes it might succeed (returning 0) or fail, but shouldn't panic - if let Ok(addr) = result { - assert_eq!(addr, 0, "Expected 0 for 0 pages allocation"); - } // Error is also acceptable for 0 pages - - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] fn test_alloc_dma32_pages_stress() { - // Allocate actual test memory - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); + let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; - // Create allocator and set address translator let mut allocator = GlobalAllocator::::new(); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - // Initialize allocator - let init_result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!( - init_result.is_ok(), - "Failed to initialize allocator: {:?}", - init_result - ); - - // Stress test: Allocate and free multiple times - let mut allocated_addrs = Vec::new(); - - // Allocate multiple times - for i in 0..10 { - let num_pages = (i % 4) + 1; // 1-4 pages - let alignment = if i % 2 == 0 { PAGE_SIZE } else { 2 * PAGE_SIZE }; - - let result = allocator.alloc_dma32_pages(num_pages, alignment); - assert!( - result.is_ok(), - "Failed to allocate {} pages with alignment {}: {:?}", - num_pages, - alignment, - result - ); - allocated_addrs.push(result.unwrap()); + let mut addrs = alloc::vec::Vec::new(); + for _ in 0..32 { + let addr = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); + addrs.push(addr); } - // Verify all addresses are valid - for addr in &allocated_addrs { - assert!( - addr >= &heap_addr, - "Allocated address is below memory start" - ); - assert!( - addr < &(heap_addr + TEST_HEAP_SIZE), - "Allocated address is beyond memory end" - ); + for addr in addrs { + allocator.dealloc_pages(addr, 1); } - // Clean up - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index b6d21ab..090b1ba 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,7 +1,4 @@ -//! Integration tests for the allocator crate -//! -//! Tests the complete allocator system working together, -//! focusing on cross-module integration scenarios. +//! Integration tests for the allocator crate. #![no_std] @@ -10,26 +7,154 @@ extern crate buddy_slab_allocator; use alloc::vec::Vec; use buddy_slab_allocator::{ - AllocError, CompositePageAllocator, GlobalAllocator, PageAllocatorForSlab, SlabByteAllocator, + AllocError, CompositePageAllocator, GlobalAllocator, Os, SlabAllocDecision, SlabByteAllocator, + SlabDeallocDecision, }; use core::alloc::Layout; +use core::sync::atomic::{AtomicUsize, Ordering}; const PAGE_SIZE: usize = 0x1000; -const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16MB +const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; -/// Allocate test memory using system allocator -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); +struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } + + fn set_cpu(&self, cpu: usize) { + self.cpu.store(cpu, Ordering::Relaxed); + } +} + +impl Os for MockOs { + fn current_cpu_idx(&self) -> usize { + self.cpu.load(Ordering::Relaxed) + } +} + +struct FixedOs; + +impl Os for FixedOs { + fn current_cpu_idx(&self) -> usize { + 0 + } +} + +static FIXED_OS: FixedOs = FixedOs; +static SWITCHING_OS: MockOs = MockOs::new(); + +fn alloc_region(size: usize, align: usize) -> (*mut u8, Layout) { + let layout = Layout::from_size_align(size, align).unwrap(); let ptr = unsafe { alloc::alloc::alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test heap"); + assert!(!ptr.is_null(), "Failed to allocate test region"); (ptr, layout) } -/// Deallocate test memory -fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { +fn dealloc_region(ptr: *mut u8, layout: Layout) { unsafe { alloc::alloc::dealloc(ptr, layout) }; } +fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { + alloc_region(size, PAGE_SIZE) +} + +fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { + alloc_region( + GlobalAllocator::::required_metadata_size(cpu_count), + PAGE_SIZE, + ) +} + +fn init_global_allocator( + allocator: &mut GlobalAllocator, + heap_addr: usize, + heap_size: usize, + cpu_count: usize, +) -> (*mut u8, Layout) { + let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); + unsafe { + allocator + .init( + meta_ptr as usize, + meta_layout.size(), + heap_addr, + heap_size, + cpu_count, + &FIXED_OS, + ) + .unwrap(); + } + (meta_ptr, meta_layout) +} + +fn init_global_allocator_with_os( + allocator: &mut GlobalAllocator, + heap_addr: usize, + heap_size: usize, + cpu_count: usize, + os: &'static dyn Os, +) -> (*mut u8, Layout) { + let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); + unsafe { + allocator + .init( + meta_ptr as usize, + meta_layout.size(), + heap_addr, + heap_size, + cpu_count, + os, + ) + .unwrap(); + } + (meta_ptr, meta_layout) +} + +fn slab_alloc_with_refill( + slab_allocator: &mut SlabByteAllocator, + page_allocator: &mut CompositePageAllocator, + owner_cpu: usize, + layout: Layout, +) -> core::ptr::NonNull { + loop { + match slab_allocator.alloc(layout).unwrap() { + SlabAllocDecision::Allocated(ptr, _) => return ptr, + SlabAllocDecision::NeedsRefill { + size_class, + page_count, + slab_bytes, + } => { + let slab_base = page_allocator.alloc_pages(page_count, slab_bytes).unwrap(); + slab_allocator + .provide_slab(size_class, owner_cpu, slab_base, slab_bytes) + .unwrap(); + } + } + } +} + +fn slab_dealloc_with_release( + slab_allocator: &mut SlabByteAllocator, + page_allocator: &mut CompositePageAllocator, + ptr: core::ptr::NonNull, + layout: Layout, +) { + if let SlabDeallocDecision::ReleaseSlab { + slab_base, + page_count, + .. + } = slab_allocator.dealloc(ptr, layout) + { + page_allocator.dealloc_pages(slab_base, page_count); + } +} + #[test] fn test_composite_page_allocator_basic() { let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); @@ -38,20 +163,16 @@ fn test_composite_page_allocator_basic() { let mut allocator = CompositePageAllocator::::new(); allocator.init(heap_addr, TEST_HEAP_SIZE); - // Just verify allocator can perform basic operations - // Allocate 1 page let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); - - // Allocate 4 pages let addr2 = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); + + assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); - // Deallocate allocator.dealloc_pages(addr1, 1); allocator.dealloc_pages(addr2, 4); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -62,39 +183,19 @@ fn test_composite_page_allocator_alignment() { let mut allocator = CompositePageAllocator::::new(); allocator.init(heap_addr, TEST_HEAP_SIZE); - // Test different alignments let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - assert_eq!(addr1 & (PAGE_SIZE - 1), 0); - let addr2 = allocator.alloc_pages(1, PAGE_SIZE * 2).unwrap(); - assert_eq!(addr2 & (PAGE_SIZE * 2 - 1), 0); - let addr3 = allocator.alloc_pages(1, PAGE_SIZE * 4).unwrap(); + + assert_eq!(addr1 & (PAGE_SIZE - 1), 0); + assert_eq!(addr2 & (PAGE_SIZE * 2 - 1), 0); assert_eq!(addr3 & (PAGE_SIZE * 4 - 1), 0); allocator.dealloc_pages(addr1, 1); allocator.dealloc_pages(addr2, 1); allocator.dealloc_pages(addr3, 1); - dealloc_test_heap(heap_ptr, heap_layout); -} - -#[test] -fn test_composite_page_allocator_large_allocation() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - // Allocate large block - let large_pages = 1024; // 4MB - let addr = allocator.alloc_pages(large_pages, PAGE_SIZE).unwrap(); - assert!(addr >= heap_addr && addr < heap_addr + TEST_HEAP_SIZE); - - allocator.dealloc_pages(addr, large_pages); - - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -105,23 +206,19 @@ fn test_composite_page_allocator_fragmentation() { let mut allocator = CompositePageAllocator::::new(); allocator.init(heap_addr, TEST_HEAP_SIZE); - // Create fragmentation pattern let mut addrs = Vec::new(); for _ in 0..10 { let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); addrs.push((addr, 1)); } - // Free every other allocation for i in (0..addrs.len()).step_by(2) { allocator.dealloc_pages(addrs[i].0, addrs[i].1); } - // Try to allocate larger block let result = allocator.alloc_pages(5, PAGE_SIZE); assert!(result.is_ok()); - // Cleanup for i in (1..addrs.len()).step_by(2) { allocator.dealloc_pages(addrs[i].0, addrs[i].1); } @@ -129,7 +226,7 @@ fn test_composite_page_allocator_fragmentation() { allocator.dealloc_pages(addr, 5); } - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -141,28 +238,26 @@ fn test_slab_allocator_basic() { page_allocator.init(heap_addr, TEST_HEAP_SIZE); let mut slab_allocator = SlabByteAllocator::::new(); - slab_allocator.init(); - let page_alloc_ptr = &mut page_allocator as *mut CompositePageAllocator - as *mut dyn PageAllocatorForSlab; - slab_allocator.set_page_allocator(page_alloc_ptr); - - // Test various small allocations let layout8 = Layout::from_size_align(8, 8).unwrap(); - let ptr8 = slab_allocator.alloc(layout8).unwrap(); + let ptr8 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout8); let layout64 = Layout::from_size_align(64, 8).unwrap(); - let ptr64 = slab_allocator.alloc(layout64).unwrap(); + let ptr64 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout64); let layout2048 = Layout::from_size_align(2048, 8).unwrap(); - let ptr2048 = slab_allocator.alloc(layout2048).unwrap(); - - // Deallocate - slab_allocator.dealloc(ptr8, layout8); - slab_allocator.dealloc(ptr64, layout64); - slab_allocator.dealloc(ptr2048, layout2048); - - dealloc_test_heap(heap_ptr, heap_layout); + let ptr2048 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout2048); + + slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr8, layout8); + slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr64, layout64); + slab_dealloc_with_release( + &mut slab_allocator, + &mut page_allocator, + ptr2048, + layout2048, + ); + + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -172,31 +267,23 @@ fn test_slab_allocator_many_objects() { let mut page_allocator = CompositePageAllocator::::new(); page_allocator.init(heap_addr, TEST_HEAP_SIZE); - let mut slab_allocator = SlabByteAllocator::::new(); - slab_allocator.init(); - - let page_alloc_ptr = &mut page_allocator as *mut CompositePageAllocator - as *mut dyn PageAllocatorForSlab; - slab_allocator.set_page_allocator(page_alloc_ptr); - // Allocate many small objects let mut ptrs = Vec::new(); let layout = Layout::from_size_align(32, 8).unwrap(); for _ in 0..100 { - let ptr = slab_allocator.alloc(layout).unwrap(); + let ptr = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout); ptrs.push(ptr); } assert_eq!(ptrs.len(), 100); - // Deallocate all for ptr in ptrs { - slab_allocator.dealloc(ptr, layout); + slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr, layout); } - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -205,17 +292,14 @@ fn test_global_allocator_init() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - let result = allocator.init(heap_addr, TEST_HEAP_SIZE); - assert!(result.is_ok()); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - // Just verify the allocator can perform allocations after init - let addr = allocator.alloc_pages(1, PAGE_SIZE); - assert!(addr.is_ok()); - if let Ok(a) = addr { - allocator.dealloc_pages(a, 1); - } + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 1); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -224,15 +308,15 @@ fn test_global_allocator_small_alloc() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - // Small allocations should go through slab let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -241,15 +325,15 @@ fn test_global_allocator_large_alloc() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - // Large allocations should go through page allocator let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -258,26 +342,25 @@ fn test_global_allocator_mixed_alloc() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); + let sizes = [8, 64, 1024, 4096, 8192]; let mut allocations = Vec::new(); - // Mix of small and large allocations - for i in 0..20 { - let size = if i % 2 == 0 { 64 } else { 8192 }; - let layout = Layout::from_size_align(size, 8).unwrap(); + for &size in &sizes { + let align = if size <= 2048 { 8 } else { PAGE_SIZE }; + let layout = Layout::from_size_align(size, align).unwrap(); let ptr = allocator.alloc(layout).unwrap(); allocations.push((ptr, layout)); } - assert_eq!(allocations.len(), 20); - - // Deallocate all for (ptr, layout) in allocations { allocator.dealloc(ptr, layout); } - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -286,42 +369,37 @@ fn test_global_allocator_page_alloc() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - // Direct page allocation let addr1 = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); - assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); - let addr2 = allocator.alloc_pages(8, PAGE_SIZE).unwrap(); - assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); allocator.dealloc_pages(addr1, 4); allocator.dealloc_pages(addr2, 8); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] fn test_global_allocator_add_memory() { let (heap_ptr1, heap_layout1) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr1 = heap_ptr1 as usize; - let (heap_ptr2, heap_layout2) = alloc_test_heap(TEST_HEAP_SIZE); + let heap_addr1 = heap_ptr1 as usize; let heap_addr2 = heap_ptr2 as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr1, TEST_HEAP_SIZE).unwrap(); - - let _total_before = allocator.total_pages(); - - // Try to add more memory (may fail if max zones reached) - let _result = allocator.add_memory(heap_addr2, TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr1, TEST_HEAP_SIZE, 1); - // Note: add_memory may fail if we've reached maximum zones - // We just verify the operation doesn't crash + allocator.add_memory(heap_addr2, TEST_HEAP_SIZE).unwrap(); + let addr = allocator.alloc_pages(16, PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 16); - dealloc_test_heap(heap_ptr1, heap_layout1); - dealloc_test_heap(heap_ptr2, heap_layout2); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr1, heap_layout1); + dealloc_region(heap_ptr2, heap_layout2); } #[test] @@ -329,19 +407,23 @@ fn test_error_conditions() { let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); let heap_addr = heap_ptr as usize; - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - // Test invalid parameter - let result = allocator.alloc_pages(0, PAGE_SIZE); - assert!(matches!(result, Err(AllocError::InvalidParam))); - - // Test allocation too large - let huge_pages = TEST_HEAP_SIZE / PAGE_SIZE + 1000; - let result = allocator.alloc_pages(huge_pages, PAGE_SIZE); - assert!(matches!(result, Err(AllocError::NoMemory))); - - dealloc_test_heap(heap_ptr, heap_layout); + let mut allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); + + assert!(matches!( + allocator.alloc_pages(0, PAGE_SIZE), + Err(AllocError::InvalidParam) + )); + + let huge_pages = TEST_HEAP_SIZE / PAGE_SIZE + 1; + assert!(matches!( + allocator.alloc_pages(huge_pages, PAGE_SIZE), + Err(AllocError::NoMemory) + )); + + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[cfg(feature = "tracking")] @@ -351,22 +433,20 @@ fn test_statistics_tracking() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); let stats_initial = allocator.get_stats(); - // Note: total_pages excludes the node pool pages (10 pages reserved for Buddy allocator) - let expected_pages = TEST_HEAP_SIZE / PAGE_SIZE - 10; - assert_eq!(stats_initial.total_pages, expected_pages); - assert_eq!(stats_initial.used_pages, 0); + assert!(stats_initial.total_pages > 0); - // Allocate some memory let layout = Layout::from_size_align(64, 8).unwrap(); let _ptr = allocator.alloc(layout).unwrap(); let stats_after = allocator.get_stats(); assert!(stats_after.slab_bytes > 0); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[cfg(feature = "tracking")] @@ -376,13 +456,16 @@ fn test_buddy_statistics() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = + init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); let buddy_stats = allocator.get_buddy_stats(); assert!(buddy_stats.total_pages > 0); assert_eq!(buddy_stats.used_pages, 0); + assert_eq!(buddy_stats.free_pages, buddy_stats.total_pages); - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } #[test] @@ -391,36 +474,58 @@ fn test_stress_allocation_deallocation() { let heap_addr = heap_ptr as usize; let mut allocator = GlobalAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE).unwrap(); - - // Stress test with many allocations - for _round in 0..5 { - let mut allocations = Vec::new(); - - for i in 0..50 { - let size = match i % 5 { - 0 => 8, - 1 => 32, - 2 => 128, - 3 => 512, - _ => 2048, - }; - let layout = Layout::from_size_align(size, 8).unwrap(); - if let Ok(ptr) = allocator.alloc(layout) { - allocations.push((ptr, layout)); - } - } + let (meta_ptr, meta_layout) = init_global_allocator_with_os( + &mut allocator, + heap_addr, + TEST_HEAP_SIZE, + 2, + &SWITCHING_OS, + ); - // Deallocate in reverse order - while let Some((ptr, layout)) = allocations.pop() { - allocator.dealloc(ptr, layout); + let mut allocations = Vec::new(); + for i in 0..100 { + SWITCHING_OS.set_cpu(i % 2); + let size = if i % 2 == 0 { 64 } else { 4096 }; + let align = if size <= 2048 { 8 } else { PAGE_SIZE }; + let layout = Layout::from_size_align(size, align).unwrap(); + if let Ok(ptr) = allocator.alloc(layout) { + allocations.push((ptr, layout)); } + } - // Check that we can still allocate after each round - let test_layout = Layout::from_size_align(64, 8).unwrap(); - let ptr = allocator.alloc(test_layout).unwrap(); - allocator.dealloc(ptr, test_layout); + while let Some((ptr, layout)) = allocations.pop() { + allocator.dealloc(ptr, layout); } - dealloc_test_heap(heap_ptr, heap_layout); + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); +} + +#[test] +fn test_global_allocator_cross_cpu_free() { + let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); + let heap_addr = heap_ptr as usize; + + let mut allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global_allocator_with_os( + &mut allocator, + heap_addr, + TEST_HEAP_SIZE, + 2, + &SWITCHING_OS, + ); + + let layout = Layout::from_size_align(64, 8).unwrap(); + SWITCHING_OS.set_cpu(0); + let ptr = allocator.alloc(layout).unwrap(); + + SWITCHING_OS.set_cpu(1); + allocator.dealloc(ptr, layout); + + SWITCHING_OS.set_cpu(0); + let ptr2 = allocator.alloc(layout).unwrap(); + allocator.dealloc(ptr2, layout); + + dealloc_region(meta_ptr, meta_layout); + dealloc_region(heap_ptr, heap_layout); } diff --git a/tests/stress_test.rs b/tests/stress_test.rs index d668c0e..b00e38d 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -1,12 +1,37 @@ //! Ignored stress tests for allocator stability and long-running behavior. -use buddy_slab_allocator::GlobalAllocator; +use buddy_slab_allocator::{GlobalAllocator, Os}; use rand::{RngExt, SeedableRng}; use std::alloc::{alloc, dealloc, Layout}; +use std::sync::atomic::{AtomicUsize, Ordering}; const PAGE_SIZE: usize = 0x1000; const HEAP_SIZE: usize = 64 * 1024 * 1024; +struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } + + fn set_cpu(&self, cpu: usize) { + self.cpu.store(cpu, Ordering::Relaxed); + } +} + +impl Os for MockOs { + fn current_cpu_idx(&self) -> usize { + self.cpu.load(Ordering::Relaxed) + } +} + +static MOCK_OS: MockOs = MockOs::new(); + struct TestHeap { ptr: *mut u8, layout: Layout, @@ -31,16 +56,47 @@ impl Drop for TestHeap { } } +fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { + let size = GlobalAllocator::::required_metadata_size(cpu_count); + let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate metadata"); + (ptr, layout) +} + +fn init_allocator( + allocator: &mut GlobalAllocator, + heap: &TestHeap, + cpu_count: usize, +) -> (*mut u8, Layout) { + let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); + MOCK_OS.set_cpu(0); + unsafe { + allocator + .init( + meta_ptr as usize, + meta_layout.size(), + heap.addr(), + HEAP_SIZE, + cpu_count, + &MOCK_OS, + ) + .unwrap(); + } + (meta_ptr, meta_layout) +} + #[test] #[ignore = "stress test"] fn stress_random_mixed_alloc_free() { let heap = TestHeap::new(HEAP_SIZE); let mut allocator = GlobalAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); let mut rng = rand::rngs::StdRng::from_seed([0; 32]); let mut allocated = Vec::new(); - for _ in 0..10_000 { + for i in 0..10_000 { + MOCK_OS.set_cpu(i % 2); if allocated.is_empty() || rng.random_bool(0.65) { let size: usize = rng.random_range(8..8193); let layout = if size <= 2048 { @@ -63,6 +119,8 @@ fn stress_random_mixed_alloc_free() { for (ptr, layout) in allocated { allocator.dealloc(ptr, layout); } + + unsafe { dealloc(meta_ptr, meta_layout) }; } #[test] @@ -70,7 +128,7 @@ fn stress_random_mixed_alloc_free() { fn stress_exhaustion_recovery() { let heap = TestHeap::new(HEAP_SIZE); let mut allocator = GlobalAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); let mut allocated = Vec::new(); @@ -78,20 +136,12 @@ fn stress_exhaustion_recovery() { allocated.push(ptr); } - assert!( - !allocated.is_empty(), - "expected to exhaust heap after allocations" - ); - for ptr in allocated.drain(..allocated.len() / 4) { allocator.dealloc(ptr, layout); } let recovered = allocator.alloc(layout); - assert!( - recovered.is_ok(), - "allocator should recover after freeing pages" - ); + assert!(recovered.is_ok()); if let Ok(ptr) = recovered { allocator.dealloc(ptr, layout); @@ -100,6 +150,8 @@ fn stress_exhaustion_recovery() { for ptr in allocated { allocator.dealloc(ptr, layout); } + + unsafe { dealloc(meta_ptr, meta_layout) }; } #[test] @@ -107,11 +159,12 @@ fn stress_exhaustion_recovery() { fn stress_fragmentation_recovery() { let heap = TestHeap::new(HEAP_SIZE); let mut allocator = GlobalAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); let small_layout = Layout::from_size_align(64, 8).unwrap(); let mut small_ptrs = Vec::new(); - for _ in 0..4000 { + for i in 0..4000 { + MOCK_OS.set_cpu(i % 2); if let Ok(ptr) = allocator.alloc(small_layout) { small_ptrs.push(ptr); } @@ -131,6 +184,8 @@ fn stress_fragmentation_recovery() { if let Ok(ptr) = large { allocator.dealloc(ptr, large_layout); } + + unsafe { dealloc(meta_ptr, meta_layout) }; } #[cfg(feature = "tracking")] @@ -139,11 +194,12 @@ fn stress_fragmentation_recovery() { fn stress_tracking_invariants() { let heap = TestHeap::new(HEAP_SIZE); let mut allocator = GlobalAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE).unwrap(); + let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); let mut rng = rand::rngs::StdRng::from_seed([1; 32]); let mut allocated = Vec::new(); - for _ in 0..5000 { + for i in 0..5000 { + MOCK_OS.set_cpu(i % 2); if allocated.is_empty() || rng.random_bool(0.6) { let size: usize = rng.random_range(8..4097); let layout = if size <= 2048 { @@ -171,4 +227,6 @@ fn stress_tracking_invariants() { assert!(stats.used_pages <= stats.total_pages); assert!(stats.free_pages <= stats.total_pages); assert_eq!(stats.used_pages + stats.free_pages, stats.total_pages); + + unsafe { dealloc(meta_ptr, meta_layout) }; } From 28640310d5c39042c246653d568874eb57adb5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Wed, 8 Apr 2026 14:56:53 +0800 Subject: [PATCH 07/19] Refactor integration and stress tests for buddy-slab-allocator - Updated integration tests to use BuddyAllocator and SlabAllocator. - Replaced MockOs with TestOs for better clarity in test context. - Simplified memory allocation and deallocation functions. - Enhanced test cases for buddy allocator, including basic allocation, alignment, exhaustion recovery, and fragmentation recovery. - Improved stress tests to ensure allocator stability and recovery from exhaustion. - Removed unused code and comments for better readability. --- Cargo.toml | 31 +- README.md | 274 +++++++++++- src/buddy/buddy_allocator.rs | 556 ------------------------ src/buddy/buddy_block.rs | 56 --- src/buddy/buddy_set.rs | 442 ------------------- src/buddy/global_node_pool.rs | 316 -------------- src/buddy/mod.rs | 407 +++++++++++++++++- src/buddy/page_meta.rs | 125 ++++++ src/buddy/pooled_list.rs | 522 ----------------------- src/buddy/stats.rs | 129 ------ src/error.rs | 17 +- src/global.rs | 319 ++++++++++++++ src/global_allocator.rs | 554 ------------------------ src/lib.rs | 192 ++------- src/os.rs | 24 -- src/page_allocator.rs | 528 ----------------------- src/slab/cache.rs | 207 +++++++++ src/slab/mod.rs | 113 ++++- src/slab/page.rs | 226 ++++++++++ src/slab/size_class.rs | 95 +++++ src/slab/slab_byte_allocator.rs | 303 ------------- src/slab/slab_cache.rs | 352 ---------------- src/slab/slab_node.rs | 305 -------------- tests/dma32_pages_test.rs | 172 +++----- tests/integration_test.rs | 723 +++++++++++++++----------------- tests/stress_test.rs | 98 ++--- 26 files changed, 2211 insertions(+), 4875 deletions(-) delete mode 100644 src/buddy/buddy_allocator.rs delete mode 100644 src/buddy/buddy_block.rs delete mode 100644 src/buddy/buddy_set.rs delete mode 100644 src/buddy/global_node_pool.rs create mode 100644 src/buddy/page_meta.rs delete mode 100644 src/buddy/pooled_list.rs delete mode 100644 src/buddy/stats.rs create mode 100644 src/global.rs delete mode 100644 src/global_allocator.rs delete mode 100644 src/os.rs delete mode 100644 src/page_allocator.rs create mode 100644 src/slab/cache.rs create mode 100644 src/slab/page.rs create mode 100644 src/slab/size_class.rs delete mode 100644 src/slab/slab_byte_allocator.rs delete mode 100644 src/slab/slab_cache.rs delete mode 100644 src/slab/slab_node.rs diff --git a/Cargo.toml b/Cargo.toml index 3f062d0..e0745d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,37 +1,22 @@ [package] -name = "buddy-slab-allocator" -version = "0.2.0" -edition = "2021" authors = ["Song Zhiyong "] +categories = ["memory-management", "no-std", "embedded"] description = "Memory allocator with Buddy and Slab allocation" documentation = "https://docs.rs/buddy-slab-allocator" -repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator" -readme = "README.md" -license = "Apache-2.0" +edition = "2021" keywords = ["buddy", "slab", "allocator"] -categories = ["memory-management", "no-std", "embedded"] +license = "Apache-2.0" +name = "buddy-slab-allocator" +readme = "README.md" +repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator" +version = "0.2.0" [features] default = [] -tracking = [] [dependencies] -cfg-if = "1.0" log = "0.4" spin = "0.10" [dev-dependencies] -criterion = { version = "0.8", features = ["html_reports"] } -rand = { version = "0.10", features = ["std_rng"] } - -[[bench]] -name = "global_allocator" -harness = false - -[[bench]] -name = "buddy_allocator" -harness = false - -[[bench]] -name = "slab_allocator" -harness = false +rand = {version = "0.10", features = ["std_rng"]} diff --git a/README.md b/README.md index 86500a1..76bde82 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,223 @@ # buddy-slab-allocator -A `no_std` buddy + slab allocator for kernel and embedded environments. +A `no_std` buddy + slab two-level memory allocator designed for kernel and embedded environments. + +## Architecture Overview + +The allocator employs a classic **two-level** architecture: + +1. **Buddy page allocator** — manages physical pages, the shared backend +2. **Slab byte allocator** — manages small objects (≤ 2048 bytes), the frontend +3. **GlobalAllocator** — a multi-core facade that routes allocations to per-CPU slab caches or the buddy backend + +```mermaid +classDiagram + direction TB + + class GlobalAllocator~PAGE_SIZE~ { + -Mutex~CompositePageAllocator~ buddy + -Option~NonNull~PerCpuSlabSlot~~ slab_slots_ptr + -usize cpu_count + -MetadataRegionInfo metadata_region + -&'static dyn Os os + -UsageStatsAtomic stats + -AtomicBool initialized + +new() GlobalAllocator + +init(meta_start, meta_size, heap_start, heap_size, cpu_count, os) AllocResult + +add_memory_region(start, size) AllocResult + +alloc(Layout) Option~NonNull~u8~~ + +dealloc(ptr, Layout) + +usage_stats() UsageStats + } + + class Os { + <> + +current_cpu_idx() usize + } + + class PerCpuSlabSlot~PAGE_SIZE~ { + -Mutex~SlabByteAllocator~ slab + } + + class CompositePageAllocator~PAGE_SIZE~ { + -BuddyPageAllocator~PAGE_SIZE~ buddy + -CompositeBlockTracker composite_tracker + +new() CompositePageAllocator + +init(start, size) + +alloc_pages(count, align) AllocResult~usize~ + +dealloc_pages(addr, count) + } + + class CompositeBlockTracker { + -blocks: [Option~CompositeBlockInfo~; 64] + -count: usize + } + + class CompositeBlockInfo { + +base_addr: usize + +part_count: u8 + +parts: [(usize, u32); 8] + } + + class BuddyPageAllocator~PAGE_SIZE~ { + -zones: [BuddySet~PAGE_SIZE~; 4] + -num_zones: usize + -global_node_pool: GlobalNodePool + -BuddyStats stats + -Option~&'static dyn AddrTranslator~ addr_translator + +alloc_pages(count, align) AllocResult~usize~ + +dealloc_pages(addr, count) + +add_memory_region(start, size) AllocResult + } + + class AddrTranslator { + <> + +virt_to_phys(va: usize) Option~usize~ + } + + class BuddySet~PAGE_SIZE~ { + -base_addr: usize + -end_addr: usize + -total_pages: usize + -zone_id: usize + -is_lowmem: bool + -free_lists: [PooledLinkedList; 11] + } + + class GlobalNodePool { + -free_head: Option~usize~ + -total_nodes: usize + -free_nodes: usize + } + + class PooledLinkedList { + -head: Option~usize~ + -tail: Option~usize~ + -len: usize + +insert_sorted(addr, order) + +pop_front() Option~BuddyBlock~ + +find_by_addr(addr) Option~BuddyBlock~ + +remove(addr) + } + + class BuddyBlock { + +order: usize + +addr: usize + } + + class SlabByteAllocator~PAGE_SIZE~ { + -caches: [SlabCache; 9] + -total_bytes: usize + -allocated_bytes: usize + +alloc(Layout) SlabAllocDecision + +dealloc(ptr, Layout) SlabDeallocDecision + +provide_slab(size_class, owner_cpu, slab_base, slab_bytes) + } + + class SizeClass { + <> + Bytes8 Bytes16 Bytes32 Bytes64 + Bytes128 Bytes256 Bytes512 + Bytes1024 Bytes2048 + } + + class SlabCache { + -partial: SlabIntrusiveList + -empty: SlabIntrusiveList + -full: SlabIntrusiveList + +alloc_object() Option~usize~ + +provide_slab(addr, size_class, owner_cpu, slab_bytes) + +dealloc_object(ptr) SlabDeallocDecision + } + + class SlabIntrusiveList { + -head: Option~usize~ + -tail: Option~usize~ + -len: usize + } + + class SlabNode { + +addr: usize + +size_class: SizeClass + +alloc_object() Option~usize~ + +dealloc_object(ptr) bool + } + + class SlabHeader { + +magic: u32 + +size_class: u16 + +object_count: u16 + +free_count: u16 + +owner_cpu: u32 + +slab_bytes: usize + +prev: usize + +next: usize + +free_bitmap: [u64; 8] + } + + class AllocError { + <> + InvalidParam + MemoryOverlap + NoMemory + NotAllocated + } + + %% Relationships + + GlobalAllocator --> Os : uses + GlobalAllocator *-- PerCpuSlabSlot : cpu_count slots + GlobalAllocator *-- CompositePageAllocator : Mutex wrapped + GlobalAllocator ..> UsageStats : returns + + CompositePageAllocator *-- BuddyPageAllocator + CompositePageAllocator *-- CompositeBlockTracker + CompositeBlockTracker *-- CompositeBlockInfo : tracks + + BuddyPageAllocator *-- BuddySet : up to 4 zones + BuddyPageAllocator *-- GlobalNodePool : shared + BuddyPageAllocator ..> AddrTranslator : optional + BuddySet *-- PooledLinkedList : 11 free lists + PooledLinkedList ..> GlobalNodePool : allocates nodes from + PooledLinkedList ..> BuddyBlock : manages + + PerCpuSlabSlot *-- SlabByteAllocator : Mutex wrapped + SlabByteAllocator *-- SlabCache : 9 size classes + SlabByteAllocator ..> SizeClass : dispatches by + SlabCache *-- SlabIntrusiveList : 3 lists + SlabIntrusiveList ..> SlabNode : manages + SlabNode *-- SlabHeader : embedded in memory +``` + +### Allocation Flow + +```mermaid +flowchart TD + A["GlobalAllocator::alloc(layout)"] --> B{size ≤ 2048B?} + B -- Yes --> C["Route to per-CPU SlabByteAllocator"] + C --> D{Slab has free object?} + D -- Yes --> E["Return object from SlabCache"] + D -- No --> F["Request pages from CompositePageAllocator"] + F --> G["Provide new slab to SlabCache"] + G --> E + B -- No --> H["Allocate pages from CompositePageAllocator"] + H --> I{Contiguous pages available?} + I -- Yes --> J["Return contiguous pages"] + I -- No --> K["Combine multiple buddy blocks (composite)"] + K --> J +``` ## Features -- Buddy page allocator as the shared page backend -- Slab byte allocator as the small-object frontend -- Multi-core `GlobalAllocator` facade with per-CPU slab fronts -- Global OS provider for CPU-local routing -- Optional `tracking` statistics -- Built-in `log` support +- **Buddy page allocator** — power-of-2 page allocation with splitting and merging +- **Slab byte allocator** — 9 fixed size classes (8B ~ 2048B) for small objects +- **Composite page allocation** — combines non-contiguous buddy blocks for large requests +- **Multi-core support** — per-CPU slab caches with `GlobalAllocator` facade +- **Multi-zone** — up to 4 memory zones with low-memory (DMA32) preference +- **Runtime memory addition** — add memory regions after initialization +- **`no_std`** — suitable for kernel and embedded environments +- **Optional `tracking`** — compile-time statistics gathering +- **Built-in `log`** — structured logging for debugging ## Quick Start @@ -17,12 +225,16 @@ A `no_std` buddy + slab allocator for kernel and embedded environments. ```toml [dependencies] -buddy-slab-allocator = "0.1.0" -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } +buddy-slab-allocator = "0.2.0" + +# With statistics tracking +buddy-slab-allocator = { version = "0.2.0", features = ["tracking"] } ``` ### Using `GlobalAllocator` +The `GlobalAllocator` is the recommended interface. It automatically routes small allocations to per-CPU slab caches and large allocations to the buddy page allocator. + ```rust use buddy_slab_allocator::{GlobalAllocator, Os}; use core::alloc::Layout; @@ -31,9 +243,7 @@ const PAGE_SIZE: usize = 0x1000; struct DemoOs; impl Os for DemoOs { - fn current_cpu_idx(&self) -> usize { - 0 - } + fn current_cpu_idx(&self) -> usize { 0 } } static OS: DemoOs = DemoOs; @@ -58,6 +268,8 @@ allocator.dealloc(ptr, layout); ### Using Buddy and Slab Separately +For lower-level control, use the components directly: + ```rust use buddy_slab_allocator::{ CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, @@ -97,18 +309,46 @@ if let SlabDeallocDecision::ReleaseSlab { } ``` -## Testing +## API Reference + +### Core Types + +- **`GlobalAllocator`** — Top-level facade combining page and slab allocation +- **`CompositePageAllocator`** — Page allocator with composite block support +- **`BuddyPageAllocator`** — Multi-zone buddy page allocator +- **`SlabByteAllocator`** — Slab allocator managing 9 size classes + +### Traits + +- **`Os`** — Provides CPU index for per-CPU slab routing +- **`AddrTranslator`** — Virtual-to-physical address translation for DMA32 + +### Allocation Decisions + +The slab allocator returns decisions rather than directly allocating pages, allowing the caller to manage page allocation: + +- **`SlabAllocDecision::Allocated(ptr, size)`** — object allocated successfully +- **`SlabAllocDecision::NeedsRefill { size_class, page_count, slab_bytes }`** — slab needs new pages + +- **`SlabDeallocDecision::Done`** — object freed within the slab +- **`SlabDeallocDecision::ReleaseSlab { slab_base, page_count, .. }`** — slab became empty, pages can be released + +## Testing & Benchmarking ```bash +# Run tests cargo test cargo test --features tracking -``` - -## Benchmarking -```bash +# Check benchmarks compile cargo check --benches + +# Run benchmarks cargo bench ``` Detailed benchmark notes are in `benches/README_CN.md`. + +## License + +Licensed under [Apache-2.0](LICENSE). diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs deleted file mode 100644 index d099a37..0000000 --- a/src/buddy/buddy_allocator.rs +++ /dev/null @@ -1,556 +0,0 @@ -//! Multi-zone buddy allocator using global node pool -//! -//! Provides buddy allocator with support for multiple memory zones and -//! a single shared global node pool for all zones and orders. - -use crate::{AllocError, AllocResult}; - -use log::{debug, error, info, warn}; - -#[cfg(feature = "tracking")] -use super::ZoneInfo; - -use super::{buddy_block::MAX_ZONES, buddy_set::BuddySet, global_node_pool::GlobalNodePool}; - -const NODE_POOL_PAGES: usize = 10; -const NODE_POOL_LOW_WATER_NODES: usize = 128; -const NODE_POOL_EXPAND_PAGES: usize = 5; -/// Threshold for low-memory (DMA32-like) physical addresses: 4GiB. -const LOWMEM_PHYS_THRESHOLD: usize = 1usize << 32; - -#[cfg(feature = "tracking")] -use super::stats::{BuddyStats, MemoryStatsReporter}; - -/// Buddy page allocator with multi-zone support and global node pool -/// -/// The `global_node_pool` stores linked-list nodes (ListNode), -/// which are used to construct the free lists in each BuddySet. -/// Memory pages themselves are tracked by BuddyBlock values, not by this pool. -pub struct BuddyPageAllocator { - zones: [BuddySet; MAX_ZONES], - num_zones: usize, - global_node_pool: GlobalNodePool, - #[cfg(feature = "tracking")] - stats: BuddyStats, - /// Optional address translator used to reason about physical addresses. - addr_translator: Option<&'static dyn crate::AddrTranslator>, -} - -impl BuddyPageAllocator { - pub const fn new() -> Self { - Self { - zones: [const { BuddySet::::empty() }; MAX_ZONES], - num_zones: 0, - global_node_pool: GlobalNodePool::new(), - #[cfg(feature = "tracking")] - stats: BuddyStats::new(), - addr_translator: None, - } - } - - /// Set the address translator so that the allocator can reason about - /// physical address ranges (e.g. low-memory regions below 4GiB). - pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) { - self.addr_translator = Some(translator); - } - - /// Initialize the global node pool and bootstrap with initial memory region - pub fn init(&mut self, base_addr: usize, size: usize) { - self.bootstrap(base_addr, size); - } - - /// Bootstrap allocator with initial memory region - pub fn bootstrap(&mut self, base_addr: usize, size: usize) { - if self.num_zones >= MAX_ZONES { - panic!("Cannot bootstrap: maximum zones reached"); - } - - // Reserve a small region at the beginning for the global node pool - let node_region_size = NODE_POOL_PAGES * PAGE_SIZE; - if size <= node_region_size { - panic!("Cannot bootstrap: region too small for node pool"); - } - let node_region_start = base_addr; - - self.global_node_pool - .init(node_region_start, node_region_size); - - let zone_base = base_addr + node_region_size; - let zone_size = size - node_region_size; - - self.zones[0] = BuddySet::new(zone_base, zone_size, 0); - self.zones[0].init(&mut self.global_node_pool, zone_base, zone_size); - // Mark whether this zone contains any low (<4G) physical memory, if translator is set. - if let Some(translator) = self.addr_translator { - if let Some(pa_start) = translator.virt_to_phys(zone_base) { - if pa_start < LOWMEM_PHYS_THRESHOLD { - self.zones[0].is_lowmem = true; - } - } - } - self.num_zones = 1; - - #[cfg(feature = "tracking")] - self.update_stats(); - } - - #[cfg(feature = "tracking")] - pub fn get_stats(&self) -> BuddyStats { - self.stats - } - - /// Get global node pool statistics - pub fn get_node_pool_stats(&self) -> super::global_node_pool::GlobalPoolStats { - self.global_node_pool.get_stats() - } - - /// Get number of zones in the allocator - pub fn get_zone_count(&self) -> usize { - self.num_zones - } - - /// Allocate contiguous low-memory pages (physical address < 4GiB). - /// - /// This API is intended for DMA32-like use cases. It only considers zones - /// that are marked as `is_lowmem`, and after allocation it performs a - /// strict physical boundary check to ensure that both the start and end - /// physical addresses are below the 4GiB threshold. - pub fn alloc_pages_lowmem(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - if num_pages == 0 { - return Err(AllocError::InvalidParam); - } - - let translator = self.addr_translator.ok_or(AllocError::InvalidParam)?; - - // Try to expand node pool if we are close to exhaustion - self.maybe_expand_node_pool(); - - let size_bytes = num_pages * PAGE_SIZE; - - for i in 0..self.num_zones { - if !self.zones[i].is_lowmem { - continue; - } - - match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) { - Ok(addr) => { - let start_va = addr; - let end_va = addr + size_bytes - 1; - - // Ensure both start and end physical addresses are below the - // low-memory threshold to avoid crossing the 4GiB boundary. - if let (Some(pa_start), Some(pa_end)) = ( - translator.virt_to_phys(start_va), - translator.virt_to_phys(end_va), - ) { - if pa_start < LOWMEM_PHYS_THRESHOLD && pa_end < LOWMEM_PHYS_THRESHOLD { - #[cfg(feature = "tracking")] - self.update_stats(); - return Ok(addr); - } - } - - // Boundary check failed: roll back this allocation and - // continue searching for another suitable block. - self.zones[i].dealloc_pages(&mut self.global_node_pool, addr, num_pages); - } - Err(_) => { - continue; - } - } - } - - info!( - "buddy allocator: Low-memory allocation failure: {} Byte, align {}", - num_pages * PAGE_SIZE, - alignment - ); - Err(AllocError::NoMemory) - } - - /// Get free blocks of a specific order from a zone - /// Returns None if zone doesn't exist - pub fn get_free_blocks_by_order( - &self, - zone_id: usize, - order: u32, - ) -> Option> { - if zone_id >= self.num_zones { - return None; - } - Some(self.zones[zone_id].get_free_blocks_by_order(&self.global_node_pool, order)) - } - - /// Update aggregated statistics from all zones - #[cfg(feature = "tracking")] - fn update_stats(&mut self) { - let mut total_stats = BuddyStats::new(); - - for i in 0..self.num_zones { - let zone_stats = self.zones[i].get_stats(); - total_stats.add(&zone_stats); - } - - self.stats = total_stats; - } - - /// Ensure the global node pool has enough free nodes. - /// - /// When the number of free nodes falls below a low-water mark, - /// try to reserve a few pages from any available zone and add them as a new node region. - fn maybe_expand_node_pool(&mut self) { - let free_nodes = self.global_node_pool.free_node_count(); - if free_nodes >= NODE_POOL_LOW_WATER_NODES { - return; - } - if self.num_zones == 0 { - error!("buddy allocator: no zones available to expand node pool"); - return; - } - - let expand_pages = NODE_POOL_EXPAND_PAGES; - let expand_size = expand_pages * PAGE_SIZE; - - // Try all zones to allocate memory for node pool expansion - for i in 0..self.num_zones { - match self.zones[i].alloc_pages(&mut self.global_node_pool, expand_pages, PAGE_SIZE) { - Ok(addr) => { - debug!( - "buddy allocator: expanding node pool from zone {}: free_nodes={} region=[{:#x}, {:#x})", - i, - free_nodes, - addr, - addr + expand_size - ); - self.global_node_pool.add_region(addr, expand_size); - return; - } - Err(_) => { - continue; - } - } - } - - warn!( - "buddy allocator: failed to expand node pool at low water: free_nodes={} tried {} zones", - free_nodes, - self.num_zones - ); - } - - /// Add a new memory region as a new zone - pub fn add_memory_region(&mut self, start: usize, size: usize) -> AllocResult<()> { - if self.num_zones >= MAX_ZONES { - error!("buddy allocator: Cannot add region: maximum zones ({MAX_ZONES}) reached"); - return Err(AllocError::NoMemory); - } - - // Align to page boundaries - let aligned_start = start & !(PAGE_SIZE - 1); - let end = start + size; - let aligned_end = (end + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); - let aligned_size = aligned_end - aligned_start; - - if aligned_size == 0 || aligned_size < PAGE_SIZE { - warn!("buddy allocator: Aligned size is too small: {aligned_size:#x}, skipping region"); - return Err(AllocError::InvalidParam); - } - - // Check for overlap with existing zones - for i in 0..self.num_zones { - let zone = &self.zones[i]; - if !(aligned_end <= zone.base_addr || aligned_start >= zone.end_addr) { - error!( - "buddy allocator: Region [{:#x}, {:#x}) overlaps with zone {} [{:#x}, {:#x})", - aligned_start, aligned_end, i, zone.base_addr, zone.end_addr - ); - return Err(AllocError::MemoryOverlap); - } - } - - let zone_id = self.num_zones; - self.zones[zone_id] = BuddySet::new(aligned_start, aligned_size, zone_id); - self.zones[zone_id].init(&mut self.global_node_pool, aligned_start, aligned_size); - // Mark whether this zone contains any low (<4G) physical memory, if translator is set. - if let Some(translator) = self.addr_translator { - if let Some(pa_start) = translator.virt_to_phys(aligned_start) { - if pa_start < LOWMEM_PHYS_THRESHOLD { - self.zones[zone_id].is_lowmem = true; - } - } - } - self.num_zones += 1; - - // Print all zone information after successfully adding a new memory region - #[cfg(feature = "tracking")] - self.print_zone_info(); - - Ok(()) - } - - pub fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { - self.add_memory_region(start, size)?; - #[cfg(feature = "tracking")] - self.update_stats(); - Ok(()) - } - - /// Find the zone that contains the given address - pub fn find_zone_for_addr(&self, addr: usize) -> Option { - (0..self.num_zones).find(|&i| self.zones[i].addr_in_zone(addr)) - } - - /// Print detailed allocation failure statistics - /// - /// This method is public to allow CompositePageAllocator to call it - /// when allocation fails, providing detailed per-zone failure information. - #[cfg(feature = "tracking")] - pub fn print_alloc_failure_stats(&self, num_pages: usize, alignment: usize) { - let mut zone_infos: [Option; MAX_ZONES] = [None; MAX_ZONES]; - let mut zone_stats: [Option; MAX_ZONES] = [None; MAX_ZONES]; - - for i in 0..self.num_zones { - zone_infos[i] = Some(self.zones[i].zone_info()); - zone_stats[i] = Some(self.zones[i].get_stats()); - } - - // Create slices from the initialized elements - let zone_infos_slice: &[ZoneInfo] = unsafe { - core::slice::from_raw_parts(zone_infos.as_ptr() as *const ZoneInfo, self.num_zones) - }; - let zone_stats_slice: &[BuddyStats] = unsafe { - core::slice::from_raw_parts(zone_stats.as_ptr() as *const BuddyStats, self.num_zones) - }; - - MemoryStatsReporter::print_alloc_failure_stats( - PAGE_SIZE, - self.num_zones, - &self.stats, - zone_infos_slice, - zone_stats_slice, - num_pages, - alignment, - ); - } - - /// Print all zone information and block distribution - pub fn print_zone_info(&self) { - info!("========== Buddy Allocator Zones Info =========="); - info!("Total zones: {}", self.num_zones); - info!("Page size: {PAGE_SIZE:#x} ({PAGE_SIZE})"); - info!(""); - - for i in 0..self.num_zones { - let zone = &self.zones[i]; - let _zone_info = zone.zone_info(); - info!("Zone {i}:"); - info!( - " Address range: [{:#x}, {:#x})", - _zone_info.start_addr, _zone_info.end_addr - ); - info!(" Total pages: {}", _zone_info.total_pages); - info!( - " Total size: {:#x} ({} MB)", - _zone_info.total_pages * PAGE_SIZE, - (_zone_info.total_pages * PAGE_SIZE) / (1024 * 1024) - ); - info!(" Free blocks distribution:"); - - // Print block distribution for each order - for order in 0..=zone.max_order() { - let block_count = zone.get_order_block_count(order); - if block_count > 0 { - let _block_size = (1 << order) * PAGE_SIZE; - info!( - " Order {}: {} blocks (size {} bytes each, total {:#x})", - order, - block_count, - _block_size, - block_count * _block_size - ); - } - } - info!(""); - } - - info!("Global node pool stats:"); - let _pool_stats = self.global_node_pool.get_stats(); - info!(" Total allocations: {}", _pool_stats.total_allocations); - info!(" Free nodes: {}", _pool_stats.free_nodes); - info!("=============================================="); - } - - #[cfg(not(feature = "tracking"))] - pub fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) { - // No-op when tracking is disabled - } - - pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - // Try to expand node pool if we are close to exhaustion - self.maybe_expand_node_pool(); - - // First, prefer zones that are NOT marked as low-memory so that - // low-memory regions can be reserved for special use (e.g. DMA32). - for i in 0..self.num_zones { - if self.zones[i].is_lowmem { - continue; - } - match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) { - Ok(addr) => { - #[cfg(feature = "tracking")] - self.update_stats(); - return Ok(addr); - } - Err(_) => { - continue; - } - } - } - - // Then, fall back to zones that are marked as low-memory. - for i in 0..self.num_zones { - if !self.zones[i].is_lowmem { - continue; - } - match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) { - Ok(addr) => { - #[cfg(feature = "tracking")] - self.update_stats(); - return Ok(addr); - } - Err(_) => { - continue; - } - } - } - - debug!( - "buddy allocator: Allocation failure: {} Byte, align {}", - num_pages * PAGE_SIZE, - alignment - ); - Err(AllocError::NoMemory) - } - - pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - self.maybe_expand_node_pool(); - if let Some(zone_idx) = self.find_zone_for_addr(pos) { - self.zones[zone_idx].dealloc_pages(&mut self.global_node_pool, pos, num_pages); - #[cfg(feature = "tracking")] - self.update_stats(); - } else { - warn!("buddy allocator: Dealloc pages at {pos:#x}: address not in any zone"); - } - } - - pub fn total_pages(&self) -> usize { - #[cfg(feature = "tracking")] - return self.stats.total_pages; - #[cfg(not(feature = "tracking"))] - return 0; - } - - pub fn used_pages(&self) -> usize { - #[cfg(feature = "tracking")] - return self.stats.used_pages; - #[cfg(not(feature = "tracking"))] - return 0; - } - - pub fn available_pages(&self) -> usize { - #[cfg(feature = "tracking")] - return self.stats.free_pages; - #[cfg(not(feature = "tracking"))] - return 0; - } -} - -impl Default for BuddyPageAllocator { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::alloc::{alloc, dealloc}; - use core::alloc::Layout; - - const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; - const TEST_PAGE_SIZE: usize = 0x1000; - - fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, TEST_PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null()); - (ptr, layout) - } - - fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; - } - - #[test] - fn test_buddy_allocator_init() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr = allocator.alloc_pages(1, TEST_PAGE_SIZE); - assert!(addr.is_ok()); - if let Ok(a) = addr { - allocator.dealloc_pages(a, 1); - } - - dealloc_test_heap(heap_ptr, heap_layout); - } - - #[test] - fn test_buddy_allocator_multi_pages() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr1 = allocator.alloc_pages(4, TEST_PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_pages(8, TEST_PAGE_SIZE).unwrap(); - assert_ne!(addr1, addr2); - - allocator.dealloc_pages(addr1, 4); - allocator.dealloc_pages(addr2, 8); - - dealloc_test_heap(heap_ptr, heap_layout); - } - - #[test] - fn test_buddy_allocator_alignment() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr = allocator.alloc_pages(1, TEST_PAGE_SIZE * 4).unwrap(); - assert_eq!(addr & (TEST_PAGE_SIZE * 4 - 1), 0); - - allocator.dealloc_pages(addr, 1); - dealloc_test_heap(heap_ptr, heap_layout); - } - - #[test] - fn test_buddy_allocator_zone_count() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - assert_eq!(allocator.get_zone_count(), 1); - - dealloc_test_heap(heap_ptr, heap_layout); - } -} diff --git a/src/buddy/buddy_block.rs b/src/buddy/buddy_block.rs deleted file mode 100644 index 297475d..0000000 --- a/src/buddy/buddy_block.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Buddy block metadata -//! -//! Represents a block of memory in the buddy system with order and address information. - -use core::cmp::PartialOrd; - -/// Maximum number of memory zones supported -pub const MAX_ZONES: usize = 32; - -/// Maximum order supported -pub const DEFAULT_MAX_ORDER: usize = 28; // Support up to 256GB allocations (2^28 * 4KB) - -/// Buddy block metadata -#[derive(Debug, Clone, Copy)] -pub struct BuddyBlock { - pub order: usize, - pub addr: usize, -} - -impl BuddyBlock { - /// Create a new buddy block - pub const fn new(order: usize, addr: usize) -> Self { - Self { order, addr } - } - - /// Calculate the buddy address for this block - /// The buddy is the other half of the parent block at the next higher order - /// For a block at order k with address A, its buddy is at A ^ (2^k * PAGE_SIZE) - #[allow(dead_code)] - pub fn buddy_addr(&self) -> usize { - self.addr ^ ((1 << self.order) * PAGE_SIZE) - } -} - -impl PartialOrd for BuddyBlock { - fn partial_cmp(&self, other: &Self) -> Option { - self.addr.partial_cmp(&other.addr) - } -} - -impl PartialEq for BuddyBlock { - fn eq(&self, other: &Self) -> bool { - self.addr == other.addr && self.order == other.order - } -} - -impl Eq for BuddyBlock {} - -/// A memory zone descriptor (similar to Linux kernel's zone) -#[derive(Debug, Clone, Copy)] -pub struct ZoneInfo { - pub start_addr: usize, - pub end_addr: usize, - pub total_pages: usize, - pub zone_id: usize, -} diff --git a/src/buddy/buddy_set.rs b/src/buddy/buddy_set.rs deleted file mode 100644 index 5018bbd..0000000 --- a/src/buddy/buddy_set.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! Single-zone buddy allocator using global node pool -//! -//! Implements the core buddy system for a single memory zone using -//! pooled linked lists that draw nodes from a shared global pool. - -use crate::{AllocError, AllocResult}; - -use log::{error, warn}; - -use super::{ - buddy_block::{BuddyBlock, ZoneInfo, DEFAULT_MAX_ORDER}, - global_node_pool::GlobalNodePool, - pooled_list::PooledLinkedList, -}; - -/// A buddy set implementation - represents a single zone -/// -/// Uses pooled linked lists with global node pool for efficient memory usage. -/// All zones share the same global node pool. -pub struct BuddySet { - pub(crate) base_addr: usize, - pub(crate) end_addr: usize, - total_pages: usize, - zone_id: usize, - /// Whether this zone contains any memory that can map to low (<4G) physical addresses. - pub(crate) is_lowmem: bool, - /// Free lists for each order - free_lists: [PooledLinkedList; DEFAULT_MAX_ORDER + 1], -} - -impl BuddySet { - /// Create a new buddy set for a zone (uninitialized, must call init()) - pub const fn new(base_addr: usize, size: usize, zone_id: usize) -> Self { - Self { - base_addr, - end_addr: base_addr + size, - total_pages: size / PAGE_SIZE, - zone_id, - is_lowmem: false, - free_lists: [const { PooledLinkedList::new() }; DEFAULT_MAX_ORDER + 1], - } - } - - /// Create an empty buddy set - pub const fn empty() -> Self { - Self::new(0, 0, 0) - } - - pub const fn max_order(&self) -> usize { - DEFAULT_MAX_ORDER - } - - /// Add a block to the appropriate list for its order - fn add_block_to_order( - &mut self, - pool: &mut GlobalNodePool, - order: usize, - block: BuddyBlock, - ) -> bool { - if order > DEFAULT_MAX_ORDER { - error!( - "zone {}: Order {} exceeds maximum order {}", - self.zone_id, order, DEFAULT_MAX_ORDER - ); - return false; - } - - self.free_lists[order].insert_sorted(pool, block) - } - - /// Find a block with the given address in the free list for its order - fn find_block_in_order( - &self, - pool: &GlobalNodePool, - order: usize, - addr: usize, - ) -> Option<(usize, Option)> { - if order > DEFAULT_MAX_ORDER { - return None; - } - self.free_lists[order].find_by_addr(pool, addr) - } - - /// Remove a block from its list - #[allow(dead_code)] - fn remove_block_from_order( - &mut self, - pool: &mut GlobalNodePool, - order: usize, - node_idx: usize, - ) -> bool { - if order > DEFAULT_MAX_ORDER { - error!( - "zone {}: Order {} exceeds maximum order {}", - self.zone_id, order, DEFAULT_MAX_ORDER - ); - return false; - } - self.free_lists[order].remove(pool, node_idx) - } - - /// Check if an address belongs to this zone - pub fn addr_in_zone(&self, addr: usize) -> bool { - addr >= self.base_addr && addr < self.end_addr - } - - /// Get zone information - pub fn zone_info(&self) -> ZoneInfo { - ZoneInfo { - start_addr: self.base_addr, - end_addr: self.end_addr, - total_pages: self.total_pages, - zone_id: self.zone_id, - } - } - - /// Initialize the buddy set with a memory region - pub fn init(&mut self, pool: &mut GlobalNodePool, base_addr: usize, size: usize) { - let aligned_base = base_addr & !(PAGE_SIZE - 1); - let end = base_addr + size; - let aligned_end = (end + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); - let aligned_size = aligned_end - aligned_base; - - if aligned_size == 0 || aligned_size < PAGE_SIZE { - panic!("Aligned size is too small: {:#x}", aligned_size); - } - - self.base_addr = aligned_base; - self.end_addr = aligned_end; - self.total_pages = aligned_size / PAGE_SIZE; - - // Reset free lists - for list in &mut self.free_lists { - list.clear(pool); - } - - self.init_free_blocks(pool); - } - - fn init_free_blocks(&mut self, pool: &mut GlobalNodePool) { - let mut remaining_pages = self.total_pages; - let mut current_addr = self.base_addr; - - while remaining_pages > 0 { - // Find the maximum order constrained by: - // 1. Remaining pages (can't allocate more than what's left) - // 2. Address alignment (block address must be aligned to block_size) - // 3. Maximum supported order - - // Maximum order based on remaining pages - let max_order_by_pages = if remaining_pages.is_power_of_two() { - remaining_pages.trailing_zeros() as usize - } else { - // For non-power-of-2, use the highest bit position - (remaining_pages.next_power_of_two() >> 1).trailing_zeros() as usize - }; - - // Maximum order based on address alignment - // For each order, check if current_addr is aligned to (2^order * PAGE_SIZE) - let mut max_order_by_alignment = 0; - for test_order in 0..=self.max_order() { - let block_size = (1 << test_order) * PAGE_SIZE; - if current_addr.is_multiple_of(block_size) { - max_order_by_alignment = test_order; - } else { - break; - } - } - - // Take the minimum of all constraints - let order = max_order_by_pages - .min(max_order_by_alignment) - .min(self.max_order()); - - let block_pages = 1 << order; - let block_size = block_pages * PAGE_SIZE; - - // Verify alignment: address must be exact multiple of block_size - assert!( - current_addr & (block_size - 1) == 0, - "Block address {current_addr:#x} not aligned to block size {block_size:#x} (order {order})" - ); - - // Construct and add the block - let block = BuddyBlock { - order, - addr: current_addr, - }; - - if !self.add_block_to_order(pool, order, block) { - error!( - "zone {}: Failed to add block during fast init: addr={:#x}, order={}, remaining_pages={}", - self.zone_id, current_addr, order, remaining_pages - ); - // Critical failure during initialization - panic!("Failed to initialize buddy system"); - } - - current_addr += block_size; - remaining_pages -= block_pages; - } - } - - /// Allocate pages using buddy system - pub fn alloc_pages( - &mut self, - pool: &mut GlobalNodePool, - num_pages: usize, - alignment: usize, - ) -> AllocResult { - if num_pages == 0 { - return Err(AllocError::InvalidParam); - } - - // Find the required order (round up to next power of 2) - let required_order = if num_pages.is_power_of_two() { - num_pages.trailing_zeros() as usize - } else { - num_pages.next_power_of_two().trailing_zeros() as usize - }; - - if required_order > self.max_order() { - error!( - "required order: {}, max order: {}", - required_order, - self.max_order() - ); - return Err(AllocError::NoMemory); - } - - let align_pages = alignment.div_ceil(PAGE_SIZE); - let align_order = align_pages.trailing_zeros() as usize; - let order_needed = required_order.max(align_order); - - // Try to find a block of the required order or higher - for order in order_needed..=self.max_order() { - if !self.free_lists[order].is_empty() { - let mut block = self.free_lists[order].pop_front(pool).unwrap(); - - // Split down to required order - while block.order > order_needed { - block.order -= 1; - let split_size = (1 << block.order) * PAGE_SIZE; - let buddy_addr = block.addr + split_size; - - // Push the second half back to free list (sorted!) - let success = self.add_block_to_order( - pool, - block.order, - BuddyBlock { - order: block.order, - addr: buddy_addr, - }, - ); - if !success { - warn!( - "Failed to push buddy block to free list during split at order {}", - block.order - ); - // Put the original block back - self.add_block_to_order(pool, block.order + 1, block); - return Err(AllocError::NoMemory); - } - } - - // Verify alignment requirement - assert!( - block.addr.is_multiple_of(alignment), - "Allocated address {:#x} is not aligned to {:#x} bytes ", - block.addr, - alignment - ); - - return Ok(block.addr); - } - } - - Err(AllocError::NoMemory) - } - - /// Deallocate pages back to buddy system with automatic merging - pub fn dealloc_pages(&mut self, pool: &mut GlobalNodePool, addr: usize, num_pages: usize) { - if num_pages == 0 { - warn!("zone {}: Trying to deallocate 0 pages", self.zone_id); - return; - } - - // Validate address belongs to this zone - if !self.addr_in_zone(addr) { - error!( - "zone {}: Address {:#x} not in zone [{:#x}, {:#x})", - self.zone_id, addr, self.base_addr, self.end_addr - ); - return; - } - - // Calculate order for this deallocation - // Handle non-power-of-2 allocations by rounding up (same as alloc_pages) - let mut order = if num_pages.is_power_of_two() { - num_pages.trailing_zeros() as usize - } else { - num_pages.next_power_of_two().trailing_zeros() as usize - }; - - if order > DEFAULT_MAX_ORDER { - error!( - "zone {}: Order {} exceeds maximum supported order {}", - self.zone_id, order, DEFAULT_MAX_ORDER - ); - return; - } - - // Convert address and pages to PFN (Page Frame Number) - let pfn = addr / PAGE_SIZE; - - // Check alignment using PFN - if pfn & ((1 << order) - 1) != 0 { - error!( - "zone {}: Page PFN {} is not properly aligned for order {} (needs alignment to {} pages)", - self.zone_id, pfn, order, 1 << order - ); - return; - } - - // Check page alignment - if addr & (PAGE_SIZE - 1) != 0 { - error!( - "zone {}: Attempt to free page at non-page-aligned address {:#x}", - self.zone_id, addr - ); - return; - } - - // Integrated double-free detection and buddy merging - let initial_order = order; - let size = (1 << initial_order) * PAGE_SIZE; - - // 1. Descendant check: Check if any part of the block being freed is already free - // This handles cases where a large block is freed but contains already free sub-blocks - for i in 0..initial_order { - if self.free_lists[i].has_block_in_range(pool, addr, addr + size) { - warn!( - "zone {}: Double free (descendant) detected at order {} in range [{:#x}, {:#x})", - self.zone_id, i, addr, addr + size - ); - return; - } - } - - // 2. Ancestor check and Merging loop - // We check from initial_order up to max_order. - // For each i, we check if the current aligned base is in free_lists[i]. - let mut current_addr = addr; - let mut merging = true; - - for i in initial_order..=self.max_order() { - let block_size = (1 << i) * PAGE_SIZE; - let current_base = current_addr & !(block_size - 1); - - // Double-free detection (Ancestor check): Check if this block or its parent is already free - if self.find_block_in_order(pool, i, current_base).is_some() { - warn!( - "zone {}: Double free detected at addr {:#x} (found at order {})", - self.zone_id, addr, i - ); - return; - } - - // Buddy merging - if merging && i < self.max_order() { - let buddy_addr = current_base ^ block_size; - if self.addr_in_zone(buddy_addr) { - if let Some((node_idx, prev_idx)) = - self.find_block_in_order(pool, i, buddy_addr) - { - // Buddy found, remove it and continue merging at next order - self.free_lists[i].remove_with_prev(pool, node_idx, prev_idx); - current_addr = current_base & buddy_addr; - order = i + 1; - continue; - } - } - // No buddy found or out of zone, stop merging but continue checking for double frees - merging = false; - } - } - - // Add the final merged block to the appropriate free list (sorted!) - let final_addr = current_addr; - let block = BuddyBlock { - order, - addr: final_addr, - }; - - if !self.add_block_to_order(pool, order, block) { - error!( - "zone {}: Failed to push block to free list: addr={:#x}, order={}", - self.zone_id, final_addr, order - ); - } - } - - /// Get statistics for this zone - #[cfg(feature = "tracking")] - pub fn get_stats(&self) -> super::stats::BuddyStats { - let mut stats = super::stats::BuddyStats::new(); - stats.total_pages = self.total_pages; - - for order in 0..=DEFAULT_MAX_ORDER { - let block_count = self.free_lists[order].len(); - stats.free_pages_by_order[order] = block_count; - stats.free_pages += block_count * (1 << order); - } - - stats.used_pages = stats.total_pages.saturating_sub(stats.free_pages); - stats - } - - /// Get free blocks of a specific order as an iterator - pub fn get_free_blocks_by_order<'a>( - &'a self, - pool: &'a GlobalNodePool, - order: u32, - ) -> super::pooled_list::PooledListIter<'a> { - self.free_lists[order as usize].iter(pool) - } - - /// Get the number of blocks in a specific order - pub fn get_order_block_count(&self, order: usize) -> usize { - if order <= DEFAULT_MAX_ORDER { - self.free_lists[order].len() - } else { - 0 - } - } -} - -impl Default for BuddySet { - fn default() -> Self { - Self::empty() - } -} diff --git a/src/buddy/global_node_pool.rs b/src/buddy/global_node_pool.rs deleted file mode 100644 index b921bd6..0000000 --- a/src/buddy/global_node_pool.rs +++ /dev/null @@ -1,316 +0,0 @@ -//! Global node pool for buddy allocator -//! -//! Provides a single pool of list nodes shared across all zones and orders. -//! This eliminates the need for per-zone list pools and improves memory efficiency. - -use super::buddy_block::BuddyBlock; - -/// Simple linked list node used by the global node pool -#[derive(Debug, Clone, Copy)] -pub struct ListNode { - pub data: T, - pub next: Option, -} - -fn align_up(addr: usize, align: usize) -> usize { - debug_assert!(align.is_power_of_two()); - (addr + align - 1) & !(align - 1) -} - -/// Global node pool - all zones and orders share nodes from this pool -/// -/// Nodes are carved from one or more contiguous memory regions provided -/// by the page allocator. The pool manages a single free list of nodes -/// using raw pointers (stored as usize indices). -pub struct GlobalNodePool { - /// Free list head - points to first available node (node address) - free_head: Option, - /// Total number of nodes ever added to the pool - total_nodes: usize, - /// Current number of free nodes in the pool - free_nodes: usize, - /// Allocation statistics - total_allocations: usize, - total_deallocations: usize, -} - -impl GlobalNodePool { - /// Create a new global node pool (uninitialized, must call init()) - pub const fn new() -> Self { - Self { - free_head: None, - total_nodes: 0, - free_nodes: 0, - total_allocations: 0, - total_deallocations: 0, - } - } - - /// Initialize the global node pool with a backing memory region - /// - /// The region is treated as raw memory and partitioned into - /// `ListNode` objects, which are all added to the - /// pool's free list. - pub fn init(&mut self, region_start: usize, region_size: usize) { - self.free_head = None; - self.total_nodes = 0; - self.free_nodes = 0; - self.total_allocations = 0; - self.total_deallocations = 0; - self.add_region(region_start, region_size); - } - - /// Add an additional memory region to the node pool - /// - /// This can be used to grow the pool at runtime by carving - /// more memory from the page allocator. - pub fn add_region(&mut self, region_start: usize, region_size: usize) { - let node_size = core::mem::size_of::>(); - let align = core::mem::align_of::>(); - if region_size < node_size { - return; - } - - let mut current = align_up(region_start, align); - let end = region_start + region_size; - - while current + node_size <= end { - unsafe { - let node_ptr = current as *mut ListNode; - core::ptr::write( - node_ptr, - ListNode { - data: core::mem::zeroed(), - next: self.free_head, - }, - ); - } - - self.free_head = Some(current); - self.total_nodes += 1; - self.free_nodes += 1; - current += node_size; - } - } - - /// Allocate a node from the pool - /// - /// Returns the index of the allocated node, or None if pool is exhausted - pub fn alloc_node(&mut self) -> Option { - let node_addr = self.free_head?; - - unsafe { - let node = &mut *(node_addr as *mut ListNode); - self.free_head = node.next; - node.next = None; - } - - self.total_allocations += 1; - if self.free_nodes > 0 { - self.free_nodes -= 1; - } else { - panic!("free_nodes: {}", self.free_nodes); - } - - Some(node_addr) - } - - /// Deallocate a node back to the pool - /// - /// The node should not be part of any active list when freed - pub fn dealloc_node(&mut self, node_idx: usize) { - unsafe { - let node_ptr = node_idx as *mut ListNode; - core::ptr::write( - node_ptr, - ListNode { - data: core::mem::zeroed(), - next: self.free_head, - }, - ); - } - - self.free_head = Some(node_idx); - self.total_deallocations += 1; - self.free_nodes += 1; - } - - /// Get a reference to a node by index - pub fn get_node(&self, node_idx: usize) -> Option<&ListNode> { - Some(unsafe { &*(node_idx as *const ListNode) }) - } - - /// Get a mutable reference to a node by index - pub fn get_node_mut(&mut self, node_idx: usize) -> Option<&mut ListNode> { - Some(unsafe { &mut *(node_idx as *mut ListNode) }) - } - - /// Get the number of free nodes in the pool - pub fn free_node_count(&self) -> usize { - self.free_nodes - } - - /// Get the number of allocated nodes - pub fn allocated_node_count(&self) -> usize { - self.total_nodes.saturating_sub(self.free_nodes) - } - - /// Get pool statistics - pub fn get_stats(&self) -> GlobalPoolStats { - GlobalPoolStats { - total_nodes: self.total_nodes, - free_nodes: self.free_nodes, - allocated_nodes: self.allocated_node_count(), - total_allocations: self.total_allocations, - total_deallocations: self.total_deallocations, - } - } -} - -impl Default for GlobalNodePool { - fn default() -> Self { - Self::new() - } -} - -/// Global pool statistics -#[derive(Debug, Default, Clone)] -pub struct GlobalPoolStats { - pub total_nodes: usize, - pub free_nodes: usize, - pub allocated_nodes: usize, - pub total_allocations: usize, - pub total_deallocations: usize, -} - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_NODE_COUNT: usize = 512; - #[test] - fn test_pool_init() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT); - assert_eq!(pool.allocated_node_count(), 0); - } - - #[test] - fn test_alloc_dealloc() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let idx1 = pool.alloc_node().unwrap(); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1); - assert_eq!(pool.allocated_node_count(), 1); - - let idx2 = pool.alloc_node().unwrap(); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 2); - - pool.dealloc_node(idx1); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1); - - pool.dealloc_node(idx2); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT); - } - - #[test] - fn test_pool_exhaustion() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - // Allocate all nodes - let mut indices = alloc::vec::Vec::new(); - for _ in 0..TEST_NODE_COUNT { - indices.push(pool.alloc_node().unwrap()); - } - - assert_eq!(pool.free_node_count(), 0); - assert!(pool.alloc_node().is_none()); - - // Free one and allocate again - pool.dealloc_node(indices[0]); - assert_eq!(pool.free_node_count(), 1); - assert!(pool.alloc_node().is_some()); - } - - #[test] - fn test_node_access() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let idx = pool.alloc_node().unwrap(); - - // Get mutable reference and set data - if let Some(node) = pool.get_node_mut(idx) { - node.data = BuddyBlock { - order: 0, - addr: 0x1000, - }; - } - - // Get reference and read data - if let Some(node) = pool.get_node(idx) { - assert_eq!(node.data.order, 0); - assert_eq!(node.data.addr, 0x1000); - } - } - - #[test] - fn test_stats() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let idx1 = pool.alloc_node().unwrap(); - let _idx2 = pool.alloc_node().unwrap(); - - let stats = pool.get_stats(); - assert_eq!(stats.total_nodes, TEST_NODE_COUNT); - assert_eq!(stats.free_nodes, TEST_NODE_COUNT - 2); - assert_eq!(stats.allocated_nodes, 2); - assert_eq!(stats.total_allocations, 2); - assert_eq!(stats.total_deallocations, 0); - - pool.dealloc_node(idx1); - let stats2 = pool.get_stats(); - assert_eq!(stats2.free_nodes, TEST_NODE_COUNT - 1); - assert_eq!(stats2.total_deallocations, 1); - } -} diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index 67be2c4..a694e08 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -1,23 +1,386 @@ -//! Buddy page allocator module +//! Buddy page allocator — page-metadata-based with intrusive free lists. //! -//! This module provides a complete buddy system implementation with: -//! - Sorted linked lists for efficient contiguity checking -//! - Multi-zone support -//! - Detailed statistics and debugging - -pub mod buddy_allocator; -pub mod buddy_block; -pub mod buddy_set; -pub mod global_node_pool; -pub mod pooled_list; -pub mod stats; - -pub use buddy_allocator::BuddyPageAllocator; -pub use buddy_block::{BuddyBlock, ZoneInfo, MAX_ZONES}; -pub use buddy_set::BuddySet; -pub use global_node_pool::{GlobalNodePool, ListNode}; -pub use pooled_list::PooledLinkedList; -#[cfg(not(feature = "tracking"))] -pub use stats::DEFAULT_MAX_ORDER; -#[cfg(feature = "tracking")] -pub use stats::{BuddyStats, DEFAULT_MAX_ORDER}; +//! The allocator manages a contiguous virtual address range ("the heap"). +//! An external metadata region stores one [`PageMeta`] per page frame, +//! enabling O(1) free-list operations without any dynamic allocation. + +pub mod page_meta; + +pub use page_meta::{PageFlags, PageMeta, PFN_NONE}; + +use crate::error::{AllocError, AllocResult}; +use crate::{is_aligned, OsImpl}; +use page_meta::{free_list_pop, free_list_push, free_list_remove}; + +/// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks. +pub const MAX_ORDER: usize = 20; + +/// DMA32 zone upper bound (4 GiB physical). +const DMA32_LIMIT: usize = 0x1_0000_0000; + +/// Page-metadata-based buddy allocator. +/// +/// `PAGE_SIZE` must be a power of two (commonly 0x1000 = 4 KiB). +pub struct BuddyAllocator { + /// Pointer to the `PageMeta` array in the external metadata region. + meta: *mut PageMeta, + /// Maximum number of pages the metadata can describe. + max_pages: usize, + /// Virtual start address of the managed heap region. + heap_start: usize, + /// Size of the managed heap in bytes. + heap_size: usize, + /// Free-list head PFN per order (`PFN_NONE` = empty). + free_lists: [u32; MAX_ORDER + 1], + /// Number of currently free pages. + free_pages: usize, + /// Total number of pages in the heap. + total_pages: usize, + /// Optional OS interface for DMA32/lowmem address translation. + os: Option<&'static dyn OsImpl>, +} + +// SAFETY: The allocator is designed to be wrapped in a SpinMutex. +// Raw pointer field `meta` points to the externally supplied metadata region +// whose lifetime is managed by the caller. +unsafe impl Send for BuddyAllocator {} + +impl BuddyAllocator { + // ------------------------------------------------------------------ + // Construction helpers + // ------------------------------------------------------------------ + + /// Calculate the metadata-region size (in bytes) required for `heap_size` bytes. + pub const fn required_meta_size(heap_size: usize) -> usize { + let pages = heap_size / PAGE_SIZE; + pages * core::mem::size_of::() + } + + /// Create an uninitialised allocator. Call [`init`](Self::init) before use. + pub const fn new() -> Self { + Self { + meta: core::ptr::null_mut(), + max_pages: 0, + heap_start: 0, + heap_size: 0, + free_lists: [PFN_NONE; MAX_ORDER + 1], + free_pages: 0, + total_pages: 0, + os: None, + } + } + + /// Initialise the allocator over a heap region, using `meta` for page metadata. + /// + /// # Safety + /// - `meta` must point to a writeable region of at least + /// [`required_meta_size`](Self::required_meta_size)`(heap_size)` bytes. + /// - `heap_start` must be page-aligned and the region `[heap_start, heap_start + heap_size)` + /// must not overlap with the metadata region. + /// - The caller is responsible for ensuring the metadata region lives as long as this allocator. + pub unsafe fn init( + &mut self, + meta: *mut u8, + meta_size: usize, + heap_start: usize, + heap_size: usize, + os: Option<&'static dyn OsImpl>, + ) -> AllocResult { + // Validate + if !PAGE_SIZE.is_power_of_two() { + return Err(AllocError::InvalidParam); + } + if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 { + return Err(AllocError::InvalidParam); + } + let total_pages = heap_size / PAGE_SIZE; + let required = Self::required_meta_size(heap_size); + if meta_size < required { + return Err(AllocError::InvalidParam); + } + + let meta_ptr = meta as *mut PageMeta; + + // Zero-init all page metadata + for i in 0..total_pages { + meta_ptr.add(i).write(PageMeta::new()); + } + + self.meta = meta_ptr; + self.max_pages = total_pages; + self.heap_start = heap_start; + self.heap_size = heap_size; + self.free_lists = [PFN_NONE; MAX_ORDER + 1]; + self.free_pages = 0; + self.total_pages = total_pages; + self.os = os; + + // Build free lists: greedily carve the largest possible blocks. + let mut pfn: usize = 0; + while pfn < total_pages { + // Find highest order block that (a) fits and (b) is naturally aligned. + let mut order = MAX_ORDER; + loop { + let block_pages = 1usize << order; + if block_pages <= total_pages - pfn && (pfn & (block_pages - 1)) == 0 { + break; + } + if order == 0 { + break; + } + order -= 1; + } + let block_pages = 1usize << order; + let m = &mut *self.meta.add(pfn); + m.flags = PageFlags::Free; + m.order = order as u8; + free_list_push(self.meta, &mut self.free_lists, pfn as u32, order); + self.free_pages += block_pages; + pfn += block_pages; + } + + log::debug!( + "BuddyAllocator: heap {:#x}..{:#x}, {} pages, {} free", + heap_start, + heap_start + heap_size, + total_pages, + self.free_pages, + ); + + Ok(()) + } + + // ------------------------------------------------------------------ + // Queries + // ------------------------------------------------------------------ + + /// Total number of pages managed. + pub fn total_pages(&self) -> usize { + self.total_pages + } + + /// Number of currently free pages. + pub fn free_pages(&self) -> usize { + self.free_pages + } + + /// Start address of the managed heap. + pub fn heap_start(&self) -> usize { + self.heap_start + } + + /// Size of the managed heap in bytes. + pub fn heap_size(&self) -> usize { + self.heap_size + } + + // ------------------------------------------------------------------ + // Allocation + // ------------------------------------------------------------------ + + /// Allocate `count` contiguous pages, returning the virtual address. + /// + /// `count` is rounded up to the nearest power of two. + /// Optional `align` must be a power of two (in bytes) and ≥ `PAGE_SIZE`; 0 means + /// default page alignment. + pub fn alloc_pages(&mut self, count: usize, align: usize) -> AllocResult { + if count == 0 { + return Err(AllocError::InvalidParam); + } + let align = if align == 0 { PAGE_SIZE } else { align }; + if !align.is_power_of_two() || align < PAGE_SIZE { + return Err(AllocError::InvalidParam); + } + + let order = count.next_power_of_two().trailing_zeros() as usize; + if order > MAX_ORDER { + return Err(AllocError::InvalidParam); + } + + // Alignment order: how many pages the alignment corresponds to. + let align_order = (align / PAGE_SIZE).trailing_zeros() as usize; + let effective_order = order.max(align_order); + + self.alloc_order(effective_order) + } + + /// Low-level allocation by buddy order. + fn alloc_order(&mut self, order: usize) -> AllocResult { + // Search upwards for the first non-empty free list + let mut found_order = order; + while found_order <= MAX_ORDER { + if self.free_lists[found_order] != PFN_NONE { + break; + } + found_order += 1; + } + if found_order > MAX_ORDER { + return Err(AllocError::NoMemory); + } + + // Pop from found_order + let pfn = unsafe { free_list_pop(self.meta, &mut self.free_lists, found_order) }; + debug_assert_ne!(pfn, PFN_NONE); + + // Split down to target order + let mut current_order = found_order; + while current_order > order { + current_order -= 1; + let buddy_pfn = pfn as usize + (1 << current_order); + unsafe { + let bm = &mut *self.meta.add(buddy_pfn); + bm.flags = PageFlags::Free; + bm.order = current_order as u8; + free_list_push( + self.meta, + &mut self.free_lists, + buddy_pfn as u32, + current_order, + ); + } + } + + // Mark head page as allocated + unsafe { + let m = &mut *self.meta.add(pfn as usize); + m.flags = PageFlags::Allocated; + m.order = order as u8; + } + + self.free_pages -= 1 << order; + let addr = self.heap_start + (pfn as usize) * PAGE_SIZE; + Ok(addr) + } + + /// Allocate pages whose *physical* address is below 4 GiB (DMA32 zone). + /// + /// Requires [`OsImpl`] to have been set during init. + pub fn alloc_pages_lowmem(&mut self, count: usize, align: usize) -> AllocResult { + let os = self.os.ok_or(AllocError::InvalidParam)?; + + if count == 0 { + return Err(AllocError::InvalidParam); + } + let align = if align == 0 { PAGE_SIZE } else { align }; + if !align.is_power_of_two() || align < PAGE_SIZE { + return Err(AllocError::InvalidParam); + } + + let order = count.next_power_of_two().trailing_zeros() as usize; + let align_order = (align / PAGE_SIZE).trailing_zeros() as usize; + let effective_order = order.max(align_order); + if effective_order > MAX_ORDER { + return Err(AllocError::InvalidParam); + } + + // Walk free lists looking for a block below DMA32_LIMIT + for search_order in effective_order..=MAX_ORDER { + let mut pfn_u32 = self.free_lists[search_order]; + while pfn_u32 != PFN_NONE { + let addr = self.heap_start + (pfn_u32 as usize) * PAGE_SIZE; + let phys = os.virt_to_phys(addr); + let block_bytes = (1usize << search_order) * PAGE_SIZE; + if phys + block_bytes <= DMA32_LIMIT { + // Found! Remove from list, then split. + unsafe { + free_list_remove(self.meta, &mut self.free_lists, pfn_u32, search_order); + } + + // Split + let mut current_order = search_order; + while current_order > effective_order { + current_order -= 1; + let buddy_pfn = pfn_u32 as usize + (1 << current_order); + unsafe { + let bm = &mut *self.meta.add(buddy_pfn); + bm.flags = PageFlags::Free; + bm.order = current_order as u8; + free_list_push( + self.meta, + &mut self.free_lists, + buddy_pfn as u32, + current_order, + ); + } + } + + unsafe { + let m = &mut *self.meta.add(pfn_u32 as usize); + m.flags = PageFlags::Allocated; + m.order = effective_order as u8; + } + self.free_pages -= 1 << effective_order; + return Ok(addr); + } + pfn_u32 = unsafe { (*self.meta.add(pfn_u32 as usize)).next }; + } + } + + Err(AllocError::NoMemory) + } + + // ------------------------------------------------------------------ + // Deallocation + // ------------------------------------------------------------------ + + /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). + /// + /// `addr` must be the exact address returned by alloc, and `count` must match. + pub fn dealloc_pages(&mut self, addr: usize, count: usize) { + debug_assert!(is_aligned(addr, PAGE_SIZE)); + debug_assert!(addr >= self.heap_start); + + let pfn = (addr - self.heap_start) / PAGE_SIZE; + debug_assert!(pfn < self.max_pages); + + let order = count.next_power_of_two().trailing_zeros() as usize; + self.dealloc_order(pfn, order); + } + + /// Mark the page at `pfn` with the given flags (used by slab to tag pages). + /// + /// # Safety + /// The caller must ensure `pfn` is valid and properly allocated. + pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) { + let pfn = (addr - self.heap_start) / PAGE_SIZE; + (*self.meta.add(pfn)).flags = flags; + } + + /// Read the flags of the page containing `addr`. + pub fn page_flags(&self, addr: usize) -> PageFlags { + let pfn = (addr - self.heap_start) / PAGE_SIZE; + unsafe { (*self.meta.add(pfn)).flags } + } + + /// Internal: free a block at `pfn` of the given `order`, merging buddies. + fn dealloc_order(&mut self, mut pfn: usize, mut order: usize) { + // Only the originally freed block's pages are newly freed; + // merged buddies were already counted as free. + let freed_pages = 1usize << order; + + while order < MAX_ORDER { + let buddy_pfn = pfn ^ (1 << order); + if buddy_pfn >= self.max_pages { + break; + } + let buddy = unsafe { &*self.meta.add(buddy_pfn) }; + if buddy.flags != PageFlags::Free || buddy.order as usize != order { + break; + } + // Merge: remove buddy from its free list + unsafe { + free_list_remove(self.meta, &mut self.free_lists, buddy_pfn as u32, order); + } + pfn = pfn.min(buddy_pfn); + order += 1; + } + + unsafe { + let m = &mut *self.meta.add(pfn); + m.flags = PageFlags::Free; + m.order = order as u8; + free_list_push(self.meta, &mut self.free_lists, pfn as u32, order); + } + self.free_pages += freed_pages; + } +} diff --git a/src/buddy/page_meta.rs b/src/buddy/page_meta.rs new file mode 100644 index 0000000..ebb6042 --- /dev/null +++ b/src/buddy/page_meta.rs @@ -0,0 +1,125 @@ +/// Per-page metadata stored in the external metadata region. +/// +/// Each page frame in the heap has a corresponding `PageMeta` entry. +/// Free pages are linked together via intrusive doubly-linked lists using PFN indices. + +/// Sentinel value indicating "no page" in free-list links. +pub const PFN_NONE: u32 = u32::MAX; + +/// Page state flags. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum PageFlags { + /// Page is free and sits in a buddy free list. + Free = 0, + /// Page is allocated (head page of a buddy block). + Allocated = 1, + /// Page is used as a slab page. + Slab = 2, +} + +/// Metadata for a single page frame (12 bytes). +/// +/// Head pages carry the `order` of the entire block. +/// Tail pages within a buddy block are marked `Allocated` (or `Slab`) with order 0. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct PageMeta { + /// Current state of this page. + pub flags: PageFlags, + /// Order of the block (only meaningful on head pages). + pub order: u8, + /// Reserved padding. + pub _pad: u16, + /// Previous PFN in the same-order free list (`PFN_NONE` if head or not free). + pub prev: u32, + /// Next PFN in the same-order free list (`PFN_NONE` if tail or not free). + pub next: u32, +} + +const _: () = assert!(core::mem::size_of::() == 12); + +impl PageMeta { + /// Create a zeroed (free, order-0) page meta. + pub const fn new() -> Self { + Self { + flags: PageFlags::Free, + order: 0, + _pad: 0, + prev: PFN_NONE, + next: PFN_NONE, + } + } +} + +// --------------------------------------------------------------------------- +// Free-list helpers operating on a `*mut PageMeta` array + head array +// --------------------------------------------------------------------------- + +/// Push `pfn` onto the front of `free_lists[order]`. +/// +/// # Safety +/// `meta` must point to an array with at least `pfn + 1` entries. +/// `pfn` must not already be in any free list. +#[inline] +pub unsafe fn free_list_push(meta: *mut PageMeta, free_lists: &mut [u32], pfn: u32, order: usize) { + let old_head = free_lists[order]; + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = old_head; + if old_head != PFN_NONE { + (*meta.add(old_head as usize)).prev = pfn; + } + free_lists[order] = pfn; +} + +/// Pop the first PFN from `free_lists[order]`, returning `PFN_NONE` if empty. +/// +/// # Safety +/// `meta` must be a valid metadata array. +#[inline] +pub unsafe fn free_list_pop(meta: *mut PageMeta, free_lists: &mut [u32], order: usize) -> u32 { + let head = free_lists[order]; + if head == PFN_NONE { + return PFN_NONE; + } + let m = &mut *meta.add(head as usize); + let next = m.next; + m.prev = PFN_NONE; + m.next = PFN_NONE; + if next != PFN_NONE { + (*meta.add(next as usize)).prev = PFN_NONE; + } + free_lists[order] = next; + head +} + +/// Remove `pfn` from the free list at `order`. +/// +/// # Safety +/// `pfn` must currently be in `free_lists[order]`. +#[inline] +pub unsafe fn free_list_remove( + meta: *mut PageMeta, + free_lists: &mut [u32], + pfn: u32, + order: usize, +) { + let m = &*meta.add(pfn as usize); + let prev = m.prev; + let next = m.next; + + if prev != PFN_NONE { + (*meta.add(prev as usize)).next = next; + } else { + // pfn was the head + free_lists[order] = next; + } + if next != PFN_NONE { + (*meta.add(next as usize)).prev = prev; + } + + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = PFN_NONE; +} diff --git a/src/buddy/pooled_list.rs b/src/buddy/pooled_list.rs deleted file mode 100644 index aa28634..0000000 --- a/src/buddy/pooled_list.rs +++ /dev/null @@ -1,522 +0,0 @@ -//! Pooled linked list implementation using global node pool -//! -//! Provides linked lists that draw nodes from a shared global pool, -//! allowing efficient use of memory across all zones and orders. - -use log::{error, warn}; - -use super::{buddy_block::BuddyBlock, global_node_pool::GlobalNodePool}; - -/// Pooled linked list - uses nodes from global pool -/// -/// This maintains only the list structure (head/tail/len), while -/// all nodes are allocated from the global pool. -pub struct PooledLinkedList { - head: Option, - tail: Option, - len: usize, -} - -impl PooledLinkedList { - /// Create a new empty pooled linked list - pub const fn new() -> Self { - Self { - head: None, - tail: None, - len: 0, - } - } - - /// Insert element in sorted order (ascending by address) - /// This is used for buddy free lists to enable efficient contiguity checking - pub fn insert_sorted(&mut self, pool: &mut GlobalNodePool, data: BuddyBlock) -> bool { - let new_node_idx = match pool.alloc_node() { - Some(idx) => idx, - None => { - error!("Global node pool exhausted"); - return false; - } - }; - - // Find insertion position - let mut prev_idx = None; - let mut current_idx = self.head; - let mut visited = 0; - - while let Some(idx) = current_idx { - if visited > self.len { - error!("Potential cycle detected during insert"); - // Deallocate the node - pool.dealloc_node(new_node_idx); - return false; - } - - if let Some(node) = pool.get_node(idx) { - if node.data.addr == data.addr { - // Block already in free list - this is a success (no-op) - // The caller tried to free something that's already free - pool.dealloc_node(new_node_idx); - return true; - } - if node.data.addr > data.addr { - break; // Found position - } - prev_idx = current_idx; - current_idx = node.next; - } else { - error!("Invalid node reference in list"); - pool.dealloc_node(new_node_idx); - return false; - } - visited += 1; - } - - // Initialize the new node - if let Some(node) = pool.get_node_mut(new_node_idx) { - node.data = data; - node.next = current_idx; - } else { - error!("Failed to get mutable reference to newly allocated node"); - pool.dealloc_node(new_node_idx); - return false; - } - - // Update links - if let Some(prev) = prev_idx { - if let Some(prev_node) = pool.get_node_mut(prev) { - prev_node.next = Some(new_node_idx); - } - } else { - self.head = Some(new_node_idx); - } - - // Update tail if needed - if current_idx.is_none() { - self.tail = Some(new_node_idx); - } - - self.len += 1; - true - } - - /// Check if any block in the list falls within the given address range [start, end) - pub fn has_block_in_range(&self, pool: &GlobalNodePool, start: usize, end: usize) -> bool { - let mut current_idx = self.head; - let mut visited = 0; - - while let Some(idx) = current_idx { - if visited > self.len { - break; - } - if let Some(node) = pool.get_node(idx) { - // Early termination: list is sorted by address - if node.data.addr >= end { - break; - } - // Check if block starts within the range - // For i < initial_order, any block in the range is a conflict. - // Since blocks are aligned to their size, a block starting before 'start' - // cannot overlap with [start, end) if its size is smaller than start's alignment. - if node.data.addr >= start { - return true; - } - current_idx = node.next; - } else { - break; - } - visited += 1; - } - false - } - - /// Pop an element from the front of the list - pub fn pop_front(&mut self, pool: &mut GlobalNodePool) -> Option { - self.head?; - - let head_idx = self.head?; - - if let Some(head_node) = pool.get_node_mut(head_idx) { - self.head = head_node.next; - if self.head.is_none() { - self.tail = None; - } - - let data = head_node.data; - - // Return node to pool - pool.dealloc_node(head_idx); - self.len -= 1; - - Some(data) - } else { - error!("Head node {head_idx} is corrupted"); - None - } - } - - /// Check if the list is empty - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Get the length of the list - pub fn len(&self) -> usize { - self.len - } - - /// Find a node by address (for buddy system) - /// - /// Returns (node_idx, prev_idx) where prev_idx is the node before it (or None if head) - pub fn find_by_addr( - &self, - pool: &GlobalNodePool, - addr: usize, - ) -> Option<(usize, Option)> { - let mut prev_idx = None; - let mut current_idx = self.head; - let mut visited = 0; - - while let Some(idx) = current_idx { - if visited > self.len { - error!("Potential cycle detected during search"); - return None; - } - - if let Some(node) = pool.get_node(idx) { - // Early termination: list is sorted by address - if node.data.addr > addr { - break; - } - if node.data.addr == addr { - return Some((idx, prev_idx)); - } - prev_idx = current_idx; - current_idx = node.next; - } else { - break; - } - visited += 1; - } - - None - } - - /// Remove a node at the given index - pub fn remove(&mut self, pool: &mut GlobalNodePool, node_idx: usize) -> bool { - // Find the node - let mut prev_idx = None; - let mut current_idx = self.head; - let mut visited = 0; - - while let Some(idx) = current_idx { - if visited > self.len { - warn!("Potential cycle detected during remove"); - return false; - } - - if idx == node_idx { - break; - } - prev_idx = current_idx; - if let Some(node) = pool.get_node(idx) { - current_idx = node.next; - } else { - break; - } - visited += 1; - } - - if current_idx != Some(node_idx) { - return false; - } - - self.remove_with_prev_impl(pool, node_idx, prev_idx) - } - - /// Remove a node using known prev_idx (O(1) operation) - /// - /// This is used when we already know the previous node index from find_by_addr(), - /// avoiding a second traversal of the list. - pub fn remove_with_prev( - &mut self, - pool: &mut GlobalNodePool, - node_idx: usize, - prev_idx: Option, - ) -> bool { - // Verify the node exists - if pool.get_node(node_idx).is_none() { - warn!("Invalid node index {node_idx} for remove_with_prev"); - return false; - } - - // Verify prev_idx leads to node_idx if provided - if let Some(prev) = prev_idx { - if let Some(prev_node) = pool.get_node(prev) { - if prev_node.next != Some(node_idx) { - warn!("prev_idx {prev} does not point to node_idx {node_idx}"); - return false; - } - } else { - warn!("Invalid prev_idx {prev}"); - return false; - } - } else if self.head != Some(node_idx) { - // prev_idx is None means node should be head - warn!("prev_idx is None but node_idx {node_idx} is not head"); - return false; - } - - self.remove_with_prev_impl(pool, node_idx, prev_idx) - } - - /// Internal implementation of remove with known prev_idx - fn remove_with_prev_impl( - &mut self, - pool: &mut GlobalNodePool, - node_idx: usize, - prev_idx: Option, - ) -> bool { - // Get the node's next pointer before deallocating - let next_idx = pool.get_node(node_idx).and_then(|n| n.next); - - // Update links - if let Some(prev) = prev_idx { - if let Some(prev_node) = pool.get_node_mut(prev) { - prev_node.next = next_idx; - } - } else { - self.head = next_idx; - } - - // Update tail if needed - if self.tail == Some(node_idx) { - if next_idx.is_none() { - self.tail = prev_idx; - } - } else if self.head.is_none() { - self.tail = None; - } - - // Return node to pool - pool.dealloc_node(node_idx); - self.len -= 1; - true - } - - /// Get iterator over elements - pub fn iter<'a>(&'a self, pool: &'a GlobalNodePool) -> PooledListIter<'a> { - PooledListIter { - pool, - current: self.head, - } - } - - /// Clear all nodes from the list - /// - /// Returns all nodes to the global pool - pub fn clear(&mut self, pool: &mut GlobalNodePool) { - while self.pop_front(pool).is_some() { - // Just pop all elements - } - } -} - -impl Default for PooledLinkedList { - fn default() -> Self { - Self::new() - } -} - -/// Iterator for PooledLinkedList -pub struct PooledListIter<'a> { - pool: &'a GlobalNodePool, - current: Option, -} - -impl<'a> Iterator for PooledListIter<'a> { - type Item = &'a BuddyBlock; - - fn next(&mut self) -> Option { - self.current.and_then(|idx| { - if let Some(node) = self.pool.get_node(idx) { - self.current = node.next; - Some(&node.data) - } else { - self.current = None; - None - } - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::buddy::ListNode; - - const TEST_NODE_COUNT: usize = 512; - - #[test] - fn test_pooled_list_basic() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let mut list: PooledLinkedList = PooledLinkedList::new(); - - assert!(list.is_empty()); - assert_eq!(list.len(), 0); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT); - - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x1000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x2000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x3000, - }, - ); - - assert_eq!(list.len(), 3); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 3); - - assert_eq!( - list.pop_front(&mut pool), - Some(BuddyBlock { - order: 0, - addr: 0x1000 - }) - ); - assert_eq!( - list.pop_front(&mut pool), - Some(BuddyBlock { - order: 0, - addr: 0x2000 - }) - ); - assert_eq!(list.len(), 1); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1); - - // Clear remaining - list.clear(&mut pool); - assert!(list.is_empty()); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT); - } - - #[test] - fn test_insert_sorted() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let mut list: PooledLinkedList = PooledLinkedList::new(); - - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x5000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x3000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x7000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x1000, - }, - ); - - let items: alloc::vec::Vec<_> = list.iter(&pool).collect(); - assert_eq!(items.len(), 4); - assert_eq!(items[0].addr, 0x1000); - assert_eq!(items[1].addr, 0x3000); - assert_eq!(items[2].addr, 0x5000); - assert_eq!(items[3].addr, 0x7000); - } - - #[test] - fn test_find_and_remove() { - let mut pool: GlobalNodePool = GlobalNodePool::new(); - let mut backing = [ListNode { - data: BuddyBlock { order: 0, addr: 0 }, - next: None, - }; TEST_NODE_COUNT]; - - let region_start = backing.as_mut_ptr() as usize; - let region_size = core::mem::size_of_val(&backing); - pool.init(region_start, region_size); - - let mut list: PooledLinkedList = PooledLinkedList::new(); - - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x1000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x2000, - }, - ); - list.insert_sorted( - &mut pool, - BuddyBlock { - order: 0, - addr: 0x3000, - }, - ); - - let (node_idx, _) = list.find_by_addr(&pool, 0x2000).unwrap(); - assert!(list.remove(&mut pool, node_idx)); - - assert_eq!(list.len(), 2); - assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 2); - - let items: alloc::vec::Vec<_> = list.iter(&pool).collect(); - assert_eq!(items[0].addr, 0x1000); - assert_eq!(items[1].addr, 0x3000); - } -} diff --git a/src/buddy/stats.rs b/src/buddy/stats.rs deleted file mode 100644 index 6c88cc4..0000000 --- a/src/buddy/stats.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Statistics and debugging for buddy allocator -//! -//! Provides detailed statistics tracking and failure reporting. - -use super::buddy_block::ZoneInfo; - -/// Maximum order supported -pub const DEFAULT_MAX_ORDER: usize = 28; - -/// Buddy system statistics -#[derive(Debug, Clone, Copy)] -pub struct BuddyStats { - pub total_pages: usize, - pub free_pages: usize, - pub used_pages: usize, - pub free_pages_by_order: [usize; DEFAULT_MAX_ORDER + 1], -} - -impl Default for BuddyStats { - fn default() -> Self { - Self { - total_pages: 0, - free_pages: 0, - used_pages: 0, - free_pages_by_order: [0; DEFAULT_MAX_ORDER + 1], - } - } -} - -impl BuddyStats { - pub const fn new() -> Self { - Self { - total_pages: 0, - free_pages: 0, - used_pages: 0, - free_pages_by_order: [0; DEFAULT_MAX_ORDER + 1], - } - } - - /// Add statistics from another BuddyStats - pub fn add(&mut self, other: &BuddyStats) { - self.total_pages += other.total_pages; - self.free_pages += other.free_pages; - self.used_pages += other.used_pages; - for (i, &count) in other.free_pages_by_order.iter().enumerate() { - self.free_pages_by_order[i] += count; - } - } -} - -/// Detailed memory statistics reporter -pub struct MemoryStatsReporter; - -impl MemoryStatsReporter { - /// Print detailed allocation failure statistics - /// This is a standalone function to keep allocation logic clean - #[allow(unused_variables)] - pub fn print_alloc_failure_stats( - page_size: usize, - num_zones: usize, - total_stats: &BuddyStats, - zone_infos: &[ZoneInfo], - zone_stats: &[BuddyStats], - request_pages: usize, - request_align: usize, - ) { - { - use log::error; - error!("========================================"); - error!( - "Request: {} pages ({} KB, alignment:{})", - request_pages, - (request_pages * page_size) / (1024), - request_align - ); - - error!("Overall Memory State:"); - error!(" Total zones: {num_zones}"); - error!( - " Total pages: {} ({} KB)", - total_stats.total_pages, - (total_stats.total_pages * page_size) / 1024 - ); - error!( - " Free pages: {} ({} KB)", - total_stats.free_pages, - (total_stats.free_pages * page_size) / (1024) - ); - error!( - " Used pages: {} ({} KB)", - total_stats.used_pages, - (total_stats.used_pages * page_size) / (1024) - ); - error!("========================================"); - - for (i, zone_info) in zone_infos.iter().take(num_zones).enumerate() { - error!("Zone {i}:"); - error!( - " Range: [{:#x}, {:#x})", - zone_info.start_addr, zone_info.end_addr - ); - error!(" Total pages: {}", zone_info.total_pages); - error!( - " Free pages: {} / {}", - zone_stats[i].free_pages, zone_info.total_pages - ); - error!(" Free blocks by order:"); - - for order in (0..=DEFAULT_MAX_ORDER).rev() { - let count = zone_stats[i].free_pages_by_order[order]; - if count > 0 { - let block_size = (1 << order) * page_size; - let total_kb = (count * block_size) / (1024); - error!( - " Order {}: {} blocks ({} KB each, {} KB total)", - order, - count, - block_size / (1024), - total_kb - ); - } - } - error!("----------------------------------------"); - } - - error!("========================================"); - } - } -} diff --git a/src/error.rs b/src/error.rs index e41867d..cc2d849 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,14 +1,27 @@ +use core::fmt; + /// The error type used for allocation operations. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AllocError { /// Invalid size, alignment, or other input parameter. InvalidParam, - /// Memory added by `add_memory` overlaps with existing memory. - MemoryOverlap, /// Not enough memory is available to satisfy the request. NoMemory, /// Attempted to deallocate memory that was not allocated. NotAllocated, + /// The allocator has not been initialized. + NotInitialized, +} + +impl fmt::Display for AllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidParam => write!(f, "invalid parameter"), + Self::NoMemory => write!(f, "out of memory"), + Self::NotAllocated => write!(f, "memory not allocated"), + Self::NotInitialized => write!(f, "allocator not initialized"), + } + } } /// A [`Result`] alias with [`AllocError`] as the error type. diff --git a/src/global.rs b/src/global.rs new file mode 100644 index 0000000..0d4441f --- /dev/null +++ b/src/global.rs @@ -0,0 +1,319 @@ +/// Global allocator composing buddy (pages) + per-CPU slab (objects). +/// +/// Implements [`core::alloc::GlobalAlloc`] so it can serve as `#[global_allocator]`. +/// Cross-CPU frees are lock-free via [`SlabPageHeader::remote_free`]. +use core::alloc::{GlobalAlloc, Layout}; +use core::ptr::{self, NonNull}; +use core::sync::atomic::{AtomicBool, Ordering}; + +use spin::Mutex as SpinMutex; + +use crate::buddy::{BuddyAllocator, PageFlags}; +use crate::error::{AllocError, AllocResult}; +use crate::slab::page::{SlabPageHeader, SLAB_MAGIC}; +use crate::slab::size_class::{SizeClass, SLAB_MAX_SIZE}; +use crate::slab::{SlabAllocResult, SlabAllocator, SlabDeallocResult}; +use crate::{align_up, OsImpl}; + +/// Unified allocator: buddy page allocator + per-CPU slab caches. +pub struct GlobalAllocator { + buddy: SpinMutex>, + per_cpu_slabs: *mut SpinMutex>, + cpu_count: usize, + os: Option<&'static dyn OsImpl>, + initialized: AtomicBool, +} + +// SAFETY: All mutable state is behind SpinMutex or AtomicBool. +// `per_cpu_slabs` is a raw pointer into the metadata region whose lifetime +// is managed by the caller (same as BuddyAllocator::meta). +unsafe impl Sync for GlobalAllocator {} +unsafe impl Send for GlobalAllocator {} + +impl GlobalAllocator { + // ------------------------------------------------------------------ + // Metadata sizing + // ------------------------------------------------------------------ + + /// Calculate the total metadata region size needed. + /// + /// The metadata region is laid out as: + /// ```text + /// [ PageMeta[max_pages] | padding | SpinMutex[cpu_count] ] + /// ``` + pub const fn required_metadata_size(heap_size: usize, cpu_count: usize) -> usize { + let buddy_meta = BuddyAllocator::::required_meta_size(heap_size); + // Align slab array to its natural alignment. + let slab_align = core::mem::align_of::>>(); + let slab_offset = (buddy_meta + slab_align - 1) & !(slab_align - 1); + let slab_size = core::mem::size_of::>>() * cpu_count; + slab_offset + slab_size + } + + /// Required alignment for the metadata region. + pub const fn required_metadata_align() -> usize { + let a1 = core::mem::align_of::(); + let a2 = core::mem::align_of::>>(); + if a1 > a2 { + a1 + } else { + a2 + } + } + + // ------------------------------------------------------------------ + // Construction / initialisation + // ------------------------------------------------------------------ + + /// Create an uninitialised global allocator. + pub const fn new() -> Self { + Self { + buddy: SpinMutex::new(BuddyAllocator::new()), + per_cpu_slabs: ptr::null_mut(), + cpu_count: 0, + os: None, + initialized: AtomicBool::new(false), + } + } + + /// Initialise the allocator. + /// + /// # Arguments + /// - `meta` / `meta_size` — writable metadata region (see [`required_metadata_size`]). + /// - `heap_start` / `heap_size` — the heap virtual address range to manage. + /// - `cpu_count` — number of CPUs (≥ 1). + /// - `os` — platform abstraction. + /// + /// # Safety + /// - The metadata and heap regions must not overlap. + /// - `meta` must be aligned to [`required_metadata_align`]. + /// - Both regions must remain valid for the lifetime of this allocator. + pub unsafe fn init( + &self, + meta: *mut u8, + meta_size: usize, + heap_start: usize, + heap_size: usize, + cpu_count: usize, + os: &'static dyn OsImpl, + ) -> AllocResult { + if cpu_count == 0 || meta.is_null() { + return Err(AllocError::InvalidParam); + } + let required = Self::required_metadata_size(heap_size, cpu_count); + if meta_size < required { + return Err(AllocError::InvalidParam); + } + + // --- Partition metadata region --- + let buddy_meta_size = BuddyAllocator::::required_meta_size(heap_size); + let buddy_meta_ptr = meta; + + let slab_align = core::mem::align_of::>>(); + let slab_offset = align_up(buddy_meta_size, slab_align); + let slab_ptr = meta.add(slab_offset) as *mut SpinMutex>; + + // --- Init buddy --- + let mut buddy = self.buddy.lock(); + buddy.init( + buddy_meta_ptr, + buddy_meta_size, + heap_start, + heap_size, + Some(os), + )?; + drop(buddy); + + // --- Init per-CPU slabs --- + for i in 0..cpu_count { + let slot = slab_ptr.add(i); + slot.write(SpinMutex::new(SlabAllocator::new())); + } + + // --- Write fields via interior mutability (self is &self for GlobalAlloc compat) --- + let self_mut = self as *const Self as *mut Self; + (*self_mut).per_cpu_slabs = slab_ptr; + (*self_mut).cpu_count = cpu_count; + (*self_mut).os = Some(os); + self.initialized.store(true, Ordering::Release); + + log::debug!( + "GlobalAllocator: {} CPUs, heap {:#x}+{:#x}, meta {:#x}+{:#x}", + cpu_count, + heap_start, + heap_size, + meta as usize, + meta_size, + ); + + Ok(()) + } + + // ------------------------------------------------------------------ + // Public page-level API (forwarded to buddy) + // ------------------------------------------------------------------ + + /// Allocate contiguous pages. Returns the virtual start address. + pub fn alloc_pages(&self, count: usize, align: usize) -> AllocResult { + self.buddy.lock().alloc_pages(count, align) + } + + /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). + pub fn dealloc_pages(&self, addr: usize, count: usize) { + self.buddy.lock().dealloc_pages(addr, count); + } + + /// Allocate pages with physical address below 4 GiB. + pub fn alloc_pages_lowmem(&self, count: usize, align: usize) -> AllocResult { + self.buddy.lock().alloc_pages_lowmem(count, align) + } + + // ------------------------------------------------------------------ + // Object-level API (slab or buddy fallback) + // ------------------------------------------------------------------ + + /// Allocate memory for `layout`. Returns a pointer on success. + pub fn alloc(&self, layout: Layout) -> AllocResult> { + if !self.initialized.load(Ordering::Acquire) { + return Err(AllocError::NotInitialized); + } + + if self.is_slab_eligible(&layout) { + self.slab_alloc(layout) + } else { + self.large_alloc(layout) + } + } + + /// Deallocate memory previously returned by [`alloc`](Self::alloc). + /// + /// # Safety + /// `ptr` must have been returned by a prior `alloc` with the same `layout`. + pub unsafe fn dealloc(&self, ptr: NonNull, layout: Layout) { + if self.is_slab_eligible(&layout) { + self.slab_dealloc(ptr, layout); + } else { + self.large_dealloc(ptr, layout); + } + } + + // ------------------------------------------------------------------ + // Internal: slab path + // ------------------------------------------------------------------ + + #[inline] + fn is_slab_eligible(&self, layout: &Layout) -> bool { + layout.size() <= SLAB_MAX_SIZE && layout.align() <= SLAB_MAX_SIZE + } + + fn slab_alloc(&self, layout: Layout) -> AllocResult> { + let os = self.os.ok_or(AllocError::NotInitialized)?; + let cpu = os.current_cpu_idx(); + debug_assert!(cpu < self.cpu_count); + + let slab_lock = unsafe { &*self.per_cpu_slabs.add(cpu) }; + let mut slab = slab_lock.lock(); + + match slab.alloc(layout)? { + SlabAllocResult::Allocated(ptr) => Ok(ptr), + SlabAllocResult::NeedsSlab { size_class, pages } => { + drop(slab); // release slab lock before locking buddy + let bytes = pages * PAGE_SIZE; + // Align to slab_bytes so base_from_obj_addr works via address masking. + let addr = self.buddy.lock().alloc_pages(pages, bytes)?; + // Tag the page as SLAB in buddy metadata. + unsafe { + self.buddy.lock().set_page_flags(addr, PageFlags::Slab); + } + let mut slab = slab_lock.lock(); + slab.add_slab(size_class, addr, bytes, cpu as u16); + match slab.alloc(layout)? { + SlabAllocResult::Allocated(ptr) => Ok(ptr), + SlabAllocResult::NeedsSlab { .. } => Err(AllocError::NoMemory), + } + } + } + } + + unsafe fn slab_dealloc(&self, ptr: NonNull, layout: Layout) { + let os = self.os.expect("not initialized"); + let sc = SizeClass::from_layout(layout).expect("layout exceeds slab"); + let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr(ptr.as_ptr() as usize, slab_bytes); + let hdr = &*(base as *const SlabPageHeader); + debug_assert_eq!(hdr.magic, SLAB_MAGIC); + + let owner_cpu = hdr.owner_cpu as usize; + let current_cpu = os.current_cpu_idx(); + + if owner_cpu == current_cpu { + // Local free path (under lock). + let slab_lock = &*self.per_cpu_slabs.add(current_cpu); + let mut slab = slab_lock.lock(); + match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + drop(slab); + self.buddy.lock().set_page_flags(base, PageFlags::Free); + self.buddy.lock().dealloc_pages(base, pages); + } + } + } else { + // Remote free path (lock-free CAS). + hdr.remote_free(ptr.as_ptr() as usize); + } + } + + // ------------------------------------------------------------------ + // Internal: large allocation (buddy pages) + // ------------------------------------------------------------------ + + fn large_alloc(&self, layout: Layout) -> AllocResult> { + let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE; + let align = layout.align().max(PAGE_SIZE); + let addr = self.buddy.lock().alloc_pages(pages, align)?; + // SAFETY: buddy returns non-null, page-aligned addresses. + Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) }) + } + + unsafe fn large_dealloc(&self, ptr: NonNull, layout: Layout) { + let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE; + self.buddy + .lock() + .dealloc_pages(ptr.as_ptr() as usize, pages); + } +} + +// --------------------------------------------------------------------------- +// GlobalAlloc implementation +// --------------------------------------------------------------------------- + +unsafe impl GlobalAlloc for GlobalAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + match self.alloc(layout) { + Ok(ptr) => ptr.as_ptr(), + Err(_) => ptr::null_mut(), + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if let Some(nn) = NonNull::new(ptr) { + self.dealloc(nn, layout); + } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new_layout = match Layout::from_size_align(new_size, layout.align()) { + Ok(l) => l, + Err(_) => return ptr::null_mut(), + }; + + let new_ptr = ::alloc(self, new_layout); + if !new_ptr.is_null() { + let copy_size = layout.size().min(new_size); + ptr::copy_nonoverlapping(ptr, new_ptr, copy_size); + ::dealloc(self, ptr, layout); + } + new_ptr + } +} diff --git a/src/global_allocator.rs b/src/global_allocator.rs deleted file mode 100644 index dd39c57..0000000 --- a/src/global_allocator.rs +++ /dev/null @@ -1,554 +0,0 @@ -//! Global allocator implementation. -//! -//! This module implements a multi-core global allocator that coordinates -//! between a shared page allocator backend and per-CPU slab frontends. - -extern crate alloc; - -use crate::os::set_os_provider; -use crate::slab::slab_node::SlabNode; -use crate::slab::{SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision}; -use crate::{AllocError, AllocResult}; -use core::alloc::Layout; -use core::mem::{align_of, size_of}; -use core::ptr::NonNull; -use core::slice; -#[cfg(feature = "tracking")] -use core::sync::atomic::AtomicUsize; -use core::sync::atomic::{AtomicBool, Ordering}; -use log::error; -use spin::Mutex; - -#[cfg(feature = "tracking")] -use super::buddy::BuddyStats; -use super::page_allocator::CompositePageAllocator; - -const MIN_HEAP_SIZE: usize = 0x8000; - -/// Memory usage statistics. -#[cfg(feature = "tracking")] -#[derive(Debug, Clone, Copy, Default)] -pub struct UsageStats { - pub total_pages: usize, - pub used_pages: usize, - pub free_pages: usize, - pub slab_bytes: usize, - pub heap_bytes: usize, -} - -#[cfg(feature = "tracking")] -struct UsageStatsAtomic { - total_pages: AtomicUsize, - used_pages: AtomicUsize, - free_pages: AtomicUsize, - slab_bytes: AtomicUsize, - heap_bytes: AtomicUsize, -} - -#[cfg(feature = "tracking")] -impl UsageStatsAtomic { - const fn new() -> Self { - Self { - total_pages: AtomicUsize::new(0), - used_pages: AtomicUsize::new(0), - free_pages: AtomicUsize::new(0), - slab_bytes: AtomicUsize::new(0), - heap_bytes: AtomicUsize::new(0), - } - } - - fn snapshot(&self) -> UsageStats { - UsageStats { - total_pages: self.total_pages.load(Ordering::Relaxed), - used_pages: self.used_pages.load(Ordering::Relaxed), - free_pages: self.free_pages.load(Ordering::Relaxed), - slab_bytes: self.slab_bytes.load(Ordering::Relaxed), - heap_bytes: self.heap_bytes.load(Ordering::Relaxed), - } - } -} - -#[cfg(feature = "tracking")] -#[inline] -fn saturating_sub_atomic(counter: &AtomicUsize, value: usize) { - let mut prev = counter.load(Ordering::Relaxed); - loop { - let new = prev.saturating_sub(value); - match counter.compare_exchange(prev, new, Ordering::AcqRel, Ordering::Relaxed) { - Ok(_) => break, - Err(actual) => prev = actual, - } - } -} - -#[derive(Clone, Copy, Default)] -struct MetadataRegionInfo { - start: usize, - size: usize, -} - -struct PerCpuSlabSlot { - slab: Mutex>, -} - -impl PerCpuSlabSlot { - const fn new() -> Self { - Self { - slab: Mutex::new(SlabByteAllocator::new()), - } - } -} - -/// Multi-core global allocator facade. -pub struct GlobalAllocator { - buddy: Mutex>, - slab_slots_ptr: Option>>, - cpu_count: usize, - metadata_region: MetadataRegionInfo, - os: &'static dyn crate::Os, - #[cfg(feature = "tracking")] - stats: UsageStatsAtomic, - initialized: AtomicBool, -} - -impl GlobalAllocator { - pub const fn new() -> Self { - Self { - buddy: Mutex::new(CompositePageAllocator::::new()), - slab_slots_ptr: None, - cpu_count: 0, - metadata_region: MetadataRegionInfo { start: 0, size: 0 }, - os: &crate::NoImplOs, - #[cfg(feature = "tracking")] - stats: UsageStatsAtomic::new(), - initialized: AtomicBool::new(false), - } - } - - pub fn required_metadata_size(cpu_count: usize) -> usize { - if cpu_count == 0 { - return 0; - } - let slot_size = size_of::>(); - let slot_align = align_of::>(); - let size = slot_size.saturating_mul(cpu_count); - crate::align_up(size, slot_align) - } - - pub fn required_metadata_align() -> usize { - align_of::>() - } - - /// Set the address translator so that the underlying page allocator can - /// reason about physical address ranges (e.g. low-memory regions below 4GiB). - pub fn set_addr_translator(&self, translator: &'static dyn crate::AddrTranslator) { - self.buddy.lock().set_addr_translator(translator); - } - - fn slab_slots(&self) -> &[PerCpuSlabSlot] { - let ptr = self - .slab_slots_ptr - .expect("global allocator slab slots accessed before init"); - unsafe { slice::from_raw_parts(ptr.as_ptr(), self.cpu_count) } - } - - fn slab_slot(&self, cpu_id: usize) -> AllocResult<&PerCpuSlabSlot> { - if !self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::NoMemory); - } - if cpu_id >= self.cpu_count { - return Err(AllocError::InvalidParam); - } - Ok(&self.slab_slots()[cpu_id]) - } - - fn current_cpu_idx_checked(&self) -> AllocResult { - let cpu_id = self.os.current_cpu_idx(); - if cpu_id >= self.cpu_count { - error!( - "global allocator: OS reported cpu_id {} >= cpu_count {}", - cpu_id, self.cpu_count - ); - return Err(AllocError::InvalidParam); - } - Ok(cpu_id) - } - - fn owner_cpu_from_ptr(&self, ptr: NonNull, layout: Layout) -> Option { - let size_class = SizeClass::from_layout(layout)?; - let slab_bytes = size_class.slab_bytes(PAGE_SIZE); - let slab_base = (ptr.as_ptr() as usize / slab_bytes) * slab_bytes; - let node = SlabNode::new(slab_base, size_class); - if node.is_valid_for_size_class() { - Some(node.owner_cpu()) - } else { - None - } - } - - fn update_page_stats_after_alloc(&self, num_pages: usize) { - #[cfg(feature = "tracking")] - { - self.stats - .used_pages - .fetch_add(num_pages, Ordering::Relaxed); - self.stats - .free_pages - .fetch_sub(num_pages, Ordering::Relaxed); - } - #[cfg(not(feature = "tracking"))] - let _ = num_pages; - } - - fn update_page_stats_after_free(&self, num_pages: usize) { - #[cfg(feature = "tracking")] - { - saturating_sub_atomic(&self.stats.used_pages, num_pages); - self.stats - .free_pages - .fetch_add(num_pages, Ordering::Relaxed); - } - #[cfg(not(feature = "tracking"))] - let _ = num_pages; - } - - /// Initialize the allocator with metadata storage, memory region, CPU topology and OS hooks. - /// - /// # Safety - /// - /// The caller must ensure that `[meta_start, meta_start + meta_size)` is valid writable memory - /// for the lifetime of the allocator and is not used for any other purpose. - pub unsafe fn init( - &mut self, - meta_start: usize, - meta_size: usize, - heap_start: usize, - heap_size: usize, - cpu_count: usize, - os: &'static dyn crate::Os, - ) -> AllocResult<()> { - if self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::InvalidParam); - } - if cpu_count == 0 || heap_size <= MIN_HEAP_SIZE { - return Err(AllocError::InvalidParam); - } - if !meta_start.is_multiple_of(Self::required_metadata_align()) { - return Err(AllocError::InvalidParam); - } - if meta_size < Self::required_metadata_size(cpu_count) { - return Err(AllocError::NoMemory); - } - - let slots_ptr = meta_start as *mut PerCpuSlabSlot; - for idx in 0..cpu_count { - slots_ptr.add(idx).write(PerCpuSlabSlot::new()); - } - - self.slab_slots_ptr = Some(NonNull::new(slots_ptr).ok_or(AllocError::InvalidParam)?); - self.cpu_count = cpu_count; - self.metadata_region = MetadataRegionInfo { - start: meta_start, - size: meta_size, - }; - self.os = os; - - self.buddy.get_mut().init(heap_start, heap_size); - - #[cfg(feature = "tracking")] - { - let buddy = self.buddy.get_mut(); - self.stats - .total_pages - .store(buddy.total_pages(), Ordering::Relaxed); - self.stats - .used_pages - .store(buddy.used_pages(), Ordering::Relaxed); - self.stats - .free_pages - .store(buddy.available_pages(), Ordering::Relaxed); - } - - set_os_provider(os); - self.initialized.store(true, Ordering::SeqCst); - Ok(()) - } - - pub fn add_memory(&self, start_vaddr: usize, size: usize) -> AllocResult<()> { - if !self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::NoMemory); - } - - let mut buddy = self.buddy.lock(); - buddy.add_memory(start_vaddr, size)?; - - #[cfg(feature = "tracking")] - { - self.stats - .total_pages - .store(buddy.total_pages(), Ordering::Relaxed); - self.stats - .free_pages - .store(buddy.available_pages(), Ordering::Relaxed); - } - - Ok(()) - } - - pub fn alloc_dma32_pages(&self, num_pages: usize, alignment: usize) -> AllocResult { - if !self.initialized.load(Ordering::SeqCst) { - error!("global allocator: Allocator not initialized"); - return Err(AllocError::NoMemory); - } - - let mut buddy = self.buddy.lock(); - let addr = buddy.alloc_pages_lowmem(num_pages, alignment)?; - self.update_page_stats_after_alloc(num_pages); - Ok(addr) - } - - pub fn alloc(&self, layout: Layout) -> AllocResult> { - if !self.initialized.load(Ordering::SeqCst) { - error!("global allocator: Allocator not initialized"); - return Err(AllocError::NoMemory); - } - - if layout.size() <= 2048 && layout.align() <= 2048 { - let cpu_id = self.current_cpu_idx_checked()?; - - loop { - let decision = { - let slot = self.slab_slot(cpu_id)?; - let mut slab = slot.slab.lock(); - slab.alloc(layout)? - }; - - match decision { - SlabAllocDecision::Allocated(ptr, _) => { - #[cfg(feature = "tracking")] - { - self.stats - .slab_bytes - .fetch_add(layout.size(), Ordering::Relaxed); - } - return Ok(ptr); - } - SlabAllocDecision::NeedsRefill { - size_class, - page_count, - slab_bytes, - } => { - let slab_base = { - let mut buddy = self.buddy.lock(); - let addr = buddy.alloc_pages(page_count, slab_bytes)?; - self.update_page_stats_after_alloc(page_count); - addr - }; - - let slot = self.slab_slot(cpu_id)?; - let mut slab = slot.slab.lock(); - slab.provide_slab(size_class, cpu_id, slab_base, slab_bytes)?; - } - } - } - } - - let pages_needed = layout.size().div_ceil(PAGE_SIZE); - let addr = { - let mut buddy = self.buddy.lock(); - buddy.alloc_pages(pages_needed, layout.align())? - }; - self.update_page_stats_after_alloc(pages_needed); - #[cfg(feature = "tracking")] - { - self.stats - .heap_bytes - .fetch_add(layout.size(), Ordering::Relaxed); - } - Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) }) - } - - pub fn alloc_pages(&self, num_pages: usize, alignment: usize) -> AllocResult { - if !self.initialized.load(Ordering::SeqCst) { - return Err(AllocError::NoMemory); - } - - let addr = { - let mut buddy = self.buddy.lock(); - buddy.alloc_pages(num_pages, alignment)? - }; - self.update_page_stats_after_alloc(num_pages); - Ok(addr) - } - - pub fn dealloc(&self, ptr: NonNull, layout: Layout) { - if !self.initialized.load(Ordering::SeqCst) { - error!("global allocator: Deallocating memory before initializing"); - return; - } - - if layout.size() <= 2048 && layout.align() <= 2048 { - let owner_cpu = self - .owner_cpu_from_ptr(ptr, layout) - .unwrap_or_else(|| self.os.current_cpu_idx()); - if let Ok(slot) = self.slab_slot(owner_cpu) { - let decision = { - let mut slab = slot.slab.lock(); - let decision = slab.dealloc(ptr, layout); - #[cfg(feature = "tracking")] - { - let actually_deallocated = match &decision { - SlabDeallocDecision::Done { - actually_deallocated, - .. - } - | SlabDeallocDecision::ReleaseSlab { - actually_deallocated, - .. - } => *actually_deallocated, - }; - if actually_deallocated { - saturating_sub_atomic(&self.stats.slab_bytes, layout.size()); - } - } - decision - }; - - if let SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - .. - } = decision - { - let mut buddy = self.buddy.lock(); - buddy.dealloc_pages(slab_base, page_count); - self.update_page_stats_after_free(page_count); - } - } - return; - } - - let pages_needed = layout.size().div_ceil(PAGE_SIZE); - { - let mut buddy = self.buddy.lock(); - buddy.dealloc_pages(ptr.as_ptr() as usize, pages_needed); - } - self.update_page_stats_after_free(pages_needed); - #[cfg(feature = "tracking")] - { - saturating_sub_atomic(&self.stats.heap_bytes, layout.size()); - } - } - - pub fn dealloc_pages(&self, pos: usize, num_pages: usize) { - if !self.initialized.load(Ordering::SeqCst) { - return; - } - - { - let mut buddy = self.buddy.lock(); - buddy.dealloc_pages(pos, num_pages); - } - self.update_page_stats_after_free(num_pages); - } - - #[allow(clippy::not_unsafe_ptr_arg_deref)] - pub fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if new_size == 0 { - if let Some(ptr) = NonNull::new(ptr) { - self.dealloc(ptr, layout); - } - return core::ptr::null_mut(); - } - - if ptr.is_null() { - let new_layout = Layout::from_size_align(new_size, layout.align()) - .unwrap_or_else(|_| Layout::new::()); - return match self.alloc(new_layout) { - Ok(ptr) => ptr.as_ptr(), - Err(_) => core::ptr::null_mut(), - }; - } - - if new_size <= layout.size() { - return ptr; - } - - let new_layout = Layout::from_size_align(new_size, layout.align()) - .unwrap_or_else(|_| Layout::new::()); - match self.alloc(new_layout) { - Ok(new_ptr) => { - let new_ptr = new_ptr.as_ptr(); - unsafe { - core::ptr::copy_nonoverlapping( - ptr, - new_ptr, - core::cmp::min(layout.size(), new_size), - ); - } - if let Some(ptr) = NonNull::new(ptr) { - self.dealloc(ptr, layout); - } - new_ptr - } - Err(_) => core::ptr::null_mut(), - } - } - - pub fn total_pages(&self) -> usize { - #[cfg(feature = "tracking")] - { - return self.stats.total_pages.load(Ordering::Relaxed); - } - #[cfg(not(feature = "tracking"))] - { - self.buddy.lock().total_pages() - } - } - - pub fn used_pages(&self) -> usize { - #[cfg(feature = "tracking")] - { - return self.stats.used_pages.load(Ordering::Relaxed); - } - #[cfg(not(feature = "tracking"))] - { - self.buddy.lock().used_pages() - } - } - - pub fn available_pages(&self) -> usize { - #[cfg(feature = "tracking")] - { - return self.stats.free_pages.load(Ordering::Relaxed); - } - #[cfg(not(feature = "tracking"))] - { - self.buddy.lock().available_pages() - } - } - - pub fn cpu_count(&self) -> usize { - self.cpu_count - } - - pub fn metadata_region(&self) -> (usize, usize) { - (self.metadata_region.start, self.metadata_region.size) - } - - #[cfg(feature = "tracking")] - pub fn get_stats(&self) -> UsageStats { - self.stats.snapshot() - } - - #[cfg(feature = "tracking")] - pub fn get_buddy_stats(&self) -> BuddyStats { - self.buddy.lock().get_buddy_stats() - } -} - -impl Default for GlobalAllocator { - fn default() -> Self { - Self::new() - } -} diff --git a/src/lib.rs b/src/lib.rs index 25a3469..f8d19be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,179 +1,55 @@ -//! buddy-slab-allocator Memory Allocator +//! # buddy-slab-allocator //! -//! This crate implements a high-performance memory allocator designed for embedded -//! and kernel environments, featuring: -//! - Buddy page allocator for page-level allocation -//! - Slab allocator for small object allocation -//! - Global allocator coordination -//! - Zero `std` dependency (fully `#![no_std]`) +//! A `#![no_std]` memory allocator featuring: //! -//! # Features +//! - **Buddy page allocator** — page-metadata-based with intrusive free lists +//! - **Slab allocator** — bitmap-based with lock-free cross-CPU freeing (Linux SLUB inspired) +//! - **Global allocator** — composes buddy + per-CPU slab, implements [`core::alloc::GlobalAlloc`] //! -//! - **Buddy Page Allocator**: Efficient page-level memory allocation with automatic merging -//! - **Slab Byte Allocator**: Fast small object allocation (≤2048 bytes) -//! - **Global Allocator**: Automatic selection between page and slab allocation based on size -//! - **No_std Compatible**: Fully `#![no_std]` for embedded/kernel use -//! - **Logging Support**: Integrates with the `log` crate for allocation diagnostics -//! - **Memory Tracking**: Detailed statistics with `tracking` feature -//! -//! # Quick Start -//! -//! ```no_run -//! use buddy_slab_allocator::GlobalAllocator; -//! use core::alloc::Layout; -//! -//! const PAGE_SIZE: usize = 0x1000; -//! let mut allocator = GlobalAllocator::::new(); -//! struct DemoOs; -//! impl buddy_slab_allocator::Os for DemoOs { -//! fn current_cpu_idx(&self) -> usize { 0 } -//! } -//! static OS: DemoOs = DemoOs; -//! -//! // Initialize with memory region -//! let heap_start = 0x8000_0000; -//! let heap_size = 16 * 1024 * 1024; // 16MB -//! let meta_start = 0x8100_0000; -//! let meta_size = GlobalAllocator::::required_metadata_size(1); -//! unsafe { allocator.init(meta_start, meta_size, heap_start, heap_size, 1, &OS).unwrap(); } -//! -//! // Allocate pages -//! let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); -//! // Use the allocated memory... -//! allocator.dealloc_pages(addr, 4); -//! ``` -//! -//! # Small Object Allocation -//! -//! ```no_run -//! use buddy_slab_allocator::GlobalAllocator; -//! use core::alloc::Layout; -//! -//! const PAGE_SIZE: usize = 0x1000; -//! let mut allocator = GlobalAllocator::::new(); -//! struct DemoOs; -//! impl buddy_slab_allocator::Os for DemoOs { -//! fn current_cpu_idx(&self) -> usize { 0 } -//! } -//! static OS: DemoOs = DemoOs; -//! let meta_start = 0x8100_0000; -//! let meta_size = GlobalAllocator::::required_metadata_size(1); -//! unsafe { -//! allocator -//! .init(meta_start, meta_size, 0x8000_0000, 16 * 1024 * 1024, 1, &OS) -//! .unwrap(); -//! } -//! -//! // Small allocations go through slab allocator -//! let layout = Layout::from_size_align(64, 8).unwrap(); -//! let ptr = allocator.alloc(layout).unwrap(); -//! // Use the allocated memory... -//! allocator.dealloc(ptr, layout); -//! ``` -//! -//! # Statistics Tracking -//! -//! ```no_run -//! # #[cfg(feature = "tracking")] -//! # { -//! use buddy_slab_allocator::GlobalAllocator; -//! -//! const PAGE_SIZE: usize = 0x1000; -//! let mut allocator = GlobalAllocator::::new(); -//! struct DemoOs; -//! impl buddy_slab_allocator::Os for DemoOs { -//! fn current_cpu_idx(&self) -> usize { 0 } -//! } -//! static OS: DemoOs = DemoOs; -//! let meta_start = 0x8100_0000; -//! let meta_size = GlobalAllocator::::required_metadata_size(1); -//! unsafe { -//! allocator -//! .init(meta_start, meta_size, 0x8000_0000, 16 * 1024 * 1024, 1, &OS) -//! .unwrap(); -//! } -//! -//! let stats = allocator.get_stats(); -//! println!("Total pages: {}", stats.total_pages); -//! println!("Used pages: {}", stats.used_pages); -//! println!("Free pages: {}", stats.free_pages); -//! # } -//! ``` +//! Both buddy and slab allocators can be used standalone. #![no_std] -extern crate alloc; - mod error; pub use error::{AllocError, AllocResult}; -mod os; -pub use os::{current_cpu_idx, NoImplOs, Os}; -/// Default page size for backward compatibility (4KB) -pub const DEFAULT_PAGE_SIZE: usize = 0x1000; +pub mod buddy; +pub use buddy::BuddyAllocator; -/// Address translator used by allocators to reason about physical addresses. -/// -/// Implementations should provide a stable virtual-to-physical mapping -/// for the allocator-managed address range. -/// -/// # Examples -/// -/// ``` -/// use buddy_slab_allocator::AddrTranslator; -/// -/// struct SimpleMapper; +pub mod slab; +pub use slab::{SizeClass, SlabAllocResult, SlabAllocator, SlabDeallocResult}; + +pub mod global; +pub use global::GlobalAllocator; + +// --------------------------------------------------------------------------- +// OsImpl trait — the only interface the allocator needs from the OS / platform +// --------------------------------------------------------------------------- + +/// Platform abstraction required by [`GlobalAllocator`]. /// -/// impl AddrTranslator for SimpleMapper { -/// fn virt_to_phys(&self, va: usize) -> Option { -/// // Identity mapping for this example -/// Some(va) -/// } -/// } -/// ``` -pub trait AddrTranslator: Sync { +/// Implementations must be safe to call from any CPU at any time. +pub trait OsImpl: Sync + Send { + /// Return the index of the current CPU (0-based). + fn current_cpu_idx(&self) -> usize; + /// Translate a virtual address to a physical address. - /// - /// Returns `None` if the address is not valid or not mapped. - fn virt_to_phys(&self, va: usize) -> Option; -} + fn virt_to_phys(&self, vaddr: usize) -> usize; -#[inline] -#[allow(dead_code)] -const fn align_down(pos: usize, align: usize) -> usize { - pos & !(align - 1) + /// Translate a physical address to a virtual address. + fn phys_to_virt(&self, paddr: usize) -> usize; } +// --------------------------------------------------------------------------- +// Utility helpers (crate-internal) +// --------------------------------------------------------------------------- + #[inline] -#[allow(dead_code)] -const fn align_up(pos: usize, align: usize) -> usize { +pub(crate) const fn align_up(pos: usize, align: usize) -> usize { (pos + align - 1) & !(align - 1) } -/// Checks whether the address has the demanded alignment. -/// -/// Equivalent to `addr % align == 0`, but the alignment must be a power of two. #[inline] -#[allow(dead_code)] -const fn is_aligned(base_addr: usize, align: usize) -> bool { - base_addr & (align - 1) == 0 +pub(crate) const fn is_aligned(addr: usize, align: usize) -> bool { + addr & (align - 1) == 0 } - -// Export our allocator implementations -pub mod buddy; -#[cfg(feature = "tracking")] -pub use buddy::BuddyStats; -pub use buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER, MAX_ZONES}; - -pub mod page_allocator; -pub use page_allocator::CompositePageAllocator; - -pub mod slab; -pub use slab::slab_byte_allocator::{ - SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, -}; - -pub mod global_allocator; -pub use global_allocator::GlobalAllocator; -#[cfg(feature = "tracking")] -pub use global_allocator::UsageStats; diff --git a/src/os.rs b/src/os.rs deleted file mode 100644 index 14b73a9..0000000 --- a/src/os.rs +++ /dev/null @@ -1,24 +0,0 @@ -use spin::RwLock; - -pub trait Os: Sync { - fn current_cpu_idx(&self) -> usize; -} - -pub struct NoImplOs; - -impl Os for NoImplOs { - fn current_cpu_idx(&self) -> usize { - panic!("buddy-slab-allocator: Os provider is not initialized") - } -} - -static NO_IMPL_OS: NoImplOs = NoImplOs; -static OS_PROVIDER: RwLock<&'static dyn Os> = RwLock::new(&NO_IMPL_OS); - -pub(crate) fn set_os_provider(os: &'static dyn Os) { - *OS_PROVIDER.write() = os; -} - -pub fn current_cpu_idx() -> usize { - OS_PROVIDER.read().current_cpu_idx() -} diff --git a/src/page_allocator.rs b/src/page_allocator.rs deleted file mode 100644 index e064e9a..0000000 --- a/src/page_allocator.rs +++ /dev/null @@ -1,528 +0,0 @@ -//! Page allocator with contiguous block combination support. - -use crate::buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER}; -use crate::{AllocError, AllocResult}; - -use log::{debug, warn}; - -/// Maximum number of buddy blocks in a single contiguous allocation -const MAX_PARTS_PER_ALLOC: usize = 8; - -/// Maximum number of concurrent composite allocations tracked -const MAX_COMPOSITE_ALLOCS: usize = 16; - -/// Metadata for a composite allocation (multiple buddy blocks combined) -#[derive(Clone, Copy, Debug)] -struct CompositeBlockInfo { - /// Base address of the composite allocation - base_addr: usize, - /// Number of parts in this composite allocation - part_count: u8, - /// Individual parts: (address, order) - parts: [(usize, u32); MAX_PARTS_PER_ALLOC], -} - -/// Tracker for composite allocations using fixed-size array -struct CompositeBlockTracker { - /// Array to store composite block metadata - blocks: [Option; MAX_COMPOSITE_ALLOCS], - /// Number of active composite allocations - count: usize, -} - -impl CompositeBlockTracker { - const fn new() -> Self { - Self { - blocks: [None; MAX_COMPOSITE_ALLOCS], - count: 0, - } - } - - /// Insert a new composite block - /// - /// Returns false if the tracker is full - fn insert(&mut self, base_addr: usize, parts: &[(usize, u32)], part_count: usize) -> bool { - if self.count >= MAX_COMPOSITE_ALLOCS { - return false; - } - - let mut info = CompositeBlockInfo { - base_addr, - part_count: part_count as u8, - parts: [(0, 0); MAX_PARTS_PER_ALLOC], - }; - - info.parts[..part_count].copy_from_slice(&parts[..part_count]); - - self.blocks[self.count] = Some(info); - self.count += 1; - true - } - - /// Find a composite block by its base address - fn find(&self, base_addr: usize) -> Option { - for i in 0..self.count { - if let Some(info) = self.blocks[i] { - if info.base_addr == base_addr { - return Some(info); - } - } - } - None - } - - /// Remove a composite block by its base address - /// - /// Returns true if the block was found and removed - fn remove(&mut self, base_addr: usize) -> bool { - for i in 0..self.count { - if let Some(info) = self.blocks[i] { - if info.base_addr == base_addr { - // Move the last element to this position to keep array compact - if i < self.count - 1 { - self.blocks[i] = self.blocks[self.count - 1]; - } - self.blocks[self.count - 1] = None; - self.count -= 1; - return true; - } - } - } - false - } -} - -pub struct CompositePageAllocator { - /// Underlying buddy allocator for standard allocations - buddy: BuddyPageAllocator, - /// Tracker for composite allocations (multiple buddy blocks combined) - composite_tracker: CompositeBlockTracker, -} - -impl CompositePageAllocator { - /// Create a new page allocator with contiguous block support - pub const fn new() -> Self { - Self { - buddy: BuddyPageAllocator::::new(), - composite_tracker: CompositeBlockTracker::new(), - } - } - - /// Set the address translator so that the underlying buddy allocator can - /// reason about physical address ranges (e.g. low-memory regions). - pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) { - self.buddy.set_addr_translator(translator); - } - - /// Allocate low-memory pages (physical address < 4GiB). - /// This is a thin wrapper over the buddy allocator's lowmem allocation. - pub fn alloc_pages_lowmem(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - self.buddy.alloc_pages_lowmem(num_pages, alignment) - } - - /// Initialize the allocator with a free memory region. - pub fn init(&mut self, start: usize, size: usize) { - self.buddy.init(start, size); - } - - /// Add a free memory region to the allocator. - pub fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> { - self.buddy.add_memory(start, size) - } - - /// Try to find and allocate contiguous small blocks from buddy free lists. - /// - /// This method searches buddy free lists for contiguous blocks that can satisfy - /// the allocation request. It uses the sorted nature of free lists to efficiently - /// check for contiguity. - /// - /// # Algorithm - /// 1. Iterate through free lists from largest to smallest blocks - /// 2. For each block, check if it's contiguous with already collected blocks - /// 3. Collect contiguous blocks until we have enough pages - /// 4. If successful, allocate all collected blocks - /// - /// # Returns - /// Base address of the first block if contiguous blocks found, otherwise None - fn try_combine_contiguous_blocks( - &mut self, - num_pages: usize, - alignment: usize, - ) -> Option { - let mut remaining_pages = num_pages; - let mut contiguous_blocks: [(usize, u32); MAX_PARTS_PER_ALLOC] = - [(0, 0); MAX_PARTS_PER_ALLOC]; - let mut block_count = 0; - let mut min_addr = usize::MAX; - let mut max_addr = 0; - - // Iterate from largest to smallest blocks - for order in (0..=DEFAULT_MAX_ORDER).rev() { - let block_pages = 1usize << order; - - if remaining_pages == 0 || block_count >= MAX_PARTS_PER_ALLOC { - break; - } - - // Get free blocks of this order from all zones - for zone_id in 0..self.buddy.get_zone_count() { - if let Some(blocks) = self.buddy.get_free_blocks_by_order(zone_id, order as u32) { - // Iterate through sorted free blocks - for block in blocks { - if block_count >= MAX_PARTS_PER_ALLOC { - break; - } - - let block_start = block.addr; - let block_end = block_start + block_pages * PAGE_SIZE; - - // Check alignment requirement - if !crate::is_aligned(block_start, alignment) { - continue; - } - - // Check contiguity with existing blocks - if block_count == 0 { - // First block - just record it - contiguous_blocks[block_count] = (block_start, order as u32); - min_addr = block_start; - max_addr = block_end; - block_count += 1; - remaining_pages -= block_pages.min(remaining_pages); - } else if block_end == min_addr { - // Block is to the left, update min_addr - contiguous_blocks[block_count] = (block_start, order as u32); - min_addr = block_start; - block_count += 1; - remaining_pages -= block_pages.min(remaining_pages); - } else if block_start == max_addr { - // Block is to the right, update max_addr - contiguous_blocks[block_count] = (block_start, order as u32); - max_addr = block_end; - block_count += 1; - remaining_pages -= block_pages.min(remaining_pages); - } - - if remaining_pages == 0 { - break; - } - } - } - - if remaining_pages == 0 { - break; - } - } - } - - // If we found enough contiguous pages, allocate them - if remaining_pages == 0 { - let mut parts = [(0usize, 0u32); MAX_PARTS_PER_ALLOC]; - - // Allocate all contiguous blocks - for i in 0..block_count { - let (addr, order) = contiguous_blocks[i]; - let block_pages = 1usize << order; - - debug!( - "Block {}: addr={:#x}, order={}, pages={}, size={} MB", - i, - addr, - order, - block_pages, - (block_pages * PAGE_SIZE) / (1024 * 1024) - ); - - // Allocate this specific block - if let Err(_e) = self.buddy.alloc_pages(block_pages, alignment) { - // Allocation failed, rollback - warn!("Contiguous block allocation failed at {i}, rolling back"); - #[allow(clippy::needless_range_loop)] - for j in 0..i { - let (dealloc_addr, dealloc_order) = parts[j]; - let dealloc_pages = 1usize << dealloc_order; - self.buddy.dealloc_pages(dealloc_addr, dealloc_pages); - } - return None; - } - - parts[i] = (addr, order); - } - - // Assertion: allocated pages must be >= requested pages - let actual_pages: usize = parts[..block_count] - .iter() - .map(|(_, order)| 1usize << *order as usize) - .sum(); - debug_assert!( - actual_pages >= num_pages, - "Allocated pages {actual_pages} < requested pages {num_pages}" - ); - - // Save metadata to tracker for proper deallocation - if !self - .composite_tracker - .insert(min_addr, &parts[..block_count], block_count) - { - // Tracker is full, rollback and fail - warn!("Composite tracker full, rolling back allocation"); - #[allow(clippy::needless_range_loop)] - for j in 0..block_count { - let (dealloc_addr, dealloc_order) = parts[j]; - let dealloc_pages = 1usize << dealloc_order; - self.buddy.dealloc_pages(dealloc_addr, dealloc_pages); - } - return None; - } - - debug!("Contiguous block allocation succeeded: base_addr={min_addr:#x}, pages={num_pages}, parts={block_count}, actual_pages={actual_pages}"); - - return Some(min_addr); - } - - None - } - - /// Print detailed statistics when allocation fails. - /// - /// This function delegates to buddy allocator's detailed statistics reporter. - #[cfg(feature = "tracking")] - fn print_alloc_failure_stats(&self, num_pages: usize, alignment: usize) { - self.buddy.print_alloc_failure_stats(num_pages, alignment); - } - - #[cfg(not(feature = "tracking"))] - fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) { - // No-op when tracking is disabled - } - - pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - if num_pages == 0 { - return Err(AllocError::InvalidParam); - } - - let buddy_pages = if num_pages.is_power_of_two() { - num_pages - } else { - num_pages.next_power_of_two() - }; - - // Try to allocate from buddy system first - let base_addr = match self.buddy.alloc_pages(buddy_pages, alignment) { - Ok(addr) => addr, - Err(_) => { - // Standard allocation failed, try contiguous block combination - debug!( - "Standard allocation failed, trying contiguous block combination for {num_pages} pages" - ); - if let Some(addr) = self.try_combine_contiguous_blocks(num_pages, alignment) { - return Ok(addr); - } - self.print_alloc_failure_stats(num_pages, alignment); - return Err(AllocError::NoMemory); - } - }; - - Ok(base_addr) - } - - pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - if num_pages == 0 { - return; - } - - // Check if this is a composite allocation - if let Some(info) = self.composite_tracker.find(pos) { - // Composite block: deallocate each part separately - debug!( - "Deallocating composite block: base_addr={:#x}, part_count={}", - pos, info.part_count - ); - - for i in 0..info.part_count as usize { - let (addr, order) = info.parts[i]; - let pages = 1usize << order; - debug!(" Part {i}: addr={addr:#x}, order={order}, pages={pages}"); - self.buddy.dealloc_pages(addr, pages); - } - - // Remove from tracker - self.composite_tracker.remove(pos); - } else { - // Regular buddy block: deallocate directly - if num_pages.is_power_of_two() { - self.buddy.dealloc_pages(pos, num_pages); - } else { - self.buddy.dealloc_pages(pos, num_pages.next_power_of_two()); - } - } - } - - /// Return total number of memory pages. - pub fn total_pages(&self) -> usize { - self.buddy.total_pages() - } - - /// Return number of allocated memory pages. - pub fn used_pages(&self) -> usize { - self.buddy.used_pages() - } - - /// Return number of available memory pages. - pub fn available_pages(&self) -> usize { - self.buddy.available_pages() - } -} - -impl CompositePageAllocator { - /// Get buddy allocator statistics - #[cfg(feature = "tracking")] - pub fn get_buddy_stats(&self) -> crate::buddy::BuddyStats { - self.buddy.get_stats() - } -} - -impl Default for CompositePageAllocator { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_composite_block_tracker() { - let mut tracker = CompositeBlockTracker::new(); - - // Test insertion - let parts = [(0x1000, 3), (0x2000, 2)]; - assert!(tracker.insert(0x1000, &parts, 2)); - assert_eq!(tracker.count, 1); - - // Test finding - let info = tracker.find(0x1000); - assert!(info.is_some()); - let info = info.unwrap(); - assert_eq!(info.base_addr, 0x1000); - assert_eq!(info.part_count, 2); - assert_eq!(info.parts[0], (0x1000, 3)); - assert_eq!(info.parts[1], (0x2000, 2)); - - // Test removal - assert!(tracker.remove(0x1000)); - assert_eq!(tracker.count, 0); - assert!(tracker.find(0x1000).is_none()); - } - - #[test] - fn test_composite_block_tracker_capacity() { - let mut tracker = CompositeBlockTracker::new(); - - // Fill the tracker to capacity - for i in 0..MAX_COMPOSITE_ALLOCS { - let parts = [(0x1000 + i * 0x1000, 3)]; - assert!(tracker.insert(0x1000 + i * 0x1000, &parts, 1)); - } - - assert_eq!(tracker.count, MAX_COMPOSITE_ALLOCS); - - // Try to insert one more - should fail - let parts = [(0x10000, 3)]; - assert!(!tracker.insert(0x10000, &parts, 1)); - - // Remove one and verify we can insert again - assert!(tracker.remove(0x1000)); - assert!(tracker.insert(0x10000, &parts, 1)); - } - - #[test] - fn test_composite_tracker_find_nonexistent() { - let tracker = CompositeBlockTracker::new(); - assert!(tracker.find(0x1000).is_none()); - } - - #[test] - fn test_composite_tracker_remove_nonexistent() { - let mut tracker = CompositeBlockTracker::new(); - assert!(!tracker.remove(0x1000)); - } - - #[test] - fn test_composite_tracker_multiple_blocks() { - let mut tracker = CompositeBlockTracker::new(); - - let parts1 = [(0x1000, 3), (0x2000, 2)]; - let parts2 = [(0x3000, 4), (0x4000, 3), (0x5000, 2)]; - - tracker.insert(0x1000, &parts1, 2); - tracker.insert(0x3000, &parts2, 3); - - assert_eq!(tracker.count, 2); - - let info1 = tracker.find(0x1000); - assert!(info1.is_some()); - assert_eq!(info1.unwrap().part_count, 2); - - let info2 = tracker.find(0x3000); - assert!(info2.is_some()); - assert_eq!(info2.unwrap().part_count, 3); - } -} - -#[cfg(test)] -mod unit_tests { - use super::*; - use alloc::alloc::{alloc, dealloc}; - use core::alloc::Layout; - - const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; - const TEST_PAGE_SIZE: usize = 0x1000; - - fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, TEST_PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null()); - (ptr, layout) - } - - fn dealloc_test_heap(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; - } - - #[test] - fn test_composite_allocator_basic() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr1 = allocator.alloc_pages(1, TEST_PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_pages(4, TEST_PAGE_SIZE).unwrap(); - - assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); - assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); - - allocator.dealloc_pages(addr1, 1); - allocator.dealloc_pages(addr2, 4); - - dealloc_test_heap(heap_ptr, heap_layout); - } - - #[test] - fn test_composite_allocator_alignment() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr = allocator.alloc_pages(1, TEST_PAGE_SIZE * 4).unwrap(); - assert_eq!(addr & (TEST_PAGE_SIZE * 4 - 1), 0); - - allocator.dealloc_pages(addr, 1); - dealloc_test_heap(heap_ptr, heap_layout); - } -} diff --git a/src/slab/cache.rs b/src/slab/cache.rs new file mode 100644 index 0000000..ae334aa --- /dev/null +++ b/src/slab/cache.rs @@ -0,0 +1,207 @@ +/// Per-size-class slab cache. +/// +/// Maintains three intrusive doubly-linked lists of slab pages: +/// - **partial**: some objects free (preferred for allocation) +/// - **full**: no objects free +/// - **empty**: all objects free (at most one cached; rest returned to buddy) +use super::page::SlabPageHeader; +use super::size_class::SizeClass; + +/// Intrusive list head (address of the first `SlabPageHeader`, 0 = empty). +#[derive(Debug, Clone, Copy)] +struct ListHead { + first: usize, +} + +impl ListHead { + const fn empty() -> Self { + Self { first: 0 } + } + + fn is_empty(&self) -> bool { + self.first == 0 + } + + /// Push a slab page onto the front of the list. + /// + /// # Safety + /// `base` must point to a valid `SlabPageHeader`. + unsafe fn push_front(&mut self, base: usize) { + let hdr = &mut *(base as *mut SlabPageHeader); + hdr.list_prev = 0; + hdr.list_next = self.first; + if self.first != 0 { + let old = &mut *(self.first as *mut SlabPageHeader); + old.list_prev = base; + } + self.first = base; + } + + /// Remove a slab page from this list. + /// + /// # Safety + /// `base` must be in this list. + unsafe fn remove(&mut self, base: usize) { + let hdr = &*(base as *const SlabPageHeader); + let prev = hdr.list_prev; + let next = hdr.list_next; + + if prev != 0 { + (*(prev as *mut SlabPageHeader)).list_next = next; + } else { + self.first = next; + } + if next != 0 { + (*(next as *mut SlabPageHeader)).list_prev = prev; + } + // Clear links + let hdr = &mut *(base as *mut SlabPageHeader); + hdr.list_prev = 0; + hdr.list_next = 0; + } + + /// Pop the first page from the list. Returns 0 if empty. + unsafe fn pop_front(&mut self) -> usize { + if self.first == 0 { + return 0; + } + let base = self.first; + self.remove(base); + base + } +} + +/// Cache for a single [`SizeClass`]. +pub struct SlabCache { + pub size_class: SizeClass, + partial: ListHead, + full: ListHead, + empty: ListHead, + /// Number of empty slabs cached (we keep at most 1). + empty_count: usize, +} + +/// Result of a per-cache deallocation. +pub enum CacheDeallocResult { + /// Object freed, slab stays. + Done, + /// Slab became empty and should be returned to the page allocator. + FreeSlab { base: usize, pages: usize }, +} + +impl SlabCache { + pub const fn new(size_class: SizeClass) -> Self { + Self { + size_class, + partial: ListHead::empty(), + full: ListHead::empty(), + empty: ListHead::empty(), + empty_count: 0, + } + } + + /// Try to allocate one object. Returns `Some(obj_addr)` or `None` if no slabs available. + pub fn alloc_object(&mut self) -> Option { + // 1. Try the first partial slab (drain remote frees first). + if let Some(addr) = self.try_alloc_from_partial::() { + return Some(addr); + } + + // 2. Try recycling an empty slab. + if !self.empty.is_empty() { + let base = unsafe { self.empty.pop_front() }; + self.empty_count -= 1; + // Move to partial and alloc from it. + unsafe { self.partial.push_front(base) }; + return self.try_alloc_from_partial::(); + } + + None + } + + /// Attempt allocation from the first partial slab. + fn try_alloc_from_partial(&mut self) -> Option { + let base = self.partial.first; + if base == 0 { + return None; + } + + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + + // Drain any remote frees first. + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + } + + if let Some(idx) = hdr.local_alloc() { + let obj_addr = hdr.object_addr(base, idx); + // If slab is now full, move to full list. + if hdr.is_local_full() && !hdr.has_remote_frees() { + unsafe { + self.partial.remove(base); + self.full.push_front(base); + } + } + return Some(obj_addr); + } + None + } + + /// Free an object back to this cache (local CPU path — under lock). + /// + /// Returns whether the slab should be returned to the page allocator. + pub fn dealloc_object( + &mut self, + obj_addr: usize, + ) -> CacheDeallocResult { + let slab_bytes = self.size_class.slab_pages(PAGE_SIZE) * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr(obj_addr, slab_bytes); + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + let was_full = hdr.is_local_full() && !hdr.has_remote_frees(); + + let idx = hdr.object_index(base, obj_addr); + hdr.local_free(idx); + + if was_full { + // Move from full to partial. + unsafe { + self.full.remove(base); + self.partial.push_front(base); + } + } + + // Check if slab is now completely empty. + // First drain remote frees so we have an accurate count. + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + } + + if hdr.is_all_free() { + if self.empty_count == 0 { + // Cache one empty slab for reuse. + unsafe { + self.partial.remove(base); + self.empty.push_front(base); + } + self.empty_count += 1; + CacheDeallocResult::Done + } else { + // Already have a cached empty slab — return this one. + unsafe { self.partial.remove(base) }; + CacheDeallocResult::FreeSlab { + base, + pages: self.size_class.slab_pages(PAGE_SIZE), + } + } + } else { + CacheDeallocResult::Done + } + } + + /// Register a newly allocated slab page (from the buddy allocator). + pub fn add_slab(&mut self, base: usize, bytes: usize, owner_cpu: u16) { + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + hdr.init(self.size_class, bytes, owner_cpu); + unsafe { self.partial.push_front(base) }; + } +} diff --git a/src/slab/mod.rs b/src/slab/mod.rs index 3c4ae9d..3233733 100644 --- a/src/slab/mod.rs +++ b/src/slab/mod.rs @@ -1,13 +1,106 @@ -//! Slab allocator implementation. +//! Slab allocator — bitmap-based with lock-free cross-CPU freeing. //! -//! This module implements an improved slab allocator for small object allocation -//! with pooled linked lists, inspired by asterinas design. +//! The [`SlabAllocator`] is a standalone component that manages object allocation +//! within pre-supplied slab pages. It does **not** allocate pages itself; instead +//! it returns [`SlabAllocResult::NeedsSlab`] to request pages from the caller. +//! +//! Cross-CPU frees go through the lock-free [`SlabPageHeader::remote_free`] path. + +pub mod cache; +pub mod page; +pub mod size_class; + +pub use page::SlabPageHeader; +pub use size_class::SizeClass; + +use cache::{CacheDeallocResult, SlabCache}; +use core::alloc::Layout; +use core::ptr::NonNull; + +use crate::error::{AllocError, AllocResult}; + +/// Result of a slab allocation attempt. +pub enum SlabAllocResult { + /// Object successfully allocated. + Allocated(NonNull), + /// The slab cache for this size class has no free objects. + /// The caller should allocate `pages` pages from the buddy allocator, + /// call [`SlabAllocator::add_slab`], and retry. + NeedsSlab { size_class: SizeClass, pages: usize }, +} + +/// Result of a slab deallocation. +pub enum SlabDeallocResult { + /// Object freed, nothing else to do. + Done, + /// The slab page at `base` became empty and should be returned to the buddy. + FreeSlab { base: usize, pages: usize }, +} + +/// Standalone slab allocator (one per CPU or standalone use). +pub struct SlabAllocator { + caches: [SlabCache; SizeClass::COUNT], +} + +impl SlabAllocator { + /// Create a new (empty) slab allocator. No pages are owned yet. + pub const fn new() -> Self { + Self { + caches: [ + SlabCache::new(SizeClass::Bytes8), + SlabCache::new(SizeClass::Bytes16), + SlabCache::new(SizeClass::Bytes32), + SlabCache::new(SizeClass::Bytes64), + SlabCache::new(SizeClass::Bytes128), + SlabCache::new(SizeClass::Bytes256), + SlabCache::new(SizeClass::Bytes512), + SlabCache::new(SizeClass::Bytes1024), + SlabCache::new(SizeClass::Bytes2048), + ], + } + } + + /// Try to allocate an object matching `layout`. + /// + /// If the matching cache is exhausted, [`SlabAllocResult::NeedsSlab`] is returned + /// so the caller can supply pages and retry. + pub fn alloc(&mut self, layout: Layout) -> AllocResult { + let sc = SizeClass::from_layout(layout).ok_or(AllocError::InvalidParam)?; + let cache = &mut self.caches[sc.index()]; + + match cache.alloc_object::() { + Some(addr) => { + // SAFETY: `addr` is non-null, aligned, and within a live slab page. + let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; + Ok(SlabAllocResult::Allocated(ptr)) + } + None => Ok(SlabAllocResult::NeedsSlab { + size_class: sc, + pages: sc.slab_pages(PAGE_SIZE), + }), + } + } + + /// Free an object previously allocated with [`alloc`](Self::alloc). + /// + /// This is the **local** (owner-CPU) path. Cross-CPU frees should go through + /// [`SlabPageHeader::remote_free`] directly (see [`GlobalAllocator`]). + pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) -> SlabDeallocResult { + let sc = SizeClass::from_layout(layout).expect("layout exceeds slab size"); + let cache = &mut self.caches[sc.index()]; -pub mod slab_byte_allocator; -pub mod slab_cache; -pub mod slab_node; + match cache.dealloc_object::(ptr.as_ptr() as usize) { + CacheDeallocResult::Done => SlabDeallocResult::Done, + CacheDeallocResult::FreeSlab { base, pages } => { + SlabDeallocResult::FreeSlab { base, pages } + } + } + } -// Re-export public types -pub use slab_byte_allocator::{ - SizeClass, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, -}; + /// Supply a freshly allocated slab page to the given size class. + /// + /// `base` is the virtual address of the page(s), `bytes` = pages × PAGE_SIZE. + pub fn add_slab(&mut self, size_class: SizeClass, base: usize, bytes: usize, owner_cpu: u16) { + self.caches[size_class.index()].add_slab(base, bytes, owner_cpu); + } +} diff --git a/src/slab/page.rs b/src/slab/page.rs new file mode 100644 index 0000000..fe88754 --- /dev/null +++ b/src/slab/page.rs @@ -0,0 +1,226 @@ +/// Slab page header, bitmap-based object tracking, and lock-free remote free. +/// +/// Each slab page starts with a [`SlabPageHeader`] followed by the object array. +/// Local (owner-CPU) operations use a bitmap under the slab lock. +/// Remote (cross-CPU) frees use an atomic CAS stack — no lock required. +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use super::size_class::SizeClass; + +/// Magic number written at the start of every slab page header. +pub const SLAB_MAGIC: u32 = 0x534C_4142; // "SLAB" + +/// Maximum objects per slab page (512 = 8 × u64 bitmap words). +pub const MAX_OBJECTS_PER_SLAB: usize = 512; + +/// Number of u64 words in the local bitmap. +pub const BITMAP_WORDS: usize = MAX_OBJECTS_PER_SLAB / 64; // 8 + +/// Header placed at the very start of each slab page. +/// +/// Object data starts at `header_end`, aligned to `size_class.size()`. +#[repr(C)] +pub struct SlabPageHeader { + /// Magic number for integrity checks. + pub magic: u32, + /// Which size class this slab serves. + pub size_class: SizeClass, + /// Total number of objects that fit. + pub object_count: u16, + /// Number of objects free in `local_bitmap`. + pub local_free_count: u16, + /// The CPU that owns this slab page (local alloc/dealloc go through this CPU's lock). + pub owner_cpu: u16, + _pad: u16, + /// Total usable bytes (slab pages × PAGE_SIZE). + pub slab_bytes: u32, + + // --- Intrusive doubly-linked list pointers (used by SlabCache) --- + pub list_prev: usize, + pub list_next: usize, + + // --- Local bitmap (under slab lock, owner CPU only) --- + // A set bit means the slot is FREE. + pub local_bitmap: [u64; BITMAP_WORDS], + + // --- Lock-free remote free stack (any CPU) --- + /// Head of the remote-free linked list (object virtual address, 0 = empty). + /// Each freed object stores `next` at its start (reuses object memory). + pub remote_free_head: AtomicUsize, + /// Number of objects in the remote-free stack. + pub remote_free_count: AtomicU32, +} + +impl SlabPageHeader { + /// Size of the header in bytes. + pub const HEADER_SIZE: usize = core::mem::size_of::(); + + /// Initialise a slab page header. All objects are marked free in the bitmap. + /// + /// `base` is the virtual address of the page, `bytes` is the total slab size. + pub fn init(&mut self, size_class: SizeClass, bytes: usize, owner_cpu: u16) { + let obj_size = size_class.size(); + let data_start = Self::data_offset(obj_size); + let usable = bytes.saturating_sub(data_start); + let count = (usable / obj_size).min(MAX_OBJECTS_PER_SLAB); + + self.magic = SLAB_MAGIC; + self.size_class = size_class; + self.object_count = count as u16; + self.local_free_count = count as u16; + self.owner_cpu = owner_cpu; + self._pad = 0; + self.slab_bytes = bytes as u32; + self.list_prev = 0; + self.list_next = 0; + self.local_bitmap = [0u64; BITMAP_WORDS]; + self.remote_free_head = AtomicUsize::new(0); + self.remote_free_count = AtomicU32::new(0); + + // Mark first `count` bits as 1 (free). + let full_words = count / 64; + let remaining_bits = count % 64; + for w in self.local_bitmap.iter_mut().take(full_words) { + *w = u64::MAX; + } + if remaining_bits > 0 { + self.local_bitmap[full_words] = (1u64 << remaining_bits) - 1; + } + } + + /// Byte offset from the start of the page to the first object. + /// Aligned up to `obj_size` for natural alignment. + pub fn data_offset(obj_size: usize) -> usize { + let raw = Self::HEADER_SIZE; + // Align to obj_size (which is always a power of two). + (raw + obj_size - 1) & !(obj_size - 1) + } + + /// Virtual address of the object region start (given `base` = page start). + #[inline] + pub fn data_start(&self, base: usize) -> usize { + base + Self::data_offset(self.size_class.size()) + } + + /// Virtual address of object at `index`. + #[inline] + pub fn object_addr(&self, base: usize, index: usize) -> usize { + self.data_start(base) + index * self.size_class.size() + } + + /// Index of the object whose address is `addr`. + #[inline] + pub fn object_index(&self, base: usize, addr: usize) -> usize { + (addr - self.data_start(base)) / self.size_class.size() + } + + /// Base address (page start) from an object address. + /// Relies on slab_bytes being a power-of-two multiple of PAGE_SIZE. + #[inline] + pub fn base_from_obj_addr(addr: usize, slab_bytes: usize) -> usize { + addr & !(slab_bytes - 1) + } + + // ------------------------------------------------------------------ + // Local allocation (under slab lock) + // ------------------------------------------------------------------ + + /// Allocate one object from the local bitmap. Returns the slot index or `None`. + pub fn local_alloc(&mut self) -> Option { + for (wi, word) in self.local_bitmap.iter_mut().enumerate() { + if *word != 0 { + let bit = word.trailing_zeros() as usize; + *word &= !(1u64 << bit); + self.local_free_count -= 1; + return Some(wi * 64 + bit); + } + } + None + } + + /// Free an object back to the local bitmap. + pub fn local_free(&mut self, index: usize) { + let wi = index / 64; + let bit = index % 64; + debug_assert!(self.local_bitmap[wi] & (1u64 << bit) == 0, "double free"); + self.local_bitmap[wi] |= 1u64 << bit; + self.local_free_count += 1; + } + + /// Whether this slab has any free objects (local bitmap only). + #[inline] + pub fn has_local_free(&self) -> bool { + self.local_free_count > 0 + } + + /// Whether every object in this slab is free (local bitmap only). + #[inline] + pub fn is_all_free(&self) -> bool { + self.local_free_count == self.object_count + } + + /// Whether the local bitmap is completely full (zero free objects locally). + #[inline] + pub fn is_local_full(&self) -> bool { + self.local_free_count == 0 + } + + // ------------------------------------------------------------------ + // Remote free (lock-free, any CPU) + // ------------------------------------------------------------------ + + /// Push `obj_addr` onto the remote-free stack (lock-free CAS). + /// + /// # Safety + /// - `obj_addr` must point to a previously allocated object within this slab. + /// - The object's first `size_of::()` bytes will be overwritten with + /// the next-pointer. + pub unsafe fn remote_free(&self, obj_addr: usize) { + loop { + let old_head = self.remote_free_head.load(Ordering::Acquire); + // Store "next" pointer inside the freed object. + (obj_addr as *mut usize).write(old_head); + if self + .remote_free_head + .compare_exchange_weak(old_head, obj_addr, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.remote_free_count.fetch_add(1, Ordering::Relaxed); + return; + } + } + } + + /// Drain all remote frees back into the local bitmap. + /// + /// Must be called under the owner-CPU slab lock. + pub fn drain_remote_frees(&mut self, base: usize) { + let head = self.remote_free_head.swap(0, Ordering::AcqRel); + if head == 0 { + return; + } + // Also zero the count (we'll re-add to local). + self.remote_free_count.store(0, Ordering::Relaxed); + + let mut ptr = head; + while ptr != 0 { + let next = unsafe { *(ptr as *const usize) }; + let idx = self.object_index(base, ptr); + let wi = idx / 64; + let bit = idx % 64; + debug_assert!( + self.local_bitmap[wi] & (1u64 << bit) == 0, + "remote double free" + ); + self.local_bitmap[wi] |= 1u64 << bit; + self.local_free_count += 1; + ptr = next; + } + } + + /// Whether any remote frees are pending. + #[inline] + pub fn has_remote_frees(&self) -> bool { + self.remote_free_count.load(Ordering::Relaxed) > 0 + } +} diff --git a/src/slab/size_class.rs b/src/slab/size_class.rs new file mode 100644 index 0000000..b3d5015 --- /dev/null +++ b/src/slab/size_class.rs @@ -0,0 +1,95 @@ +/// Object size classes for the slab allocator. +/// +/// Each size class corresponds to a fixed object size. +/// Allocations are rounded up to the nearest size class. +use core::alloc::Layout; + +/// Number of distinct size classes. +pub const SIZE_CLASS_COUNT: usize = 9; + +/// Fixed set of object sizes served by the slab allocator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum SizeClass { + Bytes8 = 0, + Bytes16 = 1, + Bytes32 = 2, + Bytes64 = 3, + Bytes128 = 4, + Bytes256 = 5, + Bytes512 = 6, + Bytes1024 = 7, + Bytes2048 = 8, +} + +/// Maximum object size handled by the slab. +pub const SLAB_MAX_SIZE: usize = 2048; + +/// Ordered table of (object_size, index) for all classes. +const CLASS_SIZES: [usize; SIZE_CLASS_COUNT] = [8, 16, 32, 64, 128, 256, 512, 1024, 2048]; + +impl SizeClass { + /// All size classes in ascending order. + pub const ALL: [SizeClass; SIZE_CLASS_COUNT] = [ + SizeClass::Bytes8, + SizeClass::Bytes16, + SizeClass::Bytes32, + SizeClass::Bytes64, + SizeClass::Bytes128, + SizeClass::Bytes256, + SizeClass::Bytes512, + SizeClass::Bytes1024, + SizeClass::Bytes2048, + ]; + + /// Number of distinct size classes. + pub const COUNT: usize = SIZE_CLASS_COUNT; + + /// Select the smallest size class that can satisfy `layout`. + /// + /// Returns `None` if the requested size or alignment exceeds the slab's capability. + pub fn from_layout(layout: Layout) -> Option { + let size = layout.size().max(layout.align()); + if size > SLAB_MAX_SIZE { + return None; + } + for (i, &class_size) in CLASS_SIZES.iter().enumerate() { + if size <= class_size { + return Some(SizeClass::ALL[i]); + } + } + None + } + + /// Object size in bytes. + pub const fn size(self) -> usize { + CLASS_SIZES[self as usize] + } + + /// Array index (0-based). + pub const fn index(self) -> usize { + self as usize + } + + /// How many pages are needed for a single slab of this class. + /// + /// Smaller classes use 1 page, larger classes may use more to amortise the + /// per-page header overhead. + pub const fn slab_pages(self, page_size: usize) -> usize { + let obj_size = self.size(); + if obj_size <= 256 { + 1 + } else if obj_size <= 1024 { + 2 + } else { + // 2048-byte objects: 4 pages → header + room for objects + let v = 16 * page_size / (obj_size * 8); + let v = if v < 4 { v } else { 4 }; + if v < 1 { + 1 + } else { + v + } + } + } +} diff --git a/src/slab/slab_byte_allocator.rs b/src/slab/slab_byte_allocator.rs deleted file mode 100644 index 3733b20..0000000 --- a/src/slab/slab_byte_allocator.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Slab byte allocator implementation. -//! -//! This module implements an improved slab allocator for small object allocation -//! with pooled linked lists, inspired by asterinas design. - -use core::alloc::Layout; -use core::ptr::NonNull; - -use log::warn; - -use crate::{AllocError, AllocResult}; - -use super::slab_cache::CacheDeallocDecision; -pub use super::slab_cache::SlabCache; -pub use super::slab_node::SlabNode; - -/// Size classes for slab allocation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(usize)] -pub enum SizeClass { - Bytes8 = 8, - Bytes16 = 16, - Bytes32 = 32, - Bytes64 = 64, - Bytes128 = 128, - Bytes256 = 256, - Bytes512 = 512, - Bytes1024 = 1024, - Bytes2048 = 2048, -} - -impl SizeClass { - pub const COUNT: usize = 9; - const MAX_OBJ_SIZE: usize = 2048; - - pub fn from_layout(layout: Layout) -> Option { - let required_size = layout.size().max(layout.align()); - - if required_size > Self::MAX_OBJ_SIZE { - warn!( - "Invalid layout: size={}, align={}", - layout.size(), - layout.align() - ); - return None; - } - - Some(match required_size { - 0..=8 => SizeClass::Bytes8, - 9..=16 => SizeClass::Bytes16, - 17..=32 => SizeClass::Bytes32, - 33..=64 => SizeClass::Bytes64, - 65..=128 => SizeClass::Bytes128, - 129..=256 => SizeClass::Bytes256, - 257..=512 => SizeClass::Bytes512, - 513..=1024 => SizeClass::Bytes1024, - 1025..=2048 => SizeClass::Bytes2048, - _ => unreachable!( - "Invalid layout: size={}, align={}", - layout.size(), - layout.align() - ), - }) - } - - pub fn size(&self) -> usize { - *self as usize - } - - pub fn to_index(&self) -> usize { - match self { - SizeClass::Bytes8 => 0, - SizeClass::Bytes16 => 1, - SizeClass::Bytes32 => 2, - SizeClass::Bytes64 => 3, - SizeClass::Bytes128 => 4, - SizeClass::Bytes256 => 5, - SizeClass::Bytes512 => 6, - SizeClass::Bytes1024 => 7, - SizeClass::Bytes2048 => 8, - } - } - - pub fn page_count(&self, page_size: usize) -> usize { - SlabNode::new(0, *self).page_count(page_size) - } - - pub fn slab_bytes(&self, page_size: usize) -> usize { - self.page_count(page_size) * page_size - } -} - -pub enum SlabAllocDecision { - Allocated(NonNull, usize), - NeedsRefill { - size_class: SizeClass, - page_count: usize, - slab_bytes: usize, - }, -} - -pub enum SlabDeallocDecision { - Done { - bytes_released: usize, - actually_deallocated: bool, - }, - ReleaseSlab { - slab_base: usize, - page_count: usize, - slab_bytes: usize, - actually_deallocated: bool, - }, -} - -/// Slab byte allocator with pooled linked lists. -pub struct SlabByteAllocator { - caches: [SlabCache; SizeClass::COUNT], - total_bytes: usize, - allocated_bytes: usize, -} - -unsafe impl Send for SlabByteAllocator {} -unsafe impl Sync for SlabByteAllocator {} - -impl SlabByteAllocator { - pub const fn new() -> Self { - Self { - caches: [ - SlabCache::new(SizeClass::Bytes8), - SlabCache::new(SizeClass::Bytes16), - SlabCache::new(SizeClass::Bytes32), - SlabCache::new(SizeClass::Bytes64), - SlabCache::new(SizeClass::Bytes128), - SlabCache::new(SizeClass::Bytes256), - SlabCache::new(SizeClass::Bytes512), - SlabCache::new(SizeClass::Bytes1024), - SlabCache::new(SizeClass::Bytes2048), - ], - total_bytes: 0, - allocated_bytes: 0, - } - } - - pub fn alloc(&mut self, layout: Layout) -> AllocResult { - let size_class = SizeClass::from_layout(layout).ok_or(AllocError::InvalidParam)?; - let cache = &mut self.caches[size_class.to_index()]; - - if let Some(obj_addr) = cache.alloc_object()? { - self.allocated_bytes += layout.size().max(layout.align()); - return Ok(SlabAllocDecision::Allocated( - unsafe { NonNull::new_unchecked(obj_addr as *mut u8) }, - 0, - )); - } - - Ok(SlabAllocDecision::NeedsRefill { - size_class, - page_count: size_class.page_count(PAGE_SIZE), - slab_bytes: size_class.slab_bytes(PAGE_SIZE), - }) - } - - pub fn provide_slab( - &mut self, - size_class: SizeClass, - owner_cpu: usize, - slab_base: usize, - slab_bytes: usize, - ) -> AllocResult<()> { - self.caches[size_class.to_index()].provide_slab(slab_base, slab_bytes, owner_cpu)?; - self.total_bytes += slab_bytes; - Ok(()) - } - - pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) -> SlabDeallocDecision { - let Some(size_class) = SizeClass::from_layout(layout) else { - warn!( - "Invalid layout during slab dealloc: size={}, align={}", - layout.size(), - layout.align() - ); - return SlabDeallocDecision::Done { - bytes_released: 0, - actually_deallocated: false, - }; - }; - - let decision = - self.caches[size_class.to_index()].dealloc_object(ptr.as_ptr() as usize, PAGE_SIZE); - - match decision { - CacheDeallocDecision::Done { - bytes_released, - actually_deallocated, - } => { - if actually_deallocated { - self.allocated_bytes = self - .allocated_bytes - .saturating_sub(layout.size().max(layout.align())); - } - SlabDeallocDecision::Done { - bytes_released, - actually_deallocated, - } - } - CacheDeallocDecision::ReleaseSlab { - slab_base, - page_count, - slab_bytes, - actually_deallocated, - } => { - if actually_deallocated { - self.allocated_bytes = self - .allocated_bytes - .saturating_sub(layout.size().max(layout.align())); - } - self.total_bytes = self.total_bytes.saturating_sub(slab_bytes); - SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - slab_bytes, - actually_deallocated, - } - } - } - } - - pub fn owner_cpu_of(&self, ptr: NonNull, layout: Layout) -> Option { - let size_class = SizeClass::from_layout(layout)?; - let slab_bytes = size_class.slab_bytes(PAGE_SIZE); - let slab_base = (ptr.as_ptr() as usize / slab_bytes) * slab_bytes; - let node = SlabNode::new(slab_base, size_class); - if node.is_valid_for_size_class() { - Some(node.owner_cpu()) - } else { - None - } - } - - pub fn total_bytes(&self) -> usize { - self.total_bytes - } - - pub fn used_bytes(&self) -> usize { - self.allocated_bytes - } - - pub fn available_bytes(&self) -> usize { - self.total_bytes.saturating_sub(self.allocated_bytes) - } -} - -impl Default for SlabByteAllocator { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_size_class() { - assert_eq!( - SizeClass::from_layout(Layout::from_size_align(8, 8).unwrap()), - Some(SizeClass::Bytes8) - ); - assert_eq!( - SizeClass::from_layout(Layout::from_size_align(16, 8).unwrap()), - Some(SizeClass::Bytes16) - ); - assert_eq!( - SizeClass::from_layout(Layout::from_size_align(2048, 8).unwrap()), - Some(SizeClass::Bytes2048) - ); - assert_eq!( - SizeClass::from_layout(Layout::from_size_align(2049, 8).unwrap()), - None - ); - } - - #[test] - fn test_size_class_boundaries() { - assert_eq!(SizeClass::Bytes8.size(), 8); - assert_eq!(SizeClass::Bytes16.size(), 16); - assert_eq!(SizeClass::Bytes32.size(), 32); - assert_eq!(SizeClass::Bytes64.size(), 64); - assert_eq!(SizeClass::Bytes128.size(), 128); - assert_eq!(SizeClass::Bytes256.size(), 256); - assert_eq!(SizeClass::Bytes512.size(), 512); - assert_eq!(SizeClass::Bytes1024.size(), 1024); - assert_eq!(SizeClass::Bytes2048.size(), 2048); - } - - #[test] - fn test_size_class_alignment_limits() { - assert_eq!( - SizeClass::from_layout(Layout::from_size_align(64, 4096).unwrap()), - None - ); - } -} diff --git a/src/slab/slab_cache.rs b/src/slab/slab_cache.rs deleted file mode 100644 index 3b61496..0000000 --- a/src/slab/slab_cache.rs +++ /dev/null @@ -1,352 +0,0 @@ -//! Slab cache implementation for a single size class. -//! -//! This module implements SlabCache which manages three lists (empty, partial, full) -//! of slab nodes for a specific size class. - -use log::{error, warn}; - -use super::slab_byte_allocator::SizeClass; -use super::slab_node::SlabNode; -use crate::{AllocError, AllocResult}; - -fn align_down_any(pos: usize, align: usize) -> usize { - if align == 0 { - return pos; - } - (pos / align) * align -} - -struct SlabIntrusiveList { - head: Option, - tail: Option, - len: usize, -} - -impl SlabIntrusiveList { - pub const fn new() -> Self { - Self { - head: None, - tail: None, - len: 0, - } - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn back(&self) -> Option { - self.tail - } - - pub fn push_back(&mut self, size_class: SizeClass, slab_base: usize) { - let mut node = SlabNode::new(slab_base, size_class); - node.set_prev(self.tail); - node.set_next(None); - - if let Some(tail) = self.tail { - let mut tail_node = SlabNode::new(tail, size_class); - tail_node.set_next(Some(slab_base)); - } else { - self.head = Some(slab_base); - } - - self.tail = Some(slab_base); - self.len += 1; - } - - pub fn pop_back(&mut self, size_class: SizeClass) -> Option { - let tail = self.tail?; - self.remove(size_class, tail); - Some(tail) - } - - pub fn remove(&mut self, size_class: SizeClass, slab_base: usize) { - let mut node = SlabNode::new(slab_base, size_class); - let prev = node.prev(); - let next = node.next(); - - if let Some(prev_base) = prev { - let mut prev_node = SlabNode::new(prev_base, size_class); - prev_node.set_next(next); - } else { - self.head = next; - } - - if let Some(next_base) = next { - let mut next_node = SlabNode::new(next_base, size_class); - next_node.set_prev(prev); - } else { - self.tail = prev; - } - - node.set_prev(None); - node.set_next(None); - self.len = self.len.saturating_sub(1); - } -} - -/// Internal deallocation result for a single size-class cache. -pub(crate) enum CacheDeallocDecision { - Done { - bytes_released: usize, - actually_deallocated: bool, - }, - ReleaseSlab { - slab_base: usize, - page_count: usize, - slab_bytes: usize, - actually_deallocated: bool, - }, -} - -/// Slab cache for a specific size class. -pub struct SlabCache { - size_class: SizeClass, - empty: SlabIntrusiveList, - partial: SlabIntrusiveList, - full: SlabIntrusiveList, -} - -impl SlabCache { - pub const fn new(size_class: SizeClass) -> Self { - Self { - size_class, - empty: SlabIntrusiveList::new(), - partial: SlabIntrusiveList::new(), - full: SlabIntrusiveList::new(), - } - } - - /// Try to allocate an object from this cache. - /// - /// Returns `Ok(Some(addr))` if allocation succeeds, `Ok(None)` if the cache - /// needs the caller to provide a slab page, or an error if metadata is invalid. - pub fn alloc_object(&mut self) -> AllocResult> { - if let Some(slab_base) = self.partial.back() { - let mut node = SlabNode::new(slab_base, self.size_class); - if !node.is_valid_for_size_class() { - return Err(AllocError::InvalidParam); - } - if let Some(obj_idx) = node.alloc_object() { - let obj_addr = node.object_addr(obj_idx); - if node.is_full() { - self.partial.remove(self.size_class, slab_base); - self.full.push_back(self.size_class, slab_base); - } - return Ok(Some(obj_addr)); - } - panic!( - "Allocation from partial slab failed despite free_count > 0, bitmap inconsistency detected" - ); - } - - if let Some(slab_base) = self.empty.pop_back(self.size_class) { - let mut node = SlabNode::new(slab_base, self.size_class); - if !node.is_valid_for_size_class() { - return Err(AllocError::InvalidParam); - } - if let Some(obj_idx) = node.alloc_object() { - let obj_addr = node.object_addr(obj_idx); - self.partial.push_back(self.size_class, slab_base); - return Ok(Some(obj_addr)); - } - panic!( - "Allocation from empty slab failed despite all objects being free, bitmap inconsistency detected" - ); - } - - Ok(None) - } - - /// Insert a freshly provided slab page into this cache. - pub fn provide_slab( - &mut self, - slab_base: usize, - slab_bytes: usize, - owner_cpu: usize, - ) -> AllocResult<()> { - let mut node = SlabNode::new(slab_base, self.size_class); - node.init_header(slab_bytes, owner_cpu); - if node.free_count() == 0 { - return Err(AllocError::InvalidParam); - } - self.empty.push_back(self.size_class, slab_base); - Ok(()) - } - - /// Deallocate an object from this cache. - pub(crate) fn dealloc_object( - &mut self, - obj_addr: usize, - page_size: usize, - ) -> CacheDeallocDecision { - let object_size = self.size_class.size(); - let bytes_needed = SlabNode::MAX_OBJECTS * object_size; - let page_count = bytes_needed.div_ceil(page_size); - let slab_bytes = page_count * page_size; - - let slab_base = align_down_any(obj_addr, slab_bytes); - let mut node = SlabNode::new(slab_base, self.size_class); - if !node.is_valid_for_size_class() { - warn!( - "slab allocator: Invalid slab base {:#x} for size class {:?}", - slab_base, self.size_class - ); - warn!( - "this can happen if the slab was already returned to the page allocator and the memory was reused, or if the pointer is completely invalid" - ); - return CacheDeallocDecision::Done { - bytes_released: 0, - actually_deallocated: false, - }; - } - - let was_full = node.is_full(); - let (should_release_slab, actually_freed) = - if let Some(obj_idx) = node.object_index_from_addr(obj_addr) { - let actually_freed = node.dealloc_object(obj_idx); - (node.is_empty() && actually_freed, actually_freed) - } else { - error!("Invalid address {obj_addr:x} in slab at {slab_base:x}: not a valid object"); - return CacheDeallocDecision::Done { - bytes_released: 0, - actually_deallocated: true, - }; - }; - - if actually_freed { - if was_full { - self.full.remove(self.size_class, slab_base); - } else { - self.partial.remove(self.size_class, slab_base); - } - - if should_release_slab { - if self.empty.len() >= 2 { - return CacheDeallocDecision::ReleaseSlab { - slab_base, - page_count, - slab_bytes, - actually_deallocated: true, - }; - } - self.empty.push_back(self.size_class, slab_base); - return CacheDeallocDecision::Done { - bytes_released: 0, - actually_deallocated: true, - }; - } - - self.partial.push_back(self.size_class, slab_base); - } - - CacheDeallocDecision::Done { - bytes_released: 0, - actually_deallocated: actually_freed, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::alloc::{alloc, dealloc}; - use core::alloc::Layout; - - use super::super::slab_byte_allocator::SizeClass; - - struct MockPageAllocator { - allocated: alloc::vec::Vec<(usize, Layout, usize)>, - } - - impl MockPageAllocator { - fn new() -> Self { - Self { - allocated: alloc::vec::Vec::new(), - } - } - - fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult { - let size = num_pages * 4096; - let layout = - Layout::from_size_align(size, alignment).map_err(|_| AllocError::InvalidParam)?; - let addr = unsafe { alloc(layout) } as usize; - if addr == 0 { - return Err(AllocError::NoMemory); - } - self.allocated.push((addr, layout, num_pages)); - Ok(addr) - } - - #[allow(dead_code)] - fn dealloc_pages(&mut self, pos: usize, num_pages: usize) { - if let Some(idx) = self - .allocated - .iter() - .position(|&(addr, _layout, count)| addr == pos && count == num_pages) - { - let (_addr, layout, _count) = self.allocated.swap_remove(idx); - unsafe { dealloc(pos as *mut u8, layout) }; - } - } - } - - fn provide_cache_slab(cache: &mut SlabCache, page_allocator: &mut MockPageAllocator) { - let page_count = SlabNode::new(0, SizeClass::Bytes64).page_count(4096); - let slab_bytes = page_count * 4096; - let slab_base = page_allocator.alloc_pages(page_count, slab_bytes).unwrap(); - cache.provide_slab(slab_base, slab_bytes, 0).unwrap(); - } - - #[test] - fn test_alloc_dealloc() { - let mut cache = SlabCache::new(SizeClass::Bytes64); - let mut page_allocator = MockPageAllocator::new(); - provide_cache_slab(&mut cache, &mut page_allocator); - - let obj_addr = cache.alloc_object().unwrap().unwrap(); - assert_ne!(obj_addr, 0); - - match cache.dealloc_object(obj_addr, 4096) { - CacheDeallocDecision::Done { .. } | CacheDeallocDecision::ReleaseSlab { .. } => {} - } - } - - #[test] - fn test_multiple_allocs() { - let mut cache = SlabCache::new(SizeClass::Bytes64); - let mut page_allocator = MockPageAllocator::new(); - provide_cache_slab(&mut cache, &mut page_allocator); - - let mut addrs = alloc::vec::Vec::new(); - for _ in 0..10 { - let addr = cache.alloc_object().unwrap().unwrap(); - addrs.push(addr); - } - - assert_eq!(addrs.len(), 10); - - for addr in addrs { - cache.dealloc_object(addr, 4096); - } - } - - #[test] - fn test_empty_node_management() { - let mut cache = SlabCache::new(SizeClass::Bytes64); - let mut page_allocator = MockPageAllocator::new(); - provide_cache_slab(&mut cache, &mut page_allocator); - - let addr1 = cache.alloc_object().unwrap().unwrap(); - cache.dealloc_object(addr1, 4096); - - let addr2 = cache.alloc_object().unwrap().unwrap(); - cache.dealloc_object(addr2, 4096); - - let addr3 = cache.alloc_object().unwrap().unwrap(); - cache.dealloc_object(addr3, 4096); - - assert!(cache.empty.len() <= 2); - } -} diff --git a/src/slab/slab_node.rs b/src/slab/slab_node.rs deleted file mode 100644 index 49a0286..0000000 --- a/src/slab/slab_node.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Slab node implementation. -//! -//! This module defines the SlabNode structure which manages exactly 512 objects -//! using a fixed bitmap. - -use log::error; - -pub use super::slab_byte_allocator::SizeClass; - -#[repr(C)] -pub(crate) struct SlabHeader { - magic: u32, - size_class: u16, - object_count: u16, - free_count: u16, - _reserved: u16, - owner_cpu: u32, - slab_bytes: usize, - prev: usize, - next: usize, - free_bitmap: [u64; FREE_BITMAP_WORDS], -} - -const SLAB_HEADER_MAGIC: u32 = 0x534c_4142; -const FREE_BITMAP_WORDS: usize = 8; - -#[derive(Debug, Clone, Copy)] -pub struct SlabNode { - pub addr: usize, // Starting physical address - pub size_class: SizeClass, // Size class -} - -impl SlabNode { - pub const MAX_OBJECTS: usize = FREE_BITMAP_WORDS * 64; - - pub const fn new(addr: usize, size_class: SizeClass) -> Self { - Self { addr, size_class } - } - - fn header_size_aligned(&self) -> usize { - crate::align_up(core::mem::size_of::(), self.size_class.size()) - } - - fn object_base(&self) -> usize { - self.addr + self.header_size_aligned() - } - - fn header(&self) -> &SlabHeader { - unsafe { &*(self.addr as *const SlabHeader) } - } - - fn header_mut(&mut self) -> &mut SlabHeader { - unsafe { &mut *(self.addr as *mut SlabHeader) } - } - - pub fn init_header(&mut self, slab_bytes: usize, owner_cpu: usize) { - let object_size = self.size_class.size(); - let header_size_aligned = self.header_size_aligned(); - let object_count = if slab_bytes > header_size_aligned { - (slab_bytes - header_size_aligned) / object_size - } else { - 0 - } - .min(Self::MAX_OBJECTS); - - if object_count == 0 { - return; - } - - let mut free_bitmap = [u64::MAX; FREE_BITMAP_WORDS]; - if object_count < Self::MAX_OBJECTS { - let full_words = object_count / 64; - let rem_bits = object_count % 64; - #[allow(clippy::needless_range_loop)] - for i in 0..FREE_BITMAP_WORDS { - if i < full_words { - continue; - } - if i == full_words { - if rem_bits == 0 { - free_bitmap[i] = 0; - } else { - free_bitmap[i] &= (1u64 << rem_bits) - 1; - } - } else { - free_bitmap[i] = 0; - } - } - } - - let header = self.header_mut(); - *header = SlabHeader { - magic: SLAB_HEADER_MAGIC, - size_class: object_size as u16, - object_count: object_count as u16, - free_count: object_count as u16, - _reserved: 0, - owner_cpu: owner_cpu as u32, - slab_bytes, - prev: 0, - next: 0, - free_bitmap, - }; - } - - pub fn is_valid_for_size_class(&self) -> bool { - let header = self.header(); - header.magic == SLAB_HEADER_MAGIC && header.size_class as usize == self.size_class.size() - } - - pub fn in_use(&self) -> u32 { - let header = self.header(); - header.object_count as u32 - header.free_count as u32 - } - - pub fn free_count(&self) -> u32 { - self.header().free_count as u32 - } - - pub fn is_full(&self) -> bool { - self.header().free_count == 0 - } - - pub fn is_empty(&self) -> bool { - self.header().free_count == self.header().object_count - } - - pub fn alloc_object(&mut self) -> Option { - let header = self.header_mut(); - if header.free_count == 0 { - return None; - } - let object_count = header.object_count as usize; - for (word_idx, &word) in header.free_bitmap.iter().enumerate() { - if word != 0 { - let bit_pos = word.trailing_zeros() as usize; - let object_index = word_idx * 64 + bit_pos; - - if object_index >= object_count { - continue; - } - - header.free_bitmap[word_idx] &= !(1u64 << bit_pos); - header.free_count -= 1; - return Some(object_index); - } - } - None - } - - pub fn dealloc_object(&mut self, object_index: usize) -> bool { - let header = self.header_mut(); - if object_index < header.object_count as usize { - let word_idx = object_index / 64; - let bit_idx = object_index % 64; - let mask = 1u64 << bit_idx; - let was_free = (header.free_bitmap[word_idx] & mask) != 0; - - if was_free { - // Object is already free (double-free), return false to indicate no actual change - return false; - } - - header.free_bitmap[word_idx] |= mask; - header.free_count = header.free_count.saturating_add(1); - true // Object was successfully deallocated - } else { - false - } - } - - pub fn object_addr(&self, object_index: usize) -> usize { - self.object_base() + object_index * self.size_class.size() - } - - pub fn object_index_from_addr(&self, obj_addr: usize) -> Option { - let base = self.object_base(); - if obj_addr < base { - return None; - } - - let offset = obj_addr - base; - if !offset.is_multiple_of(self.size_class.size()) { - error!("Invalid object address: {obj_addr:x}"); - return None; - } - - let object_index = offset / self.size_class.size(); - - if object_index < self.header().object_count as usize { - Some(object_index) - } else { - None - } - } - - pub fn page_count(&self, page_size: usize) -> usize { - let object_size = self.size_class.size(); - let bytes_needed = Self::MAX_OBJECTS * object_size; - bytes_needed.div_ceil(page_size) - } - - pub fn owner_cpu(&self) -> usize { - self.header().owner_cpu as usize - } - - pub fn prev(&self) -> Option { - let prev = self.header().prev; - if prev == 0 { - None - } else { - Some(prev) - } - } - - pub fn next(&self) -> Option { - let next = self.header().next; - if next == 0 { - None - } else { - Some(next) - } - } - - pub fn set_prev(&mut self, prev: Option) { - self.header_mut().prev = prev.unwrap_or(0); - } - - pub fn set_next(&mut self, next: Option) { - self.header_mut().next = next.unwrap_or(0); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::alloc::{alloc, dealloc}; - use core::alloc::Layout; - - #[test] - fn test_slab_node() { - let mut node = SlabNode::new(0, SizeClass::Bytes64); - let slab_pages = node.page_count(4096); - let slab_bytes = slab_pages * 4096; - let layout = Layout::from_size_align(slab_bytes, slab_bytes).unwrap(); - let base = unsafe { alloc(layout) } as usize; - assert_ne!(base, 0); - node.addr = base; - node.init_header(slab_bytes, 0); - - assert!(node.is_empty()); - assert!(!node.is_full()); - assert!(node.free_count() <= SlabNode::MAX_OBJECTS as u32); - assert_eq!(node.in_use(), 0); - - // Test allocation - let obj_idx = node.alloc_object().unwrap(); - assert_eq!(node.object_addr(obj_idx), node.object_base()); - assert_eq!(node.in_use(), 1); - assert_eq!( - node.free_count(), - (node.header().object_count as u32).saturating_sub(1) - ); - - // Test deallocation - node.dealloc_object(obj_idx); - assert!(node.is_empty()); - assert_eq!(node.in_use(), 0); - assert_eq!(node.free_count(), node.header().object_count as u32); - - unsafe { dealloc(base as *mut u8, layout) }; - } - - #[test] - fn test_object_index_from_addr() { - let mut node = SlabNode::new(0, SizeClass::Bytes64); - let slab_pages = node.page_count(4096); - let slab_bytes = slab_pages * 4096; - let layout = Layout::from_size_align(slab_bytes, slab_bytes).unwrap(); - let base = unsafe { alloc(layout) } as usize; - assert_ne!(base, 0); - node.addr = base; - node.init_header(slab_bytes, 0); - - let obj0 = node.object_addr(0); - assert_eq!(node.object_index_from_addr(obj0), Some(0)); - assert_eq!(node.object_index_from_addr(obj0 + 64), Some(1)); - assert_eq!(node.object_index_from_addr(obj0 + 63), None); - assert_eq!(node.object_index_from_addr(obj0 + slab_bytes), None); - - unsafe { dealloc(base as *mut u8, layout) }; - } - - #[test] - fn test_page_count() { - let node8 = SlabNode::new(0, SizeClass::Bytes8); - assert_eq!(node8.page_count(4096), 1); // SlabNode::MAX_OBJECTS * 8 = 4096 - - let node64 = SlabNode::new(0, SizeClass::Bytes64); - assert_eq!(node64.page_count(4096), 8); // SlabNode::MAX_OBJECTS * 64 = 32768 - - let node2048 = SlabNode::new(0, SizeClass::Bytes2048); - assert_eq!(node2048.page_count(4096), 256); // SlabNode::MAX_OBJECTS * 2048 = 1,048,576 - } -} diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index fc9ab5a..8e1364b 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -1,185 +1,131 @@ -//! Tests for GlobalAllocator's alloc_dma32_pages method. +//! Tests for lowmem (DMA32) page allocation via GlobalAllocator. -#![no_std] +extern crate buddy_slab_allocator; -extern crate alloc; - -use buddy_slab_allocator::{AddrTranslator, AllocError, GlobalAllocator, Os}; +use buddy_slab_allocator::{GlobalAllocator, OsImpl}; use core::alloc::Layout; -use core::sync::atomic::{AtomicUsize, Ordering}; +use std::alloc::{alloc, dealloc}; const PAGE_SIZE: usize = 0x1000; const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; -struct MockAddrTranslator; +struct MockOs; -impl AddrTranslator for MockAddrTranslator { - fn virt_to_phys(&self, va: usize) -> Option { - Some(va & 0x7fff_ffff) +impl OsImpl for MockOs { + fn current_cpu_idx(&self) -> usize { + 0 } -} - -struct MockOs { - cpu: AtomicUsize, -} - -impl MockOs { - const fn new() -> Self { - Self { - cpu: AtomicUsize::new(0), - } + fn virt_to_phys(&self, vaddr: usize) -> usize { + // Map virtual addresses below 4 GiB so lowmem checks pass. + vaddr & 0x0FFF_FFFF } -} - -impl Os for MockOs { - fn current_cpu_idx(&self) -> usize { - self.cpu.load(Ordering::Relaxed) + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr } } -static MOCK_TRANSLATOR: MockAddrTranslator = MockAddrTranslator; -static MOCK_OS: MockOs = MockOs::new(); +static MOCK_OS: MockOs = MockOs; -fn alloc_region(size: usize, align: usize) -> (*mut u8, Layout) { +fn host_alloc(size: usize, align: usize) -> (*mut u8, Layout) { let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc::alloc::alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate region"); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null()); (ptr, layout) } -fn dealloc_region(ptr: *mut u8, layout: Layout) { - unsafe { alloc::alloc::dealloc(ptr, layout) }; -} - -fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { - alloc_region( - GlobalAllocator::::required_metadata_size(cpu_count), - PAGE_SIZE, - ) +fn host_dealloc(ptr: *mut u8, layout: Layout) { + unsafe { dealloc(ptr, layout) }; } fn init_allocator( - allocator: &mut GlobalAllocator, + allocator: &GlobalAllocator, heap_addr: usize, heap_size: usize, ) -> (*mut u8, Layout) { - let (meta_ptr, meta_layout) = alloc_metadata(1); - allocator.set_addr_translator(&MOCK_TRANSLATOR); + let meta_size = GlobalAllocator::::required_metadata_size(heap_size, 1); + let meta_align = GlobalAllocator::::required_metadata_align().max(8); + let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align); unsafe { allocator - .init( - meta_ptr as usize, - meta_layout.size(), - heap_addr, - heap_size, - 1, - &MOCK_OS, - ) + .init(meta_ptr, meta_size, heap_addr, heap_size, 1, &MOCK_OS) .unwrap(); } (meta_ptr, meta_layout) } #[test] -fn test_alloc_dma32_pages_uninitialized() { - let allocator = GlobalAllocator::::new(); - let result = allocator.alloc_dma32_pages(1, PAGE_SIZE); - assert!(matches!(result, Err(AllocError::NoMemory))); -} - -#[test] -fn test_alloc_dma32_pages_initialized() { - let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); +fn test_lowmem_basic() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - - let addr1 = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_dma32_pages(4, PAGE_SIZE).unwrap(); - let addr3 = allocator.alloc_dma32_pages(1, 2 * PAGE_SIZE).unwrap(); + let addr1 = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + let addr2 = allocator.alloc_pages_lowmem(4, PAGE_SIZE).unwrap(); assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); - assert!(addr3 >= heap_addr && addr3 < heap_addr + TEST_HEAP_SIZE); assert_eq!(addr1 % PAGE_SIZE, 0); assert_eq!(addr2 % PAGE_SIZE, 0); - assert_eq!(addr3 % (2 * PAGE_SIZE), 0); allocator.dealloc_pages(addr1, 1); allocator.dealloc_pages(addr2, 4); - allocator.dealloc_pages(addr3, 1); - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_alloc_dma32_pages_memory_stats() { - let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); +fn test_lowmem_aligned() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 2); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - - #[cfg(feature = "tracking")] - { - let stats_before = allocator.get_stats(); - assert!(stats_before.total_pages > 0); - } - - let addr = allocator.alloc_dma32_pages(2, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, 2); - - #[cfg(feature = "tracking")] - { - let stats_after = allocator.get_stats(); - assert!(stats_after.total_pages > 0); - } + let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); + assert_eq!(addr % (2 * PAGE_SIZE), 0); + allocator.dealloc_pages(addr, 1); - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_alloc_dma32_pages_vs_normal_pages() { - let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); +fn test_lowmem_vs_normal() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - - let addr_dma32 = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); + let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - assert!(addr_dma32 >= heap_addr && addr_dma32 < heap_addr + TEST_HEAP_SIZE); - assert!(addr_normal >= heap_addr && addr_normal < heap_addr + TEST_HEAP_SIZE); + assert!(addr_low >= heap_addr); + assert!(addr_normal >= heap_addr); - allocator.dealloc_pages(addr_dma32, 1); + allocator.dealloc_pages(addr_low, 1); allocator.dealloc_pages(addr_normal, 1); - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_alloc_dma32_pages_stress() { - let (heap_ptr, heap_layout) = alloc_region(TEST_HEAP_SIZE, PAGE_SIZE); +fn test_lowmem_stress() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE); - - let mut addrs = alloc::vec::Vec::new(); + let mut addrs = Vec::new(); for _ in 0..32 { - let addr = allocator.alloc_dma32_pages(1, PAGE_SIZE).unwrap(); + let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); addrs.push(addr); } - for addr in addrs { allocator.dealloc_pages(addr, 1); } - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 090b1ba..693aefb 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,531 +1,502 @@ -//! Integration tests for the allocator crate. +//! Integration tests for the buddy-slab-allocator crate. -#![no_std] - -extern crate alloc; extern crate buddy_slab_allocator; -use alloc::vec::Vec; use buddy_slab_allocator::{ - AllocError, CompositePageAllocator, GlobalAllocator, Os, SlabAllocDecision, SlabByteAllocator, - SlabDeallocDecision, + BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, SlabAllocResult, SlabAllocator, + SlabDeallocResult, }; use core::alloc::Layout; use core::sync::atomic::{AtomicUsize, Ordering}; +use std::alloc::{alloc, dealloc}; const PAGE_SIZE: usize = 0x1000; -const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; +const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16 MiB + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- -struct MockOs { +struct TestOs { cpu: AtomicUsize, } -impl MockOs { +impl TestOs { const fn new() -> Self { Self { cpu: AtomicUsize::new(0), } } + #[allow(dead_code)] fn set_cpu(&self, cpu: usize) { self.cpu.store(cpu, Ordering::Relaxed); } } -impl Os for MockOs { +impl OsImpl for TestOs { fn current_cpu_idx(&self) -> usize { self.cpu.load(Ordering::Relaxed) } -} - -struct FixedOs; - -impl Os for FixedOs { - fn current_cpu_idx(&self) -> usize { - 0 + fn virt_to_phys(&self, vaddr: usize) -> usize { + vaddr // identity mapping + } + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr } } -static FIXED_OS: FixedOs = FixedOs; -static SWITCHING_OS: MockOs = MockOs::new(); +static TEST_OS: TestOs = TestOs::new(); -fn alloc_region(size: usize, align: usize) -> (*mut u8, Layout) { +/// Allocate a region from the host allocator. +fn host_alloc(size: usize, align: usize) -> (*mut u8, Layout) { let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc::alloc::alloc(layout) }; - assert!(!ptr.is_null(), "Failed to allocate test region"); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "host alloc failed"); (ptr, layout) } -fn dealloc_region(ptr: *mut u8, layout: Layout) { - unsafe { alloc::alloc::dealloc(ptr, layout) }; +fn host_dealloc(ptr: *mut u8, layout: Layout) { + unsafe { dealloc(ptr, layout) }; } -fn alloc_test_heap(size: usize) -> (*mut u8, Layout) { - alloc_region(size, PAGE_SIZE) -} +// ====================================================================== +// Buddy allocator (standalone) tests +// ====================================================================== -fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { - alloc_region( - GlobalAllocator::::required_metadata_size(cpu_count), - PAGE_SIZE, - ) -} - -fn init_global_allocator( - allocator: &mut GlobalAllocator, - heap_addr: usize, - heap_size: usize, - cpu_count: usize, -) -> (*mut u8, Layout) { - let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); - unsafe { - allocator - .init( - meta_ptr as usize, - meta_layout.size(), - heap_addr, - heap_size, - cpu_count, - &FIXED_OS, - ) - .unwrap(); - } - (meta_ptr, meta_layout) -} +#[test] +fn buddy_basic_alloc_dealloc() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); + let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); -fn init_global_allocator_with_os( - allocator: &mut GlobalAllocator, - heap_addr: usize, - heap_size: usize, - cpu_count: usize, - os: &'static dyn Os, -) -> (*mut u8, Layout) { - let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); + let mut buddy = BuddyAllocator::::new(); unsafe { - allocator - .init( - meta_ptr as usize, - meta_layout.size(), - heap_addr, - heap_size, - cpu_count, - os, - ) + buddy + .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) .unwrap(); } - (meta_ptr, meta_layout) -} - -fn slab_alloc_with_refill( - slab_allocator: &mut SlabByteAllocator, - page_allocator: &mut CompositePageAllocator, - owner_cpu: usize, - layout: Layout, -) -> core::ptr::NonNull { - loop { - match slab_allocator.alloc(layout).unwrap() { - SlabAllocDecision::Allocated(ptr, _) => return ptr, - SlabAllocDecision::NeedsRefill { - size_class, - page_count, - slab_bytes, - } => { - let slab_base = page_allocator.alloc_pages(page_count, slab_bytes).unwrap(); - slab_allocator - .provide_slab(size_class, owner_cpu, slab_base, slab_bytes) - .unwrap(); - } - } - } -} - -fn slab_dealloc_with_release( - slab_allocator: &mut SlabByteAllocator, - page_allocator: &mut CompositePageAllocator, - ptr: core::ptr::NonNull, - layout: Layout, -) { - if let SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - .. - } = slab_allocator.dealloc(ptr, layout) - { - page_allocator.dealloc_pages(slab_base, page_count); - } -} - -#[test] -fn test_composite_page_allocator_basic() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); - - let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); + let addr1 = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); - assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); + assert_eq!(addr1 % PAGE_SIZE, 0); + + let addr4 = buddy.alloc_pages(4, PAGE_SIZE).unwrap(); + assert_eq!(addr4 % PAGE_SIZE, 0); - allocator.dealloc_pages(addr1, 1); - allocator.dealloc_pages(addr2, 4); + let free_before = buddy.free_pages(); + buddy.dealloc_pages(addr1, 1); + buddy.dealloc_pages(addr4, 4); + assert!(buddy.free_pages() > free_before); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_composite_page_allocator_alignment() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn buddy_alignment() { + // Heap must be aligned to the highest alignment we test (PAGE_SIZE * 4) + // so that PFN-based alignment maps to absolute address alignment. + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .unwrap(); + } - let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_pages(1, PAGE_SIZE * 2).unwrap(); - let addr3 = allocator.alloc_pages(1, PAGE_SIZE * 4).unwrap(); + let addr2 = buddy.alloc_pages(1, PAGE_SIZE * 2).unwrap(); + assert_eq!(addr2 % (PAGE_SIZE * 2), 0); - assert_eq!(addr1 & (PAGE_SIZE - 1), 0); - assert_eq!(addr2 & (PAGE_SIZE * 2 - 1), 0); - assert_eq!(addr3 & (PAGE_SIZE * 4 - 1), 0); + let addr4 = buddy.alloc_pages(1, PAGE_SIZE * 4).unwrap(); + assert_eq!(addr4 % (PAGE_SIZE * 4), 0); - allocator.dealloc_pages(addr1, 1); - allocator.dealloc_pages(addr2, 1); - allocator.dealloc_pages(addr3, 1); + buddy.dealloc_pages(addr2, 1); + buddy.dealloc_pages(addr4, 1); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_composite_page_allocator_fragmentation() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn buddy_exhaust_and_recover() { + let heap_size = 64 * PAGE_SIZE; // Small heap + let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(heap_size); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap_addr, TEST_HEAP_SIZE); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .unwrap(); + } let mut addrs = Vec::new(); - for _ in 0..10 { - let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - addrs.push((addr, 1)); + while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); } + assert_eq!(buddy.free_pages(), 0); - for i in (0..addrs.len()).step_by(2) { - allocator.dealloc_pages(addrs[i].0, addrs[i].1); + // Free half + for addr in addrs.drain(..addrs.len() / 2) { + buddy.dealloc_pages(addr, 1); } + assert!(buddy.free_pages() > 0); - let result = allocator.alloc_pages(5, PAGE_SIZE); - assert!(result.is_ok()); + // Allocate again + let addr = buddy.alloc_pages(1, PAGE_SIZE); + assert!(addr.is_ok()); - for i in (1..addrs.len()).step_by(2) { - allocator.dealloc_pages(addrs[i].0, addrs[i].1); + // Cleanup + if let Ok(a) = addr { + buddy.dealloc_pages(a, 1); } - if let Ok(addr) = result { - allocator.dealloc_pages(addr, 5); + for a in addrs { + buddy.dealloc_pages(a, 1); } - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_slab_allocator_basic() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn buddy_merge_coalescing() { + let heap_size = 16 * PAGE_SIZE; + let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(heap_size); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut page_allocator = CompositePageAllocator::::new(); - page_allocator.init(heap_addr, TEST_HEAP_SIZE); - - let mut slab_allocator = SlabByteAllocator::::new(); - - let layout8 = Layout::from_size_align(8, 8).unwrap(); - let ptr8 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout8); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .unwrap(); + } - let layout64 = Layout::from_size_align(64, 8).unwrap(); - let ptr64 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout64); + let initial_free = buddy.free_pages(); - let layout2048 = Layout::from_size_align(2048, 8).unwrap(); - let ptr2048 = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout2048); + // Allocate two single pages + let a = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + let b = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + buddy.dealloc_pages(a, 1); + buddy.dealloc_pages(b, 1); - slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr8, layout8); - slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr64, layout64); - slab_dealloc_with_release( - &mut slab_allocator, - &mut page_allocator, - ptr2048, - layout2048, - ); + // After freeing both, free_pages should return to initial + assert_eq!(buddy.free_pages(), initial_free); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } +// ====================================================================== +// Slab allocator (standalone) tests +// ====================================================================== + #[test] -fn test_slab_allocator_many_objects() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn slab_basic() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut page_allocator = CompositePageAllocator::::new(); - page_allocator.init(heap_addr, TEST_HEAP_SIZE); - let mut slab_allocator = SlabByteAllocator::::new(); - - let mut ptrs = Vec::new(); - let layout = Layout::from_size_align(32, 8).unwrap(); - - for _ in 0..100 { - let ptr = slab_alloc_with_refill(&mut slab_allocator, &mut page_allocator, 0, layout); - ptrs.push(ptr); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .unwrap(); } - assert_eq!(ptrs.len(), 100); + let mut slab = SlabAllocator::::new(); - for ptr in ptrs { - slab_dealloc_with_release(&mut slab_allocator, &mut page_allocator, ptr, layout); + let layout = Layout::from_size_align(64, 8).unwrap(); + // First alloc should request pages + match slab.alloc(layout).unwrap() { + SlabAllocResult::NeedsSlab { size_class, pages } => { + let addr = buddy.alloc_pages(pages, PAGE_SIZE).unwrap(); + slab.add_slab(size_class, addr, pages * PAGE_SIZE, 0); + } + SlabAllocResult::Allocated(_) => panic!("should need slab first"), } - dealloc_region(heap_ptr, heap_layout); -} + // Now allocation should succeed + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => p, + _ => panic!("expected allocated"), + }; -#[test] -fn test_global_allocator_init() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - - let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, 1); + // Dealloc + match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { .. } => {} // also valid + } - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_global_allocator_small_alloc() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn slab_many_objects() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - - let layout = Layout::from_size_align(64, 8).unwrap(); - let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); - - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); -} + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .unwrap(); + } -#[test] -fn test_global_allocator_large_alloc() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; + let mut slab = SlabAllocator::::new(); + let layout = Layout::from_size_align(32, 8).unwrap(); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); + let mut ptrs = Vec::new(); + for _ in 0..200 { + loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => { + ptrs.push(p); + break; + } + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } + } + } - let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); - let ptr = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr, layout); + assert_eq!(ptrs.len(), 200); + for ptr in ptrs { + let _ = slab.dealloc(ptr, layout); + } - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_global_allocator_mixed_alloc() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn slab_all_size_classes() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .unwrap(); + } - let sizes = [8, 64, 1024, 4096, 8192]; + let mut slab = SlabAllocator::::new(); let mut allocations = Vec::new(); - for &size in &sizes { - let align = if size <= 2048 { 8 } else { PAGE_SIZE }; - let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = allocator.alloc(layout).unwrap(); - allocations.push((ptr, layout)); + for sc in SizeClass::ALL { + let layout = Layout::from_size_align(sc.size(), sc.size()).unwrap(); + loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => { + allocations.push((p, layout)); + break; + } + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } + } } + assert_eq!(allocations.len(), SizeClass::COUNT); for (ptr, layout) in allocations { - allocator.dealloc(ptr, layout); + let _ = slab.dealloc(ptr, layout); } - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } -#[test] -fn test_global_allocator_page_alloc() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr = heap_ptr as usize; - - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - - let addr1 = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); - let addr2 = allocator.alloc_pages(8, PAGE_SIZE).unwrap(); +// ====================================================================== +// Global allocator tests +// ====================================================================== - allocator.dealloc_pages(addr1, 4); - allocator.dealloc_pages(addr2, 8); - - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); -} - -#[test] -fn test_global_allocator_add_memory() { - let (heap_ptr1, heap_layout1) = alloc_test_heap(TEST_HEAP_SIZE); - let (heap_ptr2, heap_layout2) = alloc_test_heap(TEST_HEAP_SIZE); - let heap_addr1 = heap_ptr1 as usize; - let heap_addr2 = heap_ptr2 as usize; - - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr1, TEST_HEAP_SIZE, 1); - - allocator.add_memory(heap_addr2, TEST_HEAP_SIZE).unwrap(); - let addr = allocator.alloc_pages(16, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, 16); - - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr1, heap_layout1); - dealloc_region(heap_ptr2, heap_layout2); +fn init_global( + allocator: &GlobalAllocator, + heap_addr: usize, + heap_size: usize, + cpu_count: usize, +) -> (*mut u8, Layout) { + let meta_size = GlobalAllocator::::required_metadata_size(heap_size, cpu_count); + let meta_align = GlobalAllocator::::required_metadata_align(); + let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align.max(8)); + unsafe { + allocator + .init( + meta_ptr, meta_size, heap_addr, heap_size, cpu_count, &TEST_OS, + ) + .unwrap(); + } + (meta_ptr, meta_layout) } #[test] -fn test_error_conditions() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn global_page_alloc() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); + let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); + assert!(addr >= heap_addr && addr < heap_addr + TEST_HEAP_SIZE); + assert_eq!(addr % PAGE_SIZE, 0); + allocator.dealloc_pages(addr, 4); - assert!(matches!( - allocator.alloc_pages(0, PAGE_SIZE), - Err(AllocError::InvalidParam) - )); - - let huge_pages = TEST_HEAP_SIZE / PAGE_SIZE + 1; - assert!(matches!( - allocator.alloc_pages(huge_pages, PAGE_SIZE), - Err(AllocError::NoMemory) - )); - - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } -#[cfg(feature = "tracking")] #[test] -fn test_statistics_tracking() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn global_small_alloc() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; - - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - - let stats_initial = allocator.get_stats(); - assert!(stats_initial.total_pages > 0); + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); let layout = Layout::from_size_align(64, 8).unwrap(); - let _ptr = allocator.alloc(layout).unwrap(); - - let stats_after = allocator.get_stats(); - assert!(stats_after.slab_bytes > 0); + let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } -#[cfg(feature = "tracking")] #[test] -fn test_buddy_statistics() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn global_large_alloc() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = - init_global_allocator(&mut allocator, heap_addr, TEST_HEAP_SIZE, 1); - - let buddy_stats = allocator.get_buddy_stats(); - assert!(buddy_stats.total_pages > 0); - assert_eq!(buddy_stats.used_pages, 0); - assert_eq!(buddy_stats.free_pages, buddy_stats.total_pages); + let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_stress_allocation_deallocation() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn global_mixed_alloc() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; - - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global_allocator_with_os( - &mut allocator, - heap_addr, - TEST_HEAP_SIZE, - 2, - &SWITCHING_OS, - ); - + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); + + let sizes: &[(usize, usize)] = &[ + (8, 8), + (64, 8), + (1024, 8), + (4096, PAGE_SIZE), + (8192, PAGE_SIZE), + ]; let mut allocations = Vec::new(); - for i in 0..100 { - SWITCHING_OS.set_cpu(i % 2); - let size = if i % 2 == 0 { 64 } else { 4096 }; - let align = if size <= 2048 { 8 } else { PAGE_SIZE }; + for &(size, align) in sizes { let layout = Layout::from_size_align(size, align).unwrap(); - if let Ok(ptr) = allocator.alloc(layout) { - allocations.push((ptr, layout)); - } + let ptr = allocator.alloc(layout).unwrap(); + allocations.push((ptr, layout)); } - - while let Some((ptr, layout)) = allocations.pop() { - allocator.dealloc(ptr, layout); + for (ptr, layout) in allocations { + unsafe { allocator.dealloc(ptr, layout) }; } - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } #[test] -fn test_global_allocator_cross_cpu_free() { - let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE); +fn global_cross_cpu_free() { + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 2); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global_allocator_with_os( - &mut allocator, - heap_addr, - TEST_HEAP_SIZE, - 2, - &SWITCHING_OS, - ); - + // Allocate on CPU 0 + TEST_OS.set_cpu(0); let layout = Layout::from_size_align(64, 8).unwrap(); - SWITCHING_OS.set_cpu(0); + let mut ptrs = Vec::new(); + for _ in 0..10 { + ptrs.push(allocator.alloc(layout).unwrap()); + } + + // Free from CPU 1 (triggers remote free path) + TEST_OS.set_cpu(1); + for ptr in ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + + // Allocate on CPU 0 again — should drain remote frees and succeed + TEST_OS.set_cpu(0); let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; - SWITCHING_OS.set_cpu(1); - allocator.dealloc(ptr, layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); +} - SWITCHING_OS.set_cpu(0); - let ptr2 = allocator.alloc(layout).unwrap(); - allocator.dealloc(ptr2, layout); +#[test] +fn global_lowmem_pages() { + // Use a custom OsImpl that maps virt addresses below 4 GiB so lowmem + // allocation can succeed on 64-bit hosts where heap addresses are > 4 GiB. + struct LowmemOs; + impl OsImpl for LowmemOs { + fn current_cpu_idx(&self) -> usize { + 0 + } + fn virt_to_phys(&self, vaddr: usize) -> usize { + vaddr & 0x0FFF_FFFF + } + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr + } + } + static LOWMEM_OS: LowmemOs = LowmemOs; + + let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let heap_addr = heap_ptr as usize; + let allocator = GlobalAllocator::::new(); + let cpu_count = 1; + let meta_size = GlobalAllocator::::required_metadata_size(TEST_HEAP_SIZE, cpu_count); + let meta_align = GlobalAllocator::::required_metadata_align(); + let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align.max(8)); + unsafe { + allocator + .init( + meta_ptr, + meta_size, + heap_addr, + TEST_HEAP_SIZE, + cpu_count, + &LOWMEM_OS, + ) + .unwrap(); + } + + let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + assert!(addr >= heap_addr); + allocator.dealloc_pages(addr, 1); - dealloc_region(meta_ptr, meta_layout); - dealloc_region(heap_ptr, heap_layout); + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); } diff --git a/tests/stress_test.rs b/tests/stress_test.rs index b00e38d..96bfb5a 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -1,6 +1,6 @@ -//! Ignored stress tests for allocator stability and long-running behavior. +//! Stress tests for allocator stability. -use buddy_slab_allocator::{GlobalAllocator, Os}; +use buddy_slab_allocator::{GlobalAllocator, OsImpl}; use rand::{RngExt, SeedableRng}; use std::alloc::{alloc, dealloc, Layout}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -24,10 +24,16 @@ impl MockOs { } } -impl Os for MockOs { +impl OsImpl for MockOs { fn current_cpu_idx(&self) -> usize { self.cpu.load(Ordering::Relaxed) } + fn virt_to_phys(&self, vaddr: usize) -> usize { + vaddr + } + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr + } } static MOCK_OS: MockOs = MockOs::new(); @@ -56,25 +62,26 @@ impl Drop for TestHeap { } } -fn alloc_metadata(cpu_count: usize) -> (*mut u8, Layout) { - let size = GlobalAllocator::::required_metadata_size(cpu_count); - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); +fn alloc_metadata(heap_size: usize, cpu_count: usize) -> (*mut u8, Layout) { + let size = GlobalAllocator::::required_metadata_size(heap_size, cpu_count); + let align = GlobalAllocator::::required_metadata_align().max(8); + let layout = Layout::from_size_align(size, align).unwrap(); let ptr = unsafe { alloc(layout) }; assert!(!ptr.is_null(), "failed to allocate metadata"); (ptr, layout) } fn init_allocator( - allocator: &mut GlobalAllocator, + allocator: &GlobalAllocator, heap: &TestHeap, cpu_count: usize, ) -> (*mut u8, Layout) { - let (meta_ptr, meta_layout) = alloc_metadata(cpu_count); + let (meta_ptr, meta_layout) = alloc_metadata(HEAP_SIZE, cpu_count); MOCK_OS.set_cpu(0); unsafe { allocator .init( - meta_ptr as usize, + meta_ptr, meta_layout.size(), heap.addr(), HEAP_SIZE, @@ -90,10 +97,10 @@ fn init_allocator( #[ignore = "stress test"] fn stress_random_mixed_alloc_free() { let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 2); let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - let mut allocated = Vec::new(); + let mut allocated: Vec<(core::ptr::NonNull, Layout)> = Vec::new(); for i in 0..10_000 { MOCK_OS.set_cpu(i % 2); @@ -112,12 +119,12 @@ fn stress_random_mixed_alloc_free() { } else { let idx = rng.random_range(0..allocated.len()); let (ptr, layout) = allocated.swap_remove(idx); - allocator.dealloc(ptr, layout); + unsafe { allocator.dealloc(ptr, layout) }; } } for (ptr, layout) in allocated { - allocator.dealloc(ptr, layout); + unsafe { allocator.dealloc(ptr, layout) }; } unsafe { dealloc(meta_ptr, meta_layout) }; @@ -127,8 +134,8 @@ fn stress_random_mixed_alloc_free() { #[ignore = "stress test"] fn stress_exhaustion_recovery() { let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 1); let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); let mut allocated = Vec::new(); @@ -137,18 +144,18 @@ fn stress_exhaustion_recovery() { } for ptr in allocated.drain(..allocated.len() / 4) { - allocator.dealloc(ptr, layout); + unsafe { allocator.dealloc(ptr, layout) }; } let recovered = allocator.alloc(layout); assert!(recovered.is_ok()); if let Ok(ptr) = recovered { - allocator.dealloc(ptr, layout); + unsafe { allocator.dealloc(ptr, layout) }; } for ptr in allocated { - allocator.dealloc(ptr, layout); + unsafe { allocator.dealloc(ptr, layout) }; } unsafe { dealloc(meta_ptr, meta_layout) }; @@ -158,8 +165,8 @@ fn stress_exhaustion_recovery() { #[ignore = "stress test"] fn stress_fragmentation_recovery() { let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); + let allocator = GlobalAllocator::::new(); + let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 2); let small_layout = Layout::from_size_align(64, 8).unwrap(); let mut small_ptrs = Vec::new(); @@ -171,62 +178,19 @@ fn stress_fragmentation_recovery() { } for i in (0..small_ptrs.len()).step_by(2) { - allocator.dealloc(small_ptrs[i], small_layout); + unsafe { allocator.dealloc(small_ptrs[i], small_layout) }; } let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap(); let large = allocator.alloc(large_layout); for ptr in small_ptrs.into_iter().skip(1).step_by(2) { - allocator.dealloc(ptr, small_layout); + unsafe { allocator.dealloc(ptr, small_layout) }; } if let Ok(ptr) = large { - allocator.dealloc(ptr, large_layout); - } - - unsafe { dealloc(meta_ptr, meta_layout) }; -} - -#[cfg(feature = "tracking")] -#[test] -#[ignore = "stress test"] -fn stress_tracking_invariants() { - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&mut allocator, &heap, 2); - let mut rng = rand::rngs::StdRng::from_seed([1; 32]); - let mut allocated = Vec::new(); - - for i in 0..5000 { - MOCK_OS.set_cpu(i % 2); - if allocated.is_empty() || rng.random_bool(0.6) { - let size: usize = rng.random_range(8..4097); - let layout = if size <= 2048 { - Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() - } else { - let aligned = size.div_ceil(PAGE_SIZE) * PAGE_SIZE; - Layout::from_size_align(aligned, PAGE_SIZE).unwrap() - }; - - if let Ok(ptr) = allocator.alloc(layout) { - allocated.push((ptr, layout)); - } - } else { - let idx = rng.random_range(0..allocated.len()); - let (ptr, layout) = allocated.swap_remove(idx); - allocator.dealloc(ptr, layout); - } + unsafe { allocator.dealloc(ptr, large_layout) }; } - for (ptr, layout) in allocated { - allocator.dealloc(ptr, layout); - } - - let stats = allocator.get_stats(); - assert!(stats.used_pages <= stats.total_pages); - assert!(stats.free_pages <= stats.total_pages); - assert_eq!(stats.used_pages + stats.free_pages, stats.total_pages); - unsafe { dealloc(meta_ptr, meta_layout) }; } From bdddcc1f139d0543406070f72a58dc78787ce02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Wed, 8 Apr 2026 15:14:26 +0800 Subject: [PATCH 08/19] feat: implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator --- src/buddy/mod.rs | 8 ++++++++ src/buddy/page_meta.rs | 14 ++++++++++---- src/global.rs | 8 ++++++++ src/slab/mod.rs | 8 ++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index a694e08..c6e2f36 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -69,7 +69,15 @@ impl BuddyAllocator { os: None, } } +} +impl Default for BuddyAllocator { + fn default() -> Self { + Self::new() + } +} + +impl BuddyAllocator { /// Initialise the allocator over a heap region, using `meta` for page metadata. /// /// # Safety diff --git a/src/buddy/page_meta.rs b/src/buddy/page_meta.rs index ebb6042..30c27f8 100644 --- a/src/buddy/page_meta.rs +++ b/src/buddy/page_meta.rs @@ -1,7 +1,7 @@ -/// Per-page metadata stored in the external metadata region. -/// -/// Each page frame in the heap has a corresponding `PageMeta` entry. -/// Free pages are linked together via intrusive doubly-linked lists using PFN indices. +//! Per-page metadata stored in the external metadata region. +//! +//! Each page frame in the heap has a corresponding [`PageMeta`] entry. +//! Free pages are linked together via intrusive doubly-linked lists using PFN indices. /// Sentinel value indicating "no page" in free-list links. pub const PFN_NONE: u32 = u32::MAX; @@ -39,6 +39,12 @@ pub struct PageMeta { const _: () = assert!(core::mem::size_of::() == 12); +impl Default for PageMeta { + fn default() -> Self { + Self::new() + } +} + impl PageMeta { /// Create a zeroed (free, order-0) page meta. pub const fn new() -> Self { diff --git a/src/global.rs b/src/global.rs index 0d4441f..f53292e 100644 --- a/src/global.rs +++ b/src/global.rs @@ -75,7 +75,15 @@ impl GlobalAllocator { initialized: AtomicBool::new(false), } } +} +impl Default for GlobalAllocator { + fn default() -> Self { + Self::new() + } +} + +impl GlobalAllocator { /// Initialise the allocator. /// /// # Arguments diff --git a/src/slab/mod.rs b/src/slab/mod.rs index 3233733..d05de3d 100644 --- a/src/slab/mod.rs +++ b/src/slab/mod.rs @@ -59,7 +59,15 @@ impl SlabAllocator { ], } } +} +impl Default for SlabAllocator { + fn default() -> Self { + Self::new() + } +} + +impl SlabAllocator { /// Try to allocate an object matching `layout`. /// /// If the matching cache is exhausted, [`SlabAllocResult::NeedsSlab`] is returned From c8519881a7985c091ce752c2c802007df35bf1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Wed, 8 Apr 2026 17:42:27 +0800 Subject: [PATCH 09/19] Refactor slab allocator benchmarks and improve global allocator initialization - Updated benchmarks in `benches/slab_allocator.rs` to use the `divan` framework, simplifying the allocation and deallocation logic. - Removed redundant code and improved clarity in benchmark functions for size class allocation, hot reuse, mixed size batch, and steady state recycling. - Enhanced the `GlobalAllocator` in `src/global.rs` to manage metadata layout more effectively, ensuring proper alignment and size calculations. - Modified the `BuddyAllocator` to enforce stricter checks during deallocation, ensuring that the correct block sizes are freed. - Improved error handling in tests to validate allocator behavior under various conditions, including unaligned region starts and insufficient managed pages. - Cleaned up memory allocation and deallocation logic in stress tests, removing unnecessary metadata handling. --- Cargo.toml | 14 ++ README.md | 28 ++-- README_CN.md | 26 ++-- benches/README_CN.md | 37 ++--- benches/buddy_allocator.rs | 266 +++++++++++------------------------- benches/common.rs | 189 ++++++++++++++++++++++--- benches/global_allocator.rs | 243 +++++++++++--------------------- benches/slab_allocator.rs | 238 ++++++++++---------------------- src/buddy/mod.rs | 17 ++- src/global.rs | 192 ++++++++++++++++++-------- src/slab/cache.rs | 2 +- src/slab/page.rs | 30 +++- tests/dma32_pages_test.rs | 63 ++++----- tests/integration_test.rs | 159 ++++++++++++--------- tests/stress_test.rs | 38 +----- 15 files changed, 780 insertions(+), 762 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e0745d3..90e2514 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [package] authors = ["Song Zhiyong "] +autobenches = false categories = ["memory-management", "no-std", "embedded"] description = "Memory allocator with Buddy and Slab allocation" documentation = "https://docs.rs/buddy-slab-allocator" @@ -19,4 +20,17 @@ log = "0.4" spin = "0.10" [dev-dependencies] +divan = "0.1.21" rand = {version = "0.10", features = ["std_rng"]} + +[[bench]] +name = "buddy_allocator" +harness = false + +[[bench]] +name = "slab_allocator" +harness = false + +[[bench]] +name = "global_allocator" +harness = false diff --git a/README.md b/README.md index 76bde82..031bd04 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ classDiagram -UsageStatsAtomic stats -AtomicBool initialized +new() GlobalAllocator - +init(meta_start, meta_size, heap_start, heap_size, cpu_count, os) AllocResult + +init(region_start, region_size, cpu_count, os) AllocResult +add_memory_region(start, size) AllocResult +alloc(Layout) Option~NonNull~u8~~ +dealloc(ptr, Layout) @@ -236,28 +236,27 @@ buddy-slab-allocator = { version = "0.2.0", features = ["tracking"] } The `GlobalAllocator` is the recommended interface. It automatically routes small allocations to per-CPU slab caches and large allocations to the buddy page allocator. ```rust -use buddy_slab_allocator::{GlobalAllocator, Os}; +use buddy_slab_allocator::{GlobalAllocator, OsImpl}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; struct DemoOs; -impl Os for DemoOs { +impl OsImpl for DemoOs { fn current_cpu_idx(&self) -> usize { 0 } + fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } + fn phys_to_virt(&self, paddr: usize) -> usize { paddr } } static OS: DemoOs = DemoOs; -let mut allocator = GlobalAllocator::::new(); - -let meta_start = 0x8100_0000; -let meta_size = GlobalAllocator::::required_metadata_size(1); -let heap_start = 0x8000_0000; -let heap_size = 16 * 1024 * 1024; +let allocator = GlobalAllocator::::new(); +let region_start = 0x8000_0000; +let region_size = 16 * 1024 * 1024; unsafe { allocator - .init(meta_start, meta_size, heap_start, heap_size, 1, &OS) + .init(region_start, region_size, 1, &OS) .unwrap(); } @@ -343,11 +342,16 @@ cargo test --features tracking # Check benchmarks compile cargo check --benches -# Run benchmarks +# Run all benchmarks cargo bench + +# Run one benchmark suite +cargo bench --bench buddy_allocator +cargo bench --bench slab_allocator +cargo bench --bench global_allocator ``` -Detailed benchmark notes are in `benches/README_CN.md`. +Benchmarks are built with `divan`. Detailed benchmark notes are in `benches/README_CN.md`. ## License diff --git a/README_CN.md b/README_CN.md index 9c9cd62..638f578 100644 --- a/README_CN.md +++ b/README_CN.md @@ -24,30 +24,33 @@ buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } ### 使用 `GlobalAllocator` ```rust -use buddy_slab_allocator::{GlobalAllocator, Os}; +use buddy_slab_allocator::{GlobalAllocator, OsImpl}; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; struct DemoOs; -impl Os for DemoOs { +impl OsImpl for DemoOs { fn current_cpu_idx(&self) -> usize { 0 } + fn virt_to_phys(&self, vaddr: usize) -> usize { + vaddr + } + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr + } } static OS: DemoOs = DemoOs; -let mut allocator = GlobalAllocator::::new(); - -let meta_start = 0x8100_0000; -let meta_size = GlobalAllocator::::required_metadata_size(1); -let heap_start = 0x8000_0000; -let heap_size = 16 * 1024 * 1024; +let allocator = GlobalAllocator::::new(); +let region_start = 0x8000_0000; +let region_size = 16 * 1024 * 1024; unsafe { allocator - .init(meta_start, meta_size, heap_start, heap_size, 1, &OS) + .init(region_start, region_size, 1, &OS) .unwrap(); } @@ -109,6 +112,9 @@ cargo test --features tracking ```bash cargo check --benches cargo bench +cargo bench --bench buddy_allocator +cargo bench --bench slab_allocator +cargo bench --bench global_allocator ``` -Benchmark 说明见 `benches/README_CN.md`。 +Benchmark 基于 `divan`，说明见 `benches/README_CN.md`。 diff --git a/benches/README_CN.md b/benches/README_CN.md index 8ea39a4..fa76b8a 100644 --- a/benches/README_CN.md +++ b/benches/README_CN.md @@ -1,15 +1,20 @@ # Benchmark 使用说明 -本目录包含 buddy-slab-allocator 的性能 benchmark，统一基于 `criterion`。 +本目录包含基于 `divan` 的 benchmark 套件，围绕当前 crate 的三层结构组织： -## 当前 benchmark 套件 - -- `global_allocator.rs` - 多核 `GlobalAllocator` 的混合 workload - `buddy_allocator.rs` - Buddy 页分配、对齐、碎片化恢复、统计查询 + `BuddyAllocator` 的页分配、对齐分配、碎片恢复、随机页 workload - `slab_allocator.rs` - Slab 前端 size class、steady-state、热路径复用 + `SlabAllocator` 的 size class alloc/free、hot reuse、mixed-size batch、steady-state recycle +- `global_allocator.rs` + `GlobalAllocator` 的小对象、大对象、页接口、混合 workload、cross-CPU free cycle + +共享的 host-side harness 放在 `common.rs`，负责： + +- region / metadata 分配 +- buddy / slab / global 初始化 +- 固定随机种子 +- mock `OsImpl` ## 运行方式 @@ -28,15 +33,13 @@ cargo bench --bench global_allocator ## 设计原则 -- 使用 `std::hint::black_box` -- 统一 `criterion` 配置 -- 尽量 benchmark 成对的 alloc/free -- 固定 workload 批次与随机种子 -- 使用 `Throughput::{Elements, Bytes}` 表达吞吐量 +- 使用 `divan::Bencher` 和 `divan::black_box` +- 不再保留旧的 `criterion` 代码路径 +- benchmark 只使用当前公开 API，不依赖历史类型名 +- 尽量让每次迭代在闭环内恢复 allocator 状态，减少跨迭代污染 +- workload 使用固定模式或固定随机种子，便于复现 -## 与压力测试的区别 +## CI 策略 -- `benches/` - 只做性能测量 -- `tests/stress_test.rs` - 负责长时间随机、碎片化恢复、耗尽恢复与 tracking 不变量 +CI 仅执行 `cargo check --benches`，确保 benchmark 工程持续可编译。 +真实性能测量和结果对比保留在本地执行。 diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index 5d39356..fa8e26a 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -1,205 +1,103 @@ -//! Benchmarks for buddy-based page allocation paths. +//! Benchmarks for the buddy page allocator. mod common; -use buddy_slab_allocator::{BuddyPageAllocator, CompositePageAllocator}; -use common::{criterion_config, seeded_rng, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; -use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; +use common::{seeded_rng, BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use divan::{black_box, Bencher}; use rand::RngExt; -use std::hint::black_box; - -fn bench_page_alloc_free(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/page_alloc_free"); - - for &num_pages in &[1usize, 2, 4, 16, 64] { - group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); - - group.bench_with_input( - BenchmarkId::from_parameter(num_pages), - &num_pages, - |b, &pages| { - b.iter(|| { - let addr = allocator.alloc_pages(black_box(pages), PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, pages); - black_box(addr); - }); - }, - ); - } - group.finish(); +fn main() { + divan::main(); } -fn bench_alignment_alloc_free(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/alignment_alloc_free"); - - for &alignment in &[PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4] { - group.throughput(Throughput::Bytes((4 * PAGE_SIZE) as u64)); - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); - - group.bench_with_input( - BenchmarkId::from_parameter(alignment), - &alignment, - |b, &align| { - b.iter(|| { - let addr = allocator - .alloc_pages(black_box(4), black_box(align)) - .unwrap(); - allocator.dealloc_pages(addr, 4); - black_box(addr); - }); - }, - ); +#[divan::bench_group] +mod buddy { + use super::*; + + #[divan::bench(args = [1usize, 2, 4, 16, 64])] + fn page_alloc_free(bencher: Bencher, pages: usize) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + + bencher.bench_local(|| { + let addr = harness + .allocator + .alloc_pages(black_box(pages), PAGE_SIZE) + .unwrap(); + harness.allocator.dealloc_pages(addr, pages); + black_box(addr) + }); } - group.finish(); -} + #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4, PAGE_SIZE * 16])] + fn aligned_page_alloc_free(bencher: Bencher, align: usize) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + + bencher.bench_local(|| { + let addr = harness + .allocator + .alloc_pages(black_box(4), black_box(align)) + .unwrap(); + harness.allocator.dealloc_pages(addr, 4); + black_box(addr) + }); + } -fn bench_fragmentation_recovery(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/fragmentation_recovery"); - group.throughput(Throughput::Bytes((64 * PAGE_SIZE) as u64)); + #[divan::bench] + fn fragmentation_recovery_cycle(bencher: Bencher) { + let mut harness = BuddyHarness::new(HEAP_SIZE); - group.bench_function("alloc_large_after_fragmentation", |b| { - b.iter_batched_ref( - || { - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); + bencher.bench_local(|| { + let mut addrs = Vec::with_capacity(FRAGMENTATION_PAGES); + for _ in 0..FRAGMENTATION_PAGES { + addrs.push(harness.allocator.alloc_pages(1, PAGE_SIZE).unwrap()); + } - let mut addrs = Vec::with_capacity(512); - for _ in 0..512 { - addrs.push(allocator.alloc_pages(1, PAGE_SIZE).unwrap()); - } + for idx in (0..addrs.len()).step_by(2) { + harness.allocator.dealloc_pages(addrs[idx], 1); + } - (heap, allocator, addrs) - }, - |(_, allocator, addrs)| { - for i in (0..addrs.len()).step_by(2) { - allocator.dealloc_pages(addrs[i], 1); - } + let large = harness.allocator.alloc_pages(64, PAGE_SIZE).unwrap(); + harness.allocator.dealloc_pages(large, 64); - let large = allocator.alloc_pages(64, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(large, 64); + for idx in (1..addrs.len()).step_by(2) { + harness.allocator.dealloc_pages(addrs[idx], 1); + } - for i in (1..addrs.len()).step_by(2) { - allocator.dealloc_pages(addrs[i], 1); - } - - black_box(large); - }, - BatchSize::SmallInput, - ); - }); - - group.finish(); -} + black_box(large) + }); + } -fn bench_random_workload(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/random_workload"); - group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - - group.bench_function("mixed_alloc_free_batch", |b| { - b.iter_batched_ref( - || { - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); - (heap, allocator) - }, - |(_, allocator)| { - let mut rng = seeded_rng(); - let mut allocated = Vec::new(); - - for _ in 0..OPERATIONS_PER_BATCH { - if allocated.is_empty() || rng.random_bool(0.65) { - let pages = 1usize << rng.random_range(0..=4); - if let Ok(addr) = allocator.alloc_pages(pages, PAGE_SIZE) { - allocated.push((addr, pages)); - } - } else { - let idx = rng.random_range(0..allocated.len()); - let (addr, pages) = allocated.swap_remove(idx); - allocator.dealloc_pages(addr, pages); + #[divan::bench] + fn random_page_workload(bencher: Bencher) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + let mut rng = seeded_rng(); + let plan: Vec<(bool, usize, usize)> = (0..OPERATIONS_PER_BATCH) + .map(|_| { + let allocate = rng.random_bool(0.65); + let pages = 1usize << rng.random_range(0..=4); + let free_hint = rng.random_range(0..OPERATIONS_PER_BATCH.max(1)); + (allocate, pages, free_hint) + }) + .collect(); + + bencher.bench_local(|| { + let mut active = Vec::new(); + + for &(allocate, pages, free_hint) in &plan { + if allocate || active.is_empty() { + if let Ok(addr) = harness.allocator.alloc_pages(pages, PAGE_SIZE) { + active.push((addr, pages)); } + } else { + let idx = free_hint % active.len(); + let (addr, count) = active.swap_remove(idx); + harness.allocator.dealloc_pages(addr, count); } + } - for (addr, pages) in allocated { - allocator.dealloc_pages(addr, pages); - } - }, - BatchSize::SmallInput, - ); - }); - - group.finish(); -} - -fn bench_composite_non_power_of_two(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/composite_non_power_of_two"); - - for &num_pages in &[3usize, 5, 9] { - group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = CompositePageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); - - group.bench_with_input( - BenchmarkId::from_parameter(num_pages), - &num_pages, - |b, &pages| { - b.iter(|| { - let addr = allocator.alloc_pages(pages, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, pages); - black_box(addr); - }); - }, - ); - } - - group.finish(); -} - -fn bench_statistics_query(c: &mut Criterion) { - let mut group = c.benchmark_group("buddy/statistics_query"); - group.throughput(Throughput::Elements(3)); - - let heap = TestHeap::new(HEAP_SIZE); - let mut allocator = BuddyPageAllocator::::new(); - allocator.init(heap.addr(), HEAP_SIZE); - - for _ in 0..64 { - let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - allocator.dealloc_pages(addr, 1); - } - - group.bench_function("page_counters", |b| { - b.iter(|| { - let snapshot = ( - allocator.total_pages(), - allocator.used_pages(), - allocator.available_pages(), - ); - black_box(snapshot); + for (addr, count) in active { + harness.allocator.dealloc_pages(addr, count); + } }); - }); - - group.finish(); -} - -criterion_group! { - name = benches; - config = criterion_config(); - targets = - bench_page_alloc_free, - bench_alignment_alloc_free, - bench_fragmentation_recovery, - bench_random_workload, - bench_composite_non_power_of_two, - bench_statistics_query + } } -criterion_main!(benches); diff --git a/benches/common.rs b/benches/common.rs index 671fa2a..1d0d925 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -1,23 +1,31 @@ -use criterion::Criterion; +#![allow(dead_code)] + +use buddy_slab_allocator::{ + buddy::PageMeta, BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, + SlabDeallocResult, +}; +use core::alloc::Layout; use rand::{rngs::StdRng, SeedableRng}; -use std::alloc::{alloc, dealloc, Layout}; -use std::time::Duration; +use std::alloc::{alloc, dealloc}; +use std::sync::atomic::{AtomicUsize, Ordering}; pub const PAGE_SIZE: usize = 0x1000; pub const HEAP_SIZE: usize = 64 * 1024 * 1024; -pub const OPERATIONS_PER_BATCH: u64 = 256; -const HEAP_ALIGN: usize = 64 * 1024; +pub const OPERATIONS_PER_BATCH: usize = 256; +pub const FRAGMENTATION_PAGES: usize = 512; + +const REGION_ALIGN: usize = 64 * 1024; -pub struct TestHeap { +pub struct HostRegion { ptr: *mut u8, layout: Layout, } -impl TestHeap { +impl HostRegion { pub fn new(size: usize) -> Self { - let layout = Layout::from_size_align(size, HEAP_ALIGN).unwrap(); + let layout = Layout::from_size_align(size, REGION_ALIGN).unwrap(); let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "failed to allocate test heap"); + assert!(!ptr.is_null(), "failed to allocate host region"); Self { ptr, layout } } @@ -26,21 +34,168 @@ impl TestHeap { } } -impl Drop for TestHeap { +impl Drop for HostRegion { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +pub struct MetadataBuf { + ptr: *mut u8, + layout: Layout, +} + +impl MetadataBuf { + pub fn for_heap(heap_size: usize) -> Self { + let size = BuddyAllocator::::required_meta_size(heap_size); + let align = core::mem::align_of::().max(8); + let layout = Layout::from_size_align(size, align).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate metadata"); + Self { ptr, layout } + } + + pub fn ptr(&self) -> *mut u8 { + self.ptr + } + + pub fn size(&self) -> usize { + self.layout.size() + } +} + +impl Drop for MetadataBuf { fn drop(&mut self) { unsafe { dealloc(self.ptr, self.layout) }; } } -pub fn criterion_config() -> Criterion { - Criterion::default() - .warm_up_time(Duration::from_secs(1)) - .measurement_time(Duration::from_secs(5)) - .sample_size(50) - .noise_threshold(0.05) +pub struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + pub const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } + + pub fn set_cpu(&self, cpu: usize) { + self.cpu.store(cpu, Ordering::Relaxed); + } +} + +impl OsImpl for MockOs { + fn current_cpu_idx(&self) -> usize { + self.cpu.load(Ordering::Relaxed) + } + + fn virt_to_phys(&self, vaddr: usize) -> usize { + vaddr + } + + fn phys_to_virt(&self, paddr: usize) -> usize { + paddr + } } -#[allow(dead_code)] +pub static MOCK_OS: MockOs = MockOs::new(); + pub fn seeded_rng() -> StdRng { StdRng::from_seed([0; 32]) } + +pub struct BuddyHarness { + _heap: HostRegion, + _meta: MetadataBuf, + pub allocator: BuddyAllocator, +} + +impl BuddyHarness { + pub fn new(heap_size: usize) -> Self { + let heap = HostRegion::new(heap_size); + let meta = MetadataBuf::for_heap(heap_size); + let mut allocator = BuddyAllocator::::new(); + unsafe { + allocator + .init(meta.ptr(), meta.size(), heap.addr(), heap_size, None) + .unwrap(); + } + Self { + _heap: heap, + _meta: meta, + allocator, + } + } +} + +pub struct SlabHarness { + _heap: HostRegion, + _meta: MetadataBuf, + buddy: BuddyAllocator, + slab: SlabAllocator, +} + +impl SlabHarness { + pub fn new(heap_size: usize) -> Self { + let heap = HostRegion::new(heap_size); + let meta = MetadataBuf::for_heap(heap_size); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta.ptr(), meta.size(), heap.addr(), heap_size, None) + .unwrap(); + } + Self { + _heap: heap, + _meta: meta, + buddy, + slab: SlabAllocator::new(), + } + } + + pub fn alloc(&mut self, layout: Layout) -> core::ptr::NonNull { + loop { + match self.slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => return ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let base = self.buddy.alloc_pages(pages, slab_bytes).unwrap(); + self.slab.add_slab(size_class, base, slab_bytes, 0); + } + } + } + } + + pub fn dealloc(&mut self, ptr: core::ptr::NonNull, layout: Layout) { + match self.slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + self.buddy.dealloc_pages(base, pages); + } + } + } +} + +pub struct GlobalHarness { + _region: HostRegion, + pub allocator: GlobalAllocator, +} + +impl GlobalHarness { + pub fn new(region_size: usize, cpu_count: usize) -> Self { + let region = HostRegion::new(region_size); + let allocator = GlobalAllocator::::new(); + MOCK_OS.set_cpu(0); + unsafe { + allocator + .init(region.addr(), region_size, cpu_count, &MOCK_OS) + .unwrap(); + } + Self { + _region: region, + allocator, + } + } +} diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index 0ce98f9..1ce78f6 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -1,196 +1,111 @@ -//! Benchmarks for the unified global allocator API. +//! Benchmarks for the unified global allocator. mod common; -use buddy_slab_allocator::{GlobalAllocator, Os}; -use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use spin::Mutex; -use std::alloc::{alloc, dealloc, Layout}; -use std::hint::black_box; -use std::sync::atomic::{AtomicUsize, Ordering}; +use common::{GlobalHarness, MOCK_OS, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use core::alloc::Layout; +use divan::{black_box, Bencher}; -struct MockOs { - cpu: AtomicUsize, +fn main() { + divan::main(); } -impl MockOs { - const fn new() -> Self { - Self { - cpu: AtomicUsize::new(0), - } - } +#[divan::bench_group] +mod global { + use super::*; - fn set_cpu(&self, cpu: usize) { - self.cpu.store(cpu, Ordering::Relaxed); - } -} + #[divan::bench(args = [8usize, 64, 512, 2048])] + fn small_alloc_free(bencher: Bencher, size: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let layout = Layout::from_size_align(size, 8).unwrap(); -impl Os for MockOs { - fn current_cpu_idx(&self) -> usize { - self.cpu.load(Ordering::Relaxed) + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + black_box(ptr) + }); } -} - -static MOCK_OS: MockOs = MockOs::new(); - -struct MetadataBuf { - ptr: *mut u8, - layout: Layout, -} -impl MetadataBuf { - fn new(cpu_count: usize) -> Self { - let size = GlobalAllocator::::required_metadata_size(cpu_count); + #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16])] + fn large_alloc_free(bencher: Bencher, size: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "failed to allocate metadata"); - Self { ptr, layout } - } -} -impl Drop for MetadataBuf { - fn drop(&mut self) { - unsafe { dealloc(self.ptr, self.layout) }; + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + black_box(ptr) + }); } -} -struct GlobalHarness { - _heap: TestHeap, - _meta: MetadataBuf, - allocator: GlobalAllocator, -} + #[divan::bench(args = [1usize, 4, 16, 64])] + fn page_interface(bencher: Bencher, pages: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); -impl GlobalHarness { - fn new(cpu_count: usize) -> Self { - let heap = TestHeap::new(HEAP_SIZE); - let meta = MetadataBuf::new(cpu_count); - let mut allocator = GlobalAllocator::::new(); - unsafe { - allocator - .init( - meta.ptr as usize, - meta.layout.size(), - heap.addr(), - HEAP_SIZE, - cpu_count, - &MOCK_OS, - ) + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let addr = harness + .allocator + .alloc_pages(black_box(pages), PAGE_SIZE) .unwrap(); - } - Self { - _heap: heap, - _meta: meta, - allocator, - } + harness.allocator.dealloc_pages(addr, pages); + black_box(addr) + }); } -} -fn bench_small_alloc_free(c: &mut Criterion) { - let mut group = c.benchmark_group("global/small_alloc_free"); + #[divan::bench] + fn mixed_object_page_workload(bencher: Bencher) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let object_layouts = [ + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(256, 8).unwrap(), + Layout::from_size_align(1024, 8).unwrap(), + Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(), + ]; + let page_counts = [1usize, 2, 4, 8]; + + bencher.bench_local(|| { + for idx in 0..OPERATIONS_PER_BATCH { + MOCK_OS.set_cpu(idx % 2); + let layout = object_layouts[idx % object_layouts.len()]; + let ptr = harness.allocator.alloc(layout).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; - for &size in &[8usize, 64, 512, 2048] { - group.throughput(Throughput::Bytes(size as u64)); - let harness = Mutex::new(GlobalHarness::new(2)); - let layout = Layout::from_size_align(size, 8).unwrap(); + let pages = page_counts[idx % page_counts.len()]; + let addr = harness.allocator.alloc_pages(pages, PAGE_SIZE).unwrap(); + harness.allocator.dealloc_pages(addr, pages); - group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { - b.iter(|| { - MOCK_OS.set_cpu(0); - let guard = harness.lock(); - let ptr = guard.allocator.alloc(black_box(layout)).unwrap(); - guard.allocator.dealloc(ptr, layout); - black_box(ptr); - }); + black_box((ptr, addr)); + } }); } - group.finish(); -} + #[divan::bench] + fn remote_free_cycle(bencher: Bencher) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let layout = Layout::from_size_align(64, 8).unwrap(); -fn bench_large_alloc_free(c: &mut Criterion) { - let mut group = c.benchmark_group("global/large_alloc_free"); - - for &size in &[PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16] { - group.throughput(Throughput::Bytes(size as u64)); - let harness = Mutex::new(GlobalHarness::new(2)); - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + bencher.bench_local(|| { + let mut ptrs = Vec::with_capacity(64); - group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { - b.iter(|| { - let guard = harness.lock(); - let ptr = guard.allocator.alloc(black_box(layout)).unwrap(); - guard.allocator.dealloc(ptr, layout); - black_box(ptr); - }); - }); - } + MOCK_OS.set_cpu(0); + for _ in 0..64 { + ptrs.push(harness.allocator.alloc(layout).unwrap()); + } - group.finish(); -} + MOCK_OS.set_cpu(1); + for &ptr in &ptrs { + unsafe { harness.allocator.dealloc(ptr, layout) }; + } -fn bench_mixed_workload(c: &mut Criterion) { - let mut group = c.benchmark_group("global/mixed_workload"); - group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - - let harness = Mutex::new(GlobalHarness::new(2)); - let layouts = [ - Layout::from_size_align(64, 8).unwrap(), - Layout::from_size_align(256, 8).unwrap(), - Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(), - Layout::from_size_align(PAGE_SIZE * 4, PAGE_SIZE).unwrap(), - ]; - - group.bench_function("alternating_small_and_large", |b| { - b.iter(|| { - for idx in 0..OPERATIONS_PER_BATCH as usize { - MOCK_OS.set_cpu(idx % 2); - let layout = layouts[idx % layouts.len()]; - let guard = harness.lock(); - let ptr = guard.allocator.alloc(layout).unwrap(); - guard.allocator.dealloc(ptr, layout); + MOCK_OS.set_cpu(0); + for _ in 0..64 { + let ptr = harness.allocator.alloc(layout).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; black_box(ptr); } }); - }); - - group.finish(); -} - -fn bench_page_interface(c: &mut Criterion) { - let mut group = c.benchmark_group("global/page_interface"); - - for &num_pages in &[1usize, 4, 16, 64] { - group.throughput(Throughput::Bytes((num_pages * PAGE_SIZE) as u64)); - let harness = Mutex::new(GlobalHarness::new(2)); - - group.bench_with_input( - BenchmarkId::from_parameter(num_pages), - &num_pages, - |b, &pages| { - b.iter(|| { - let guard = harness.lock(); - let addr = guard - .allocator - .alloc_pages(black_box(pages), PAGE_SIZE) - .unwrap(); - guard.allocator.dealloc_pages(addr, pages); - black_box(addr); - }); - }, - ); } - - group.finish(); -} - -criterion_group! { - name = benches; - config = criterion_config(); - targets = - bench_small_alloc_free, - bench_large_alloc_free, - bench_mixed_workload, - bench_page_interface } -criterion_main!(benches); diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index 38c7f95..773f45c 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -1,193 +1,101 @@ -//! Benchmarks for slab-based small-object allocation paths. +//! Benchmarks for the slab allocator with buddy-backed slab refill. mod common; -use buddy_slab_allocator::{ - CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, -}; -use common::{criterion_config, TestHeap, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use std::alloc::Layout; -use std::hint::black_box; +use common::{SlabHarness, HEAP_SIZE, OPERATIONS_PER_BATCH}; +use core::alloc::Layout; +use divan::{black_box, Bencher}; -struct SlabHarness { - _heap: TestHeap, - page_allocator: CompositePageAllocator, - slab_allocator: SlabByteAllocator, +fn main() { + divan::main(); } -impl SlabHarness { - fn new() -> Self { - let heap = TestHeap::new(HEAP_SIZE); - let mut page_allocator = CompositePageAllocator::::new(); - page_allocator.init(heap.addr(), HEAP_SIZE); +#[divan::bench_group] +mod slab { + use super::*; - Self { - _heap: heap, - page_allocator, - slab_allocator: SlabByteAllocator::new(), - } - } - - fn alloc_small(&mut self, layout: Layout) -> core::ptr::NonNull { - loop { - match self.slab_allocator.alloc(layout).unwrap() { - SlabAllocDecision::Allocated(ptr, _) => return ptr, - SlabAllocDecision::NeedsRefill { - size_class, - page_count, - slab_bytes, - } => { - let slab_base = self - .page_allocator - .alloc_pages(page_count, slab_bytes) - .unwrap(); - self.slab_allocator - .provide_slab(size_class, 0, slab_base, slab_bytes) - .unwrap(); - } - } - } - } - - fn dealloc_small(&mut self, ptr: core::ptr::NonNull, layout: Layout) { - if let SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - .. - } = self.slab_allocator.dealloc(ptr, layout) - { - self.page_allocator.dealloc_pages(slab_base, page_count); - } - } -} + #[divan::bench(args = [8usize, 64, 256, 512, 1024, 2048])] + fn size_class_alloc_free(bencher: Bencher, size: usize) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(size, size).unwrap(); -fn bench_size_class_alloc_free(c: &mut Criterion) { - let mut group = c.benchmark_group("slab/size_class_alloc_free"); - - for &size in &[8usize, 64, 256, 512, 1024, 2048] { - group.throughput(Throughput::Bytes(size as u64)); - let mut harness = SlabHarness::new(); - let layout = Layout::from_size_align(size, 8).unwrap(); - - group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { - b.iter(|| { - let ptr = harness.alloc_small(black_box(layout)); - harness.dealloc_small(ptr, layout); - black_box(ptr); - }); + bencher.bench_local(|| { + let ptr = harness.alloc(black_box(layout)); + harness.dealloc(ptr, layout); + black_box(ptr) }); } - group.finish(); -} - -fn bench_hot_reuse(c: &mut Criterion) { - let mut group = c.benchmark_group("slab/hot_reuse"); - group.throughput(Throughput::Elements(1)); - - let mut harness = SlabHarness::new(); - let layout = Layout::from_size_align(128, 8).unwrap(); - let ptr = harness.alloc_small(layout); - harness.dealloc_small(ptr, layout); - - group.bench_function("alloc_free_hot_cache", |b| { - b.iter(|| { - let ptr = harness.alloc_small(layout); - harness.dealloc_small(ptr, layout); - black_box(ptr); + #[divan::bench] + fn hot_reuse(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(128, 128).unwrap(); + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + + bencher.bench_local(|| { + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + black_box(ptr) }); - }); - - group.finish(); -} - -fn bench_mixed_size_batch(c: &mut Criterion) { - let mut group = c.benchmark_group("slab/mixed_size_batch"); - let layouts = [ - Layout::from_size_align(8, 8).unwrap(), - Layout::from_size_align(64, 8).unwrap(), - Layout::from_size_align(256, 8).unwrap(), - Layout::from_size_align(512, 8).unwrap(), - Layout::from_size_align(1024, 8).unwrap(), - Layout::from_size_align(2048, 8).unwrap(), - ]; - let total_bytes: usize = layouts.iter().map(Layout::size).sum(); - group.throughput(Throughput::Bytes(total_bytes as u64)); - - let mut harness = SlabHarness::new(); + } - group.bench_function("alloc_free_layout_batch", |b| { - b.iter(|| { + #[divan::bench] + fn mixed_size_batch(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 64).unwrap(), + Layout::from_size_align(256, 256).unwrap(), + Layout::from_size_align(512, 512).unwrap(), + Layout::from_size_align(1024, 1024).unwrap(), + Layout::from_size_align(2048, 2048).unwrap(), + ]; + + bencher.bench_local(|| { for layout in layouts { - let ptr = harness.alloc_small(layout); - harness.dealloc_small(ptr, layout); + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); black_box(ptr); } }); - }); - - group.finish(); -} - -fn bench_steady_state(c: &mut Criterion) { - let mut group = c.benchmark_group("slab/steady_state"); - group.throughput(Throughput::Elements(1)); - - let mut harness = SlabHarness::new(); - let layout = Layout::from_size_align(64, 8).unwrap(); - let mut active = Vec::with_capacity(256); - for _ in 0..256 { - active.push(harness.alloc_small(layout)); } - group.bench_function("recycle_live_object", |b| { - b.iter(|| { + #[divan::bench] + fn steady_state_recycle(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(64, 64).unwrap(); + let mut active = Vec::with_capacity(256); + for _ in 0..256 { + active.push(harness.alloc(layout)); + } + + bencher.bench_local(|| { let ptr = active.pop().unwrap(); - harness.dealloc_small(ptr, layout); - let new_ptr = harness.alloc_small(layout); + harness.dealloc(ptr, layout); + let new_ptr = harness.alloc(layout); active.push(new_ptr); - black_box(new_ptr); + black_box(new_ptr) }); - }); - - group.finish(); -} - -fn bench_batch_workload(c: &mut Criterion) { - let mut group = c.benchmark_group("slab/batch_workload"); - group.throughput(Throughput::Elements(OPERATIONS_PER_BATCH)); - - let mut harness = SlabHarness::new(); - let layouts = [ - Layout::from_size_align(8, 8).unwrap(), - Layout::from_size_align(64, 8).unwrap(), - Layout::from_size_align(256, 8).unwrap(), - Layout::from_size_align(1024, 8).unwrap(), - ]; + } - group.bench_function("round_robin_layouts", |b| { - b.iter(|| { - for idx in 0..OPERATIONS_PER_BATCH as usize { + #[divan::bench] + fn mixed_size_workload(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 64).unwrap(), + Layout::from_size_align(256, 256).unwrap(), + Layout::from_size_align(1024, 1024).unwrap(), + ]; + + bencher.bench_local(|| { + for idx in 0..OPERATIONS_PER_BATCH { let layout = layouts[idx % layouts.len()]; - let ptr = harness.alloc_small(layout); - harness.dealloc_small(ptr, layout); + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); black_box(ptr); } }); - }); - - group.finish(); -} - -criterion_group! { - name = benches; - config = criterion_config(); - targets = - bench_size_class_alloc_free, - bench_hot_reuse, - bench_mixed_size_batch, - bench_steady_state, - bench_batch_workload + } } -criterion_main!(benches); diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index c6e2f36..9fa8266 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -333,15 +333,28 @@ impl BuddyAllocator { /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). /// - /// `addr` must be the exact address returned by alloc, and `count` must match. + /// `addr` must be the exact address returned by alloc. The allocator frees + /// the full block size recorded in page metadata, which may be larger than + /// `count` if the original allocation was rounded up for buddy order or alignment. pub fn dealloc_pages(&mut self, addr: usize, count: usize) { debug_assert!(is_aligned(addr, PAGE_SIZE)); debug_assert!(addr >= self.heap_start); + debug_assert!(count > 0); let pfn = (addr - self.heap_start) / PAGE_SIZE; debug_assert!(pfn < self.max_pages); + let stored = unsafe { &*self.meta.add(pfn) }; + debug_assert!( + stored.flags == PageFlags::Allocated || stored.flags == PageFlags::Slab, + "dealloc_pages called on non-allocated block" + ); - let order = count.next_power_of_two().trailing_zeros() as usize; + let expected_order = count.next_power_of_two().trailing_zeros() as usize; + let order = stored.order as usize; + debug_assert!( + expected_order <= order, + "dealloc_pages count implies larger order than the allocated block" + ); self.dealloc_order(pfn, order); } diff --git a/src/global.rs b/src/global.rs index f53292e..b125d0c 100644 --- a/src/global.rs +++ b/src/global.rs @@ -15,6 +15,15 @@ use crate::slab::size_class::{SizeClass, SLAB_MAX_SIZE}; use crate::slab::{SlabAllocResult, SlabAllocator, SlabDeallocResult}; use crate::{align_up, OsImpl}; +struct RegionLayout { + meta_start: usize, + meta_size: usize, + buddy_meta_size: usize, + slab_offset: usize, + managed_heap_start: usize, + managed_heap_size: usize, +} + /// Unified allocator: buddy page allocator + per-CPU slab caches. pub struct GlobalAllocator { buddy: SpinMutex>, @@ -25,8 +34,8 @@ pub struct GlobalAllocator { } // SAFETY: All mutable state is behind SpinMutex or AtomicBool. -// `per_cpu_slabs` is a raw pointer into the metadata region whose lifetime -// is managed by the caller (same as BuddyAllocator::meta). +// `per_cpu_slabs` is a raw pointer into the reserved metadata prefix of the +// caller-provided region (same lifetime model as BuddyAllocator::meta). unsafe impl Sync for GlobalAllocator {} unsafe impl Send for GlobalAllocator {} @@ -35,23 +44,8 @@ impl GlobalAllocator { // Metadata sizing // ------------------------------------------------------------------ - /// Calculate the total metadata region size needed. - /// - /// The metadata region is laid out as: - /// ```text - /// [ PageMeta[max_pages] | padding | SpinMutex[cpu_count] ] - /// ``` - pub const fn required_metadata_size(heap_size: usize, cpu_count: usize) -> usize { - let buddy_meta = BuddyAllocator::::required_meta_size(heap_size); - // Align slab array to its natural alignment. - let slab_align = core::mem::align_of::>>(); - let slab_offset = (buddy_meta + slab_align - 1) & !(slab_align - 1); - let slab_size = core::mem::size_of::>>() * cpu_count; - slab_offset + slab_size - } - - /// Required alignment for the metadata region. - pub const fn required_metadata_align() -> usize { + /// Required alignment for the reserved metadata prefix. + const fn metadata_align() -> usize { let a1 = core::mem::align_of::(); let a2 = core::mem::align_of::>>(); if a1 > a2 { @@ -61,6 +55,96 @@ impl GlobalAllocator { } } + fn metadata_layout_for_pages( + pages: usize, + cpu_count: usize, + ) -> Option<(usize, usize, usize)> { + let buddy_meta_size = pages.checked_mul(core::mem::size_of::())?; + let slab_align = core::mem::align_of::>>(); + let slab_offset = align_up(buddy_meta_size, slab_align); + let slab_size = + core::mem::size_of::>>().checked_mul(cpu_count)?; + let meta_size = slab_offset.checked_add(slab_size)?; + Some((buddy_meta_size, slab_offset, meta_size)) + } + + fn available_heap_pages(region_end: usize, meta_start: usize, meta_size: usize) -> Option { + let managed_heap_start = align_up(meta_start.checked_add(meta_size)?, PAGE_SIZE); + if managed_heap_start > region_end { + return Some(0); + } + Some((region_end - managed_heap_start) / PAGE_SIZE) + } + + fn can_manage_pages( + region_end: usize, + meta_start: usize, + cpu_count: usize, + pages: usize, + ) -> bool { + let Some((_, _, meta_size)) = Self::metadata_layout_for_pages(pages, cpu_count) else { + return false; + }; + let Some(available_pages) = + Self::available_heap_pages(region_end, meta_start, meta_size) + else { + return false; + }; + available_pages >= pages + } + + fn compute_region_layout( + region_start: usize, + region_size: usize, + cpu_count: usize, + ) -> Option { + if cpu_count == 0 || region_size == 0 || !PAGE_SIZE.is_power_of_two() { + return None; + } + + let region_end = region_start.checked_add(region_size)?; + let meta_start = align_up(region_start, Self::metadata_align()); + if meta_start >= region_end { + return None; + } + + let heap_search_start = align_up(region_start, PAGE_SIZE); + let max_pages = if heap_search_start >= region_end { + 0 + } else { + (region_end - heap_search_start) / PAGE_SIZE + }; + + let mut low = 0usize; + let mut high = max_pages; + while low < high { + let mid = low + (high - low).div_ceil(2); + if Self::can_manage_pages(region_end, meta_start, cpu_count, mid) { + low = mid; + } else { + high = mid - 1; + } + } + + if low == 0 { + return None; + } + + let (buddy_meta_size, slab_offset, meta_size) = + Self::metadata_layout_for_pages(low, cpu_count)?; + let managed_heap_start = align_up(meta_start.checked_add(meta_size)?, PAGE_SIZE); + let managed_heap_size = low.checked_mul(PAGE_SIZE)?; + + Some(RegionLayout { + meta_start, + meta_size, + buddy_meta_size, + slab_offset, + managed_heap_start, + managed_heap_size, + }) + } + // ------------------------------------------------------------------ // Construction / initialisation // ------------------------------------------------------------------ @@ -87,47 +171,35 @@ impl GlobalAllocator { /// Initialise the allocator. /// /// # Arguments - /// - `meta` / `meta_size` — writable metadata region (see [`required_metadata_size`]). - /// - `heap_start` / `heap_size` — the heap virtual address range to manage. + /// - `region_start` / `region_size` — total writable region to reserve for this allocator. + /// The allocator carves metadata from the region prefix and manages the remaining tail. /// - `cpu_count` — number of CPUs (≥ 1). /// - `os` — platform abstraction. /// /// # Safety - /// - The metadata and heap regions must not overlap. - /// - `meta` must be aligned to [`required_metadata_align`]. - /// - Both regions must remain valid for the lifetime of this allocator. + /// - `region_start..region_start + region_size` must be writable and remain valid for the + /// lifetime of this allocator. + /// - Any bytes consumed by metadata or alignment padding become unavailable for allocation. pub unsafe fn init( &self, - meta: *mut u8, - meta_size: usize, - heap_start: usize, - heap_size: usize, + region_start: usize, + region_size: usize, cpu_count: usize, os: &'static dyn OsImpl, ) -> AllocResult { - if cpu_count == 0 || meta.is_null() { - return Err(AllocError::InvalidParam); - } - let required = Self::required_metadata_size(heap_size, cpu_count); - if meta_size < required { - return Err(AllocError::InvalidParam); - } - - // --- Partition metadata region --- - let buddy_meta_size = BuddyAllocator::::required_meta_size(heap_size); - let buddy_meta_ptr = meta; - - let slab_align = core::mem::align_of::>>(); - let slab_offset = align_up(buddy_meta_size, slab_align); - let slab_ptr = meta.add(slab_offset) as *mut SpinMutex>; + let layout = Self::compute_region_layout(region_start, region_size, cpu_count) + .ok_or(AllocError::InvalidParam)?; + let meta_ptr = layout.meta_start as *mut u8; + let slab_ptr = + meta_ptr.add(layout.slab_offset) as *mut SpinMutex>; // --- Init buddy --- let mut buddy = self.buddy.lock(); buddy.init( - buddy_meta_ptr, - buddy_meta_size, - heap_start, - heap_size, + meta_ptr, + layout.buddy_meta_size, + layout.managed_heap_start, + layout.managed_heap_size, Some(os), )?; drop(buddy); @@ -146,17 +218,29 @@ impl GlobalAllocator { self.initialized.store(true, Ordering::Release); log::debug!( - "GlobalAllocator: {} CPUs, heap {:#x}+{:#x}, meta {:#x}+{:#x}", + "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, managed heap {:#x}+{:#x}", cpu_count, - heap_start, - heap_size, - meta as usize, - meta_size, + region_start, + region_size, + layout.meta_start, + layout.meta_size, + layout.managed_heap_start, + layout.managed_heap_size, ); Ok(()) } + /// Start address of the heap range actually managed by the buddy allocator. + pub fn managed_heap_start(&self) -> usize { + self.buddy.lock().heap_start() + } + + /// Size in bytes of the heap range actually managed by the buddy allocator. + pub fn managed_heap_size(&self) -> usize { + self.buddy.lock().heap_size() + } + // ------------------------------------------------------------------ // Public page-level API (forwarded to buddy) // ------------------------------------------------------------------ @@ -227,7 +311,6 @@ impl GlobalAllocator { SlabAllocResult::NeedsSlab { size_class, pages } => { drop(slab); // release slab lock before locking buddy let bytes = pages * PAGE_SIZE; - // Align to slab_bytes so base_from_obj_addr works via address masking. let addr = self.buddy.lock().alloc_pages(pages, bytes)?; // Tag the page as SLAB in buddy metadata. unsafe { @@ -247,7 +330,8 @@ impl GlobalAllocator { let os = self.os.expect("not initialized"); let sc = SizeClass::from_layout(layout).expect("layout exceeds slab"); let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE; - let base = SlabPageHeader::base_from_obj_addr(ptr.as_ptr() as usize, slab_bytes); + let base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); let hdr = &*(base as *const SlabPageHeader); debug_assert_eq!(hdr.magic, SLAB_MAGIC); diff --git a/src/slab/cache.rs b/src/slab/cache.rs index ae334aa..47e8b8f 100644 --- a/src/slab/cache.rs +++ b/src/slab/cache.rs @@ -155,7 +155,7 @@ impl SlabCache { obj_addr: usize, ) -> CacheDeallocResult { let slab_bytes = self.size_class.slab_pages(PAGE_SIZE) * PAGE_SIZE; - let base = SlabPageHeader::base_from_obj_addr(obj_addr, slab_bytes); + let base = SlabPageHeader::base_from_obj_addr::(obj_addr, slab_bytes); let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; let was_full = hdr.is_local_full() && !hdr.has_remote_frees(); diff --git a/src/slab/page.rs b/src/slab/page.rs index fe88754..12a7a4c 100644 --- a/src/slab/page.rs +++ b/src/slab/page.rs @@ -114,11 +114,33 @@ impl SlabPageHeader { (addr - self.data_start(base)) / self.size_class.size() } - /// Base address (page start) from an object address. - /// Relies on slab_bytes being a power-of-two multiple of PAGE_SIZE. + /// Base address (slab start) from an object address. + /// + /// Searches backward by page because the slab base may no longer have + /// absolute `slab_bytes` alignment after metadata is carved from a region + /// prefix by [`GlobalAllocator`](crate::GlobalAllocator). #[inline] - pub fn base_from_obj_addr(addr: usize, slab_bytes: usize) -> usize { - addr & !(slab_bytes - 1) + pub fn base_from_obj_addr(addr: usize, slab_bytes: usize) -> usize { + let slab_pages = slab_bytes / PAGE_SIZE; + debug_assert!(slab_pages > 0); + + let page_base = addr & !(PAGE_SIZE - 1); + for page_idx in 0..slab_pages { + let Some(candidate) = page_base.checked_sub(page_idx * PAGE_SIZE) else { + break; + }; + let hdr = unsafe { &*(candidate as *const SlabPageHeader) }; + if hdr.magic == SLAB_MAGIC + && hdr.slab_bytes as usize == slab_bytes + && addr >= candidate + && addr < candidate + slab_bytes + { + return candidate; + } + } + + debug_assert!(false, "object address does not belong to a live slab"); + page_base } // ------------------------------------------------------------------ diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index 8e1364b..08ba43c 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -39,83 +39,73 @@ fn host_dealloc(ptr: *mut u8, layout: Layout) { fn init_allocator( allocator: &GlobalAllocator, - heap_addr: usize, - heap_size: usize, -) -> (*mut u8, Layout) { - let meta_size = GlobalAllocator::::required_metadata_size(heap_size, 1); - let meta_align = GlobalAllocator::::required_metadata_align().max(8); - let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align); - unsafe { - allocator - .init(meta_ptr, meta_size, heap_addr, heap_size, 1, &MOCK_OS) - .unwrap(); - } - (meta_ptr, meta_layout) + region_addr: usize, + region_size: usize, +) { + unsafe { allocator.init(region_addr, region_size, 1, &MOCK_OS).unwrap() }; } #[test] fn test_lowmem_basic() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); + init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); + let managed_start = allocator.managed_heap_start(); + let managed_end = managed_start + allocator.managed_heap_size(); let addr1 = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr2 = allocator.alloc_pages_lowmem(4, PAGE_SIZE).unwrap(); - assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); - assert!(addr2 >= heap_addr && addr2 < heap_addr + TEST_HEAP_SIZE); + assert!(addr1 >= managed_start && addr1 < managed_end); + assert!(addr2 >= managed_start && addr2 < managed_end); assert_eq!(addr1 % PAGE_SIZE, 0); assert_eq!(addr2 % PAGE_SIZE, 0); allocator.dealloc_pages(addr1, 1); allocator.dealloc_pages(addr2, 4); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_aligned() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 2); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 2); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); + init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); - assert_eq!(addr % (2 * PAGE_SIZE), 0); + assert_eq!( + (addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), + 0 + ); allocator.dealloc_pages(addr, 1); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_vs_normal() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); + init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - assert!(addr_low >= heap_addr); - assert!(addr_normal >= heap_addr); + assert!(addr_low >= allocator.managed_heap_start()); + assert!(addr_normal >= allocator.managed_heap_start()); allocator.dealloc_pages(addr_low, 1); allocator.dealloc_pages(addr_normal, 1); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_stress() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, heap_addr, TEST_HEAP_SIZE); + init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); let mut addrs = Vec::new(); for _ in 0..32 { @@ -126,6 +116,5 @@ fn test_lowmem_stress() { allocator.dealloc_pages(addr, 1); } - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 693aefb..167053a 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -3,7 +3,7 @@ extern crate buddy_slab_allocator; use buddy_slab_allocator::{ - BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, SlabAllocResult, SlabAllocator, + AllocError, BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; @@ -60,6 +60,10 @@ fn host_dealloc(ptr: *mut u8, layout: Layout) { unsafe { dealloc(ptr, layout) }; } +fn align_up(value: usize, align: usize) -> usize { + (value + align - 1) & !(align - 1) +} + // ====================================================================== // Buddy allocator (standalone) tests // ====================================================================== @@ -123,6 +127,30 @@ fn buddy_alignment() { host_dealloc(heap_ptr, heap_layout); } +#[test] +fn buddy_aligned_alloc_dealloc_uses_recorded_order() { + let heap_size = 64 * PAGE_SIZE; + let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE * 16); + let heap_addr = heap_ptr as usize; + let meta_size = BuddyAllocator::::required_meta_size(heap_size); + let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .unwrap(); + } + + let free_before = buddy.free_pages(); + let addr = buddy.alloc_pages(4, PAGE_SIZE * 16).unwrap(); + buddy.dealloc_pages(addr, 4); + assert_eq!(buddy.free_pages(), free_before); + + host_dealloc(meta_ptr, meta_layout); + host_dealloc(heap_ptr, heap_layout); +} + #[test] fn buddy_exhaust_and_recover() { let heap_size = 64 * PAGE_SIZE; // Small heap @@ -334,75 +362,63 @@ fn slab_all_size_classes() { fn init_global( allocator: &GlobalAllocator, - heap_addr: usize, - heap_size: usize, + region_start: usize, + region_size: usize, cpu_count: usize, -) -> (*mut u8, Layout) { - let meta_size = GlobalAllocator::::required_metadata_size(heap_size, cpu_count); - let meta_align = GlobalAllocator::::required_metadata_align(); - let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align.max(8)); - unsafe { - allocator - .init( - meta_ptr, meta_size, heap_addr, heap_size, cpu_count, &TEST_OS, - ) - .unwrap(); - } - (meta_ptr, meta_layout) +) { + unsafe { allocator.init(region_start, region_size, cpu_count, &TEST_OS).unwrap() }; } #[test] fn global_page_alloc() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let region_addr = region_ptr as usize; let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); + init_global(&allocator, region_addr, TEST_HEAP_SIZE, 1); + + let managed_start = allocator.managed_heap_start(); + let managed_end = managed_start + allocator.managed_heap_size(); let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); - assert!(addr >= heap_addr && addr < heap_addr + TEST_HEAP_SIZE); + assert!(managed_start > region_addr); + assert!(addr >= managed_start && addr < managed_end); assert_eq!(addr % PAGE_SIZE, 0); allocator.dealloc_pages(addr, 4); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn global_small_alloc() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn global_large_alloc() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn global_mixed_alloc() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 1); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); let sizes: &[(usize, usize)] = &[ (8, 8), @@ -421,16 +437,14 @@ fn global_mixed_alloc() { unsafe { allocator.dealloc(ptr, layout) }; } - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] fn global_cross_cpu_free() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_global(&allocator, heap_addr, TEST_HEAP_SIZE, 2); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); // Allocate on CPU 0 TEST_OS.set_cpu(0); @@ -451,8 +465,7 @@ fn global_cross_cpu_free() { let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); } #[test] @@ -473,30 +486,52 @@ fn global_lowmem_pages() { } static LOWMEM_OS: LowmemOs = LowmemOs; - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let region_addr = region_ptr as usize; let allocator = GlobalAllocator::::new(); - let cpu_count = 1; - let meta_size = GlobalAllocator::::required_metadata_size(TEST_HEAP_SIZE, cpu_count); - let meta_align = GlobalAllocator::::required_metadata_align(); - let (meta_ptr, meta_layout) = host_alloc(meta_size, meta_align.max(8)); - unsafe { - allocator - .init( - meta_ptr, - meta_size, - heap_addr, - TEST_HEAP_SIZE, - cpu_count, - &LOWMEM_OS, - ) - .unwrap(); - } + unsafe { allocator.init(region_addr, TEST_HEAP_SIZE, 1, &LOWMEM_OS).unwrap() }; let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); - assert!(addr >= heap_addr); + assert!(addr >= allocator.managed_heap_start()); allocator.dealloc_pages(addr, 1); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + host_dealloc(region_ptr, region_layout); +} + +#[test] +fn global_unaligned_region_start() { + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE + PAGE_SIZE, PAGE_SIZE * 4); + let region_start = region_ptr as usize + 1; + let region_size = TEST_HEAP_SIZE; + let allocator = GlobalAllocator::::new(); + init_global(&allocator, region_start, region_size, 1); + + let managed_start = allocator.managed_heap_start(); + let managed_end = managed_start + allocator.managed_heap_size(); + + assert_eq!(managed_start % PAGE_SIZE, 0); + assert!(managed_start >= region_start); + assert!(managed_end <= region_start + region_size); + + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= managed_start && addr < managed_end); + allocator.dealloc_pages(addr, 1); + + host_dealloc(region_ptr, region_layout); +} + +#[test] +fn global_rejects_region_without_one_managed_page() { + let buddy_meta_size = BuddyAllocator::::required_meta_size(PAGE_SIZE); + let slab_align = core::mem::align_of::>>(); + let slab_offset = align_up(buddy_meta_size, slab_align); + let slab_size = core::mem::size_of::>>(); + let region_size = PAGE_SIZE + slab_offset + slab_size - 1; + let (region_ptr, region_layout) = host_alloc(region_size, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + + let err = unsafe { allocator.init(region_ptr as usize, region_size, 1, &TEST_OS) }.unwrap_err(); + assert_eq!(err, AllocError::InvalidParam); + + host_dealloc(region_ptr, region_layout); } diff --git a/tests/stress_test.rs b/tests/stress_test.rs index 96bfb5a..192ae8e 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -62,35 +62,13 @@ impl Drop for TestHeap { } } -fn alloc_metadata(heap_size: usize, cpu_count: usize) -> (*mut u8, Layout) { - let size = GlobalAllocator::::required_metadata_size(heap_size, cpu_count); - let align = GlobalAllocator::::required_metadata_align().max(8); - let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "failed to allocate metadata"); - (ptr, layout) -} - fn init_allocator( allocator: &GlobalAllocator, heap: &TestHeap, cpu_count: usize, -) -> (*mut u8, Layout) { - let (meta_ptr, meta_layout) = alloc_metadata(HEAP_SIZE, cpu_count); +) { MOCK_OS.set_cpu(0); - unsafe { - allocator - .init( - meta_ptr, - meta_layout.size(), - heap.addr(), - HEAP_SIZE, - cpu_count, - &MOCK_OS, - ) - .unwrap(); - } - (meta_ptr, meta_layout) + unsafe { allocator.init(heap.addr(), HEAP_SIZE, cpu_count, &MOCK_OS).unwrap() }; } #[test] @@ -98,7 +76,7 @@ fn init_allocator( fn stress_random_mixed_alloc_free() { let heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 2); + init_allocator(&allocator, &heap, 2); let mut rng = rand::rngs::StdRng::from_seed([0; 32]); let mut allocated: Vec<(core::ptr::NonNull, Layout)> = Vec::new(); @@ -126,8 +104,6 @@ fn stress_random_mixed_alloc_free() { for (ptr, layout) in allocated { unsafe { allocator.dealloc(ptr, layout) }; } - - unsafe { dealloc(meta_ptr, meta_layout) }; } #[test] @@ -135,7 +111,7 @@ fn stress_random_mixed_alloc_free() { fn stress_exhaustion_recovery() { let heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 1); + init_allocator(&allocator, &heap, 1); let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); let mut allocated = Vec::new(); @@ -157,8 +133,6 @@ fn stress_exhaustion_recovery() { for ptr in allocated { unsafe { allocator.dealloc(ptr, layout) }; } - - unsafe { dealloc(meta_ptr, meta_layout) }; } #[test] @@ -166,7 +140,7 @@ fn stress_exhaustion_recovery() { fn stress_fragmentation_recovery() { let heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - let (meta_ptr, meta_layout) = init_allocator(&allocator, &heap, 2); + init_allocator(&allocator, &heap, 2); let small_layout = Layout::from_size_align(64, 8).unwrap(); let mut small_ptrs = Vec::new(); @@ -191,6 +165,4 @@ fn stress_fragmentation_recovery() { if let Ok(ptr) = large { unsafe { allocator.dealloc(ptr, large_layout) }; } - - unsafe { dealloc(meta_ptr, meta_layout) }; } From 41607aefe17d37bb7c72489ddbe512af9dc3a009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 08:58:07 +0800 Subject: [PATCH 10/19] feat: enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation --- benches/buddy_allocator.rs | 4 +- benches/global_allocator.rs | 2 +- src/global.rs | 17 +++--- src/slab/cache.rs | 25 ++++++++- tests/dma32_pages_test.rs | 17 +++--- tests/integration_test.rs | 100 ++++++++++++++++++++++++++++++++++-- tests/stress_test.rs | 12 ++--- 7 files changed, 145 insertions(+), 32 deletions(-) diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index fa8e26a..19ffe5b 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -2,7 +2,9 @@ mod common; -use common::{seeded_rng, BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use common::{ + seeded_rng, BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE, +}; use divan::{black_box, Bencher}; use rand::RngExt; diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index 1ce78f6..42f6109 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -2,7 +2,7 @@ mod common; -use common::{GlobalHarness, MOCK_OS, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use common::{GlobalHarness, HEAP_SIZE, MOCK_OS, OPERATIONS_PER_BATCH, PAGE_SIZE}; use core::alloc::Layout; use divan::{black_box, Bencher}; diff --git a/src/global.rs b/src/global.rs index b125d0c..92f7f55 100644 --- a/src/global.rs +++ b/src/global.rs @@ -55,10 +55,7 @@ impl GlobalAllocator { } } - fn metadata_layout_for_pages( - pages: usize, - cpu_count: usize, - ) -> Option<(usize, usize, usize)> { + fn metadata_layout_for_pages(pages: usize, cpu_count: usize) -> Option<(usize, usize, usize)> { let buddy_meta_size = pages.checked_mul(core::mem::size_of::())?; let slab_align = core::mem::align_of::>>(); let slab_offset = align_up(buddy_meta_size, slab_align); @@ -68,7 +65,11 @@ impl GlobalAllocator { Some((buddy_meta_size, slab_offset, meta_size)) } - fn available_heap_pages(region_end: usize, meta_start: usize, meta_size: usize) -> Option { + fn available_heap_pages( + region_end: usize, + meta_start: usize, + meta_size: usize, + ) -> Option { let managed_heap_start = align_up(meta_start.checked_add(meta_size)?, PAGE_SIZE); if managed_heap_start > region_end { return Some(0); @@ -85,8 +86,7 @@ impl GlobalAllocator { let Some((_, _, meta_size)) = Self::metadata_layout_for_pages(pages, cpu_count) else { return false; }; - let Some(available_pages) = - Self::available_heap_pages(region_end, meta_start, meta_size) + let Some(available_pages) = Self::available_heap_pages(region_end, meta_start, meta_size) else { return false; }; @@ -190,8 +190,7 @@ impl GlobalAllocator { let layout = Self::compute_region_layout(region_start, region_size, cpu_count) .ok_or(AllocError::InvalidParam)?; let meta_ptr = layout.meta_start as *mut u8; - let slab_ptr = - meta_ptr.add(layout.slab_offset) as *mut SpinMutex>; + let slab_ptr = meta_ptr.add(layout.slab_offset) as *mut SpinMutex>; // --- Init buddy --- let mut buddy = self.buddy.lock(); diff --git a/src/slab/cache.rs b/src/slab/cache.rs index 47e8b8f..3473fa3 100644 --- a/src/slab/cache.rs +++ b/src/slab/cache.rs @@ -107,7 +107,13 @@ impl SlabCache { return Some(addr); } - // 2. Try recycling an empty slab. + // 2. A full slab may have gained free objects via lock-free remote frees. + if let Some(base) = self.reclaim_full_with_remote_frees() { + unsafe { self.partial.push_front(base) }; + return self.try_alloc_from_partial::(); + } + + // 3. Try recycling an empty slab. if !self.empty.is_empty() { let base = unsafe { self.empty.pop_front() }; self.empty_count -= 1; @@ -119,6 +125,23 @@ impl SlabCache { None } + /// Drain remote frees from the first full slab that has them and move it + /// back to the partial list. + fn reclaim_full_with_remote_frees(&mut self) -> Option { + let mut base = self.full.first; + while base != 0 { + let next = unsafe { (*(base as *const SlabPageHeader)).list_next }; + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + unsafe { self.full.remove(base) }; + return Some(base); + } + base = next; + } + None + } + /// Attempt allocation from the first partial slab. fn try_alloc_from_partial(&mut self) -> Option { let base = self.partial.first; diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index 08ba43c..47e1471 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -37,12 +37,12 @@ fn host_dealloc(ptr: *mut u8, layout: Layout) { unsafe { dealloc(ptr, layout) }; } -fn init_allocator( - allocator: &GlobalAllocator, - region_addr: usize, - region_size: usize, -) { - unsafe { allocator.init(region_addr, region_size, 1, &MOCK_OS).unwrap() }; +fn init_allocator(allocator: &GlobalAllocator, region_addr: usize, region_size: usize) { + unsafe { + allocator + .init(region_addr, region_size, 1, &MOCK_OS) + .unwrap() + }; } #[test] @@ -74,10 +74,7 @@ fn test_lowmem_aligned() { init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); - assert_eq!( - (addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), - 0 - ); + assert_eq!((addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), 0); allocator.dealloc_pages(addr, 1); host_dealloc(region_ptr, region_layout); diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 167053a..ea234f5 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -3,8 +3,8 @@ extern crate buddy_slab_allocator; use buddy_slab_allocator::{ - AllocError, BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, SlabAllocResult, SlabAllocator, - SlabDeallocResult, + slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, + SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -366,7 +366,11 @@ fn init_global( region_size: usize, cpu_count: usize, ) { - unsafe { allocator.init(region_start, region_size, cpu_count, &TEST_OS).unwrap() }; + unsafe { + allocator + .init(region_start, region_size, cpu_count, &TEST_OS) + .unwrap() + }; } #[test] @@ -468,6 +472,90 @@ fn global_cross_cpu_free() { host_dealloc(region_ptr, region_layout); } +#[test] +fn global_cross_cpu_free_drains_remote_queue() { + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); + + TEST_OS.set_cpu(0); + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + assert_eq!(hdr.owner_cpu, 0); + assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); + + TEST_OS.set_cpu(1); + unsafe { allocator.dealloc(ptr, layout) }; + assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 1); + assert_ne!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + + TEST_OS.set_cpu(0); + let ptr2 = allocator.alloc(layout).unwrap(); + assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); + assert_eq!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + unsafe { allocator.dealloc(ptr2, layout) }; + + host_dealloc(region_ptr, region_layout); +} + +#[test] +fn global_cross_cpu_free_multiple_rounds_same_slab() { + let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); + + let layout = Layout::from_size_align(64, 8).unwrap(); + + TEST_OS.set_cpu(0); + let first = allocator.alloc(layout).unwrap(); + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + let ptr = allocator.alloc(layout).unwrap(); + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + ptrs.push(ptr); + } + + TEST_OS.set_cpu(1); + for &ptr in &ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + assert_eq!( + hdr.remote_free_count.load(Ordering::Relaxed) as usize, + object_count + ); + + TEST_OS.set_cpu(0); + let mut drained = Vec::with_capacity(object_count); + for _ in 0..object_count { + drained.push(allocator.alloc(layout).unwrap()); + } + assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); + assert_eq!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; + } + + host_dealloc(region_ptr, region_layout); +} + #[test] fn global_lowmem_pages() { // Use a custom OsImpl that maps virt addresses below 4 GiB so lowmem @@ -489,7 +577,11 @@ fn global_lowmem_pages() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let region_addr = region_ptr as usize; let allocator = GlobalAllocator::::new(); - unsafe { allocator.init(region_addr, TEST_HEAP_SIZE, 1, &LOWMEM_OS).unwrap() }; + unsafe { + allocator + .init(region_addr, TEST_HEAP_SIZE, 1, &LOWMEM_OS) + .unwrap() + }; let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); assert!(addr >= allocator.managed_heap_start()); diff --git a/tests/stress_test.rs b/tests/stress_test.rs index 192ae8e..de6b51b 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -62,13 +62,13 @@ impl Drop for TestHeap { } } -fn init_allocator( - allocator: &GlobalAllocator, - heap: &TestHeap, - cpu_count: usize, -) { +fn init_allocator(allocator: &GlobalAllocator, heap: &TestHeap, cpu_count: usize) { MOCK_OS.set_cpu(0); - unsafe { allocator.init(heap.addr(), HEAP_SIZE, cpu_count, &MOCK_OS).unwrap() }; + unsafe { + allocator + .init(heap.addr(), HEAP_SIZE, cpu_count, &MOCK_OS) + .unwrap() + }; } #[test] From 61360c6d3ea622dc49c2ad7ce1cabb60304e8c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 09:11:39 +0800 Subject: [PATCH 11/19] refactor: update allocator initialization to use a mutable slice instead of separate start and size parameters --- README.md | 10 +++---- src/global.rs | 10 +++---- src/lib.rs | 3 --- tests/dma32_pages_test.rs | 23 +++++++--------- tests/integration_test.rs | 55 ++++++++++++++++----------------------- tests/stress_test.rs | 23 +++++++--------- 6 files changed, 52 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 031bd04..e3a48e7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ classDiagram -UsageStatsAtomic stats -AtomicBool initialized +new() GlobalAllocator - +init(region_start, region_size, cpu_count, os) AllocResult + +init(region, cpu_count, os) AllocResult +add_memory_region(start, size) AllocResult +alloc(Layout) Option~NonNull~u8~~ +dealloc(ptr, Layout) @@ -245,19 +245,17 @@ struct DemoOs; impl OsImpl for DemoOs { fn current_cpu_idx(&self) -> usize { 0 } fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } - fn phys_to_virt(&self, paddr: usize) -> usize { paddr } } static OS: DemoOs = DemoOs; let allocator = GlobalAllocator::::new(); -let region_start = 0x8000_0000; +let region_start = 0x8000_0000 as *mut u8; let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; unsafe { - allocator - .init(region_start, region_size, 1, &OS) - .unwrap(); + allocator.init(region, 1, &OS).unwrap(); } let layout = Layout::from_size_align(64, 8).unwrap(); diff --git a/src/global.rs b/src/global.rs index 92f7f55..6961cec 100644 --- a/src/global.rs +++ b/src/global.rs @@ -171,22 +171,22 @@ impl GlobalAllocator { /// Initialise the allocator. /// /// # Arguments - /// - `region_start` / `region_size` — total writable region to reserve for this allocator. + /// - `region` — total writable region to reserve for this allocator. /// The allocator carves metadata from the region prefix and manages the remaining tail. /// - `cpu_count` — number of CPUs (≥ 1). /// - `os` — platform abstraction. /// /// # Safety - /// - `region_start..region_start + region_size` must be writable and remain valid for the - /// lifetime of this allocator. + /// - `region` must be writable and remain valid for the lifetime of this allocator. /// - Any bytes consumed by metadata or alignment padding become unavailable for allocation. pub unsafe fn init( &self, - region_start: usize, - region_size: usize, + region: &mut [u8], cpu_count: usize, os: &'static dyn OsImpl, ) -> AllocResult { + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); let layout = Self::compute_region_layout(region_start, region_size, cpu_count) .ok_or(AllocError::InvalidParam)?; let meta_ptr = layout.meta_start as *mut u8; diff --git a/src/lib.rs b/src/lib.rs index f8d19be..98d8b5c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,9 +35,6 @@ pub trait OsImpl: Sync + Send { /// Translate a virtual address to a physical address. fn virt_to_phys(&self, vaddr: usize) -> usize; - - /// Translate a physical address to a virtual address. - fn phys_to_virt(&self, paddr: usize) -> usize; } // --------------------------------------------------------------------------- diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index 47e1471..77714aa 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -19,9 +19,6 @@ impl OsImpl for MockOs { // Map virtual addresses below 4 GiB so lowmem checks pass. vaddr & 0x0FFF_FFFF } - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } } static MOCK_OS: MockOs = MockOs; @@ -37,19 +34,16 @@ fn host_dealloc(ptr: *mut u8, layout: Layout) { unsafe { dealloc(ptr, layout) }; } -fn init_allocator(allocator: &GlobalAllocator, region_addr: usize, region_size: usize) { - unsafe { - allocator - .init(region_addr, region_size, 1, &MOCK_OS) - .unwrap() - }; +fn init_allocator(allocator: &GlobalAllocator, region: &mut [u8]) { + unsafe { allocator.init(region, 1, &MOCK_OS).unwrap() }; } #[test] fn test_lowmem_basic() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_allocator(&allocator, region); let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -71,7 +65,8 @@ fn test_lowmem_basic() { fn test_lowmem_aligned() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 2); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_allocator(&allocator, region); let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); assert_eq!((addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), 0); @@ -84,7 +79,8 @@ fn test_lowmem_aligned() { fn test_lowmem_vs_normal() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_allocator(&allocator, region); let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); @@ -102,7 +98,8 @@ fn test_lowmem_vs_normal() { fn test_lowmem_stress() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, region_ptr as usize, TEST_HEAP_SIZE); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_allocator(&allocator, region); let mut addrs = Vec::new(); for _ in 0..32 { diff --git a/tests/integration_test.rs b/tests/integration_test.rs index ea234f5..62f504c 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -41,9 +41,6 @@ impl OsImpl for TestOs { fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr // identity mapping } - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } } static TEST_OS: TestOs = TestOs::new(); @@ -360,17 +357,8 @@ fn slab_all_size_classes() { // Global allocator tests // ====================================================================== -fn init_global( - allocator: &GlobalAllocator, - region_start: usize, - region_size: usize, - cpu_count: usize, -) { - unsafe { - allocator - .init(region_start, region_size, cpu_count, &TEST_OS) - .unwrap() - }; +fn init_global(allocator: &GlobalAllocator, region: &mut [u8], cpu_count: usize) { + unsafe { allocator.init(region, cpu_count, &TEST_OS).unwrap() }; } #[test] @@ -378,7 +366,8 @@ fn global_page_alloc() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let region_addr = region_ptr as usize; let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_addr, TEST_HEAP_SIZE, 1); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 1); let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -396,7 +385,8 @@ fn global_page_alloc() { fn global_small_alloc() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 1); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); @@ -409,7 +399,8 @@ fn global_small_alloc() { fn global_large_alloc() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 1); let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); let ptr = allocator.alloc(layout).unwrap(); @@ -422,7 +413,8 @@ fn global_large_alloc() { fn global_mixed_alloc() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 1); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 1); let sizes: &[(usize, usize)] = &[ (8, 8), @@ -448,7 +440,8 @@ fn global_mixed_alloc() { fn global_cross_cpu_free() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 2); // Allocate on CPU 0 TEST_OS.set_cpu(0); @@ -476,7 +469,8 @@ fn global_cross_cpu_free() { fn global_cross_cpu_free_drains_remote_queue() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 2); TEST_OS.set_cpu(0); let layout = Layout::from_size_align(64, 8).unwrap(); @@ -509,7 +503,8 @@ fn global_cross_cpu_free_drains_remote_queue() { fn global_cross_cpu_free_multiple_rounds_same_slab() { let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_ptr as usize, TEST_HEAP_SIZE, 2); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + init_global(&allocator, region, 2); let layout = Layout::from_size_align(64, 8).unwrap(); @@ -568,20 +563,13 @@ fn global_lowmem_pages() { fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr & 0x0FFF_FFFF } - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } } static LOWMEM_OS: LowmemOs = LowmemOs; let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let region_addr = region_ptr as usize; let allocator = GlobalAllocator::::new(); - unsafe { - allocator - .init(region_addr, TEST_HEAP_SIZE, 1, &LOWMEM_OS) - .unwrap() - }; + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; + unsafe { allocator.init(region, 1, &LOWMEM_OS).unwrap() }; let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); assert!(addr >= allocator.managed_heap_start()); @@ -596,7 +584,9 @@ fn global_unaligned_region_start() { let region_start = region_ptr as usize + 1; let region_size = TEST_HEAP_SIZE; let allocator = GlobalAllocator::::new(); - init_global(&allocator, region_start, region_size, 1); + let region = + unsafe { core::slice::from_raw_parts_mut(region_ptr.wrapping_add(1), region_size) }; + init_global(&allocator, region, 1); let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -621,8 +611,9 @@ fn global_rejects_region_without_one_managed_page() { let region_size = PAGE_SIZE + slab_offset + slab_size - 1; let (region_ptr, region_layout) = host_alloc(region_size, PAGE_SIZE); let allocator = GlobalAllocator::::new(); + let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, region_size) }; - let err = unsafe { allocator.init(region_ptr as usize, region_size, 1, &TEST_OS) }.unwrap_err(); + let err = unsafe { allocator.init(region, 1, &TEST_OS) }.unwrap_err(); assert_eq!(err, AllocError::InvalidParam); host_dealloc(region_ptr, region_layout); diff --git a/tests/stress_test.rs b/tests/stress_test.rs index de6b51b..10d0d2a 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -31,9 +31,6 @@ impl OsImpl for MockOs { fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } } static MOCK_OS: MockOs = MockOs::new(); @@ -51,8 +48,8 @@ impl TestHeap { Self { ptr, layout } } - fn addr(&self) -> usize { - self.ptr as usize + fn as_region(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } } } @@ -62,11 +59,11 @@ impl Drop for TestHeap { } } -fn init_allocator(allocator: &GlobalAllocator, heap: &TestHeap, cpu_count: usize) { +fn init_allocator(allocator: &GlobalAllocator, heap: &mut TestHeap, cpu_count: usize) { MOCK_OS.set_cpu(0); unsafe { allocator - .init(heap.addr(), HEAP_SIZE, cpu_count, &MOCK_OS) + .init(heap.as_region(), cpu_count, &MOCK_OS) .unwrap() }; } @@ -74,9 +71,9 @@ fn init_allocator(allocator: &GlobalAllocator, heap: &TestHeap, cpu_c #[test] #[ignore = "stress test"] fn stress_random_mixed_alloc_free() { - let heap = TestHeap::new(HEAP_SIZE); + let mut heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &heap, 2); + init_allocator(&allocator, &mut heap, 2); let mut rng = rand::rngs::StdRng::from_seed([0; 32]); let mut allocated: Vec<(core::ptr::NonNull, Layout)> = Vec::new(); @@ -109,9 +106,9 @@ fn stress_random_mixed_alloc_free() { #[test] #[ignore = "stress test"] fn stress_exhaustion_recovery() { - let heap = TestHeap::new(HEAP_SIZE); + let mut heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &heap, 1); + init_allocator(&allocator, &mut heap, 1); let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); let mut allocated = Vec::new(); @@ -138,9 +135,9 @@ fn stress_exhaustion_recovery() { #[test] #[ignore = "stress test"] fn stress_fragmentation_recovery() { - let heap = TestHeap::new(HEAP_SIZE); + let mut heap = TestHeap::new(HEAP_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &heap, 2); + init_allocator(&allocator, &mut heap, 2); let small_layout = Layout::from_size_align(64, 8).unwrap(); let mut small_ptrs = Vec::new(); From dc443aea9fdf4748e01b909727655b0b5f8e94e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 09:57:12 +0800 Subject: [PATCH 12/19] Refactor stress tests for allocator stability - Removed the MockOs struct and its implementation, simplifying the test setup. - Introduced HostRegion for managing heap memory in tests. - Updated allocation and deallocation logic to use usize pointers instead of NonNull. - Added multithreaded stress tests for mixed allocation/freeing, remote freeing, page allocation/freeing, fragmentation recovery, and exhaustion recovery. - Enhanced assertions to check for recovery with cached slabs. - Improved random allocation logic and CPU management in tests. --- .github/workflows/check.yml | 6 +- .github/workflows/test.yml | 8 +- Cargo.toml | 11 +- README.md | 390 ++++++++---------------- README_CN.md | 202 +++++++++---- docs/design.md | 567 +++++++++++++++++++++++++++++++++++ src/global.rs | 1 - tests/README.md | 18 +- tests/common/mod.rs | 120 ++++++++ tests/dma32_pages_test.rs | 64 +--- tests/integration_test.rs | 582 ++++++++++++++++++++++++------------ tests/stress_test.rs | 430 ++++++++++++++++++++------ 12 files changed, 1730 insertions(+), 669 deletions(-) create mode 100644 docs/design.md create mode 100644 tests/common/mod.rs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index af80fd1..bc2347e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -37,7 +37,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run clippy - run: cargo clippy --features tracking -- -D warnings + run: cargo clippy --all-targets -- -D warnings build: name: Build @@ -55,7 +55,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build library - run: cargo build --features tracking + run: cargo build doc: name: Documentation @@ -73,7 +73,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build documentation - run: cargo doc --no-deps --features tracking + run: cargo doc --no-deps bench-check: name: Bench Compile Check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0ad077b..45ecba0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,4 +20,10 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run all tests - run: cargo test --features tracking + run: cargo test + + - name: Run tests serially + run: cargo test -- --test-threads=1 + + - name: Run ignored stress tests + run: cargo test --test stress_test -- --ignored --nocapture diff --git a/Cargo.toml b/Cargo.toml index 90e2514..5aeccfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -authors = ["Song Zhiyong "] +authors = ["Song Zhiyong ", "周睿 "] autobenches = false categories = ["memory-management", "no-std", "embedded"] description = "Memory allocator with Buddy and Slab allocation" @@ -12,9 +12,6 @@ readme = "README.md" repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator" version = "0.2.0" -[features] -default = [] - [dependencies] log = "0.4" spin = "0.10" @@ -24,13 +21,13 @@ divan = "0.1.21" rand = {version = "0.10", features = ["std_rng"]} [[bench]] -name = "buddy_allocator" harness = false +name = "buddy_allocator" [[bench]] -name = "slab_allocator" harness = false +name = "slab_allocator" [[bench]] -name = "global_allocator" harness = false +name = "global_allocator" diff --git a/README.md b/README.md index e3a48e7..a89586c 100644 --- a/README.md +++ b/README.md @@ -1,239 +1,91 @@ # buddy-slab-allocator -A `no_std` buddy + slab two-level memory allocator designed for kernel and embedded environments. +A `no_std` two-level allocator for kernel and embedded environments, combining a buddy page allocator with per-CPU slab allocators. -## Architecture Overview +## Overview -The allocator employs a classic **two-level** architecture: +The current implementation is built from three layers: -1. **Buddy page allocator** — manages physical pages, the shared backend -2. **Slab byte allocator** — manages small objects (≤ 2048 bytes), the frontend -3. **GlobalAllocator** — a multi-core facade that routes allocations to per-CPU slab caches or the buddy backend +1. `BuddyAllocator` + Manages a contiguous virtual heap in page units, with power-of-two splitting and merging. +2. `SlabAllocator` + Manages small objects up to 2048 bytes with fixed size classes. +3. `GlobalAllocator` + Combines the two, routing small allocations to per-CPU slab caches and large allocations to buddy pages. -```mermaid -classDiagram - direction TB - - class GlobalAllocator~PAGE_SIZE~ { - -Mutex~CompositePageAllocator~ buddy - -Option~NonNull~PerCpuSlabSlot~~ slab_slots_ptr - -usize cpu_count - -MetadataRegionInfo metadata_region - -&'static dyn Os os - -UsageStatsAtomic stats - -AtomicBool initialized - +new() GlobalAllocator - +init(region, cpu_count, os) AllocResult - +add_memory_region(start, size) AllocResult - +alloc(Layout) Option~NonNull~u8~~ - +dealloc(ptr, Layout) - +usage_stats() UsageStats - } - - class Os { - <> - +current_cpu_idx() usize - } - - class PerCpuSlabSlot~PAGE_SIZE~ { - -Mutex~SlabByteAllocator~ slab - } - - class CompositePageAllocator~PAGE_SIZE~ { - -BuddyPageAllocator~PAGE_SIZE~ buddy - -CompositeBlockTracker composite_tracker - +new() CompositePageAllocator - +init(start, size) - +alloc_pages(count, align) AllocResult~usize~ - +dealloc_pages(addr, count) - } - - class CompositeBlockTracker { - -blocks: [Option~CompositeBlockInfo~; 64] - -count: usize - } - - class CompositeBlockInfo { - +base_addr: usize - +part_count: u8 - +parts: [(usize, u32); 8] - } - - class BuddyPageAllocator~PAGE_SIZE~ { - -zones: [BuddySet~PAGE_SIZE~; 4] - -num_zones: usize - -global_node_pool: GlobalNodePool - -BuddyStats stats - -Option~&'static dyn AddrTranslator~ addr_translator - +alloc_pages(count, align) AllocResult~usize~ - +dealloc_pages(addr, count) - +add_memory_region(start, size) AllocResult - } - - class AddrTranslator { - <> - +virt_to_phys(va: usize) Option~usize~ - } - - class BuddySet~PAGE_SIZE~ { - -base_addr: usize - -end_addr: usize - -total_pages: usize - -zone_id: usize - -is_lowmem: bool - -free_lists: [PooledLinkedList; 11] - } +The design details are documented in [docs/design.md](docs/design.md). - class GlobalNodePool { - -free_head: Option~usize~ - -total_nodes: usize - -free_nodes: usize - } +## Architecture - class PooledLinkedList { - -head: Option~usize~ - -tail: Option~usize~ - -len: usize - +insert_sorted(addr, order) - +pop_front() Option~BuddyBlock~ - +find_by_addr(addr) Option~BuddyBlock~ - +remove(addr) - } - - class BuddyBlock { - +order: usize - +addr: usize - } - - class SlabByteAllocator~PAGE_SIZE~ { - -caches: [SlabCache; 9] - -total_bytes: usize - -allocated_bytes: usize - +alloc(Layout) SlabAllocDecision - +dealloc(ptr, Layout) SlabDeallocDecision - +provide_slab(size_class, owner_cpu, slab_base, slab_bytes) - } - - class SizeClass { - <> - Bytes8 Bytes16 Bytes32 Bytes64 - Bytes128 Bytes256 Bytes512 - Bytes1024 Bytes2048 - } - - class SlabCache { - -partial: SlabIntrusiveList - -empty: SlabIntrusiveList - -full: SlabIntrusiveList - +alloc_object() Option~usize~ - +provide_slab(addr, size_class, owner_cpu, slab_bytes) - +dealloc_object(ptr) SlabDeallocDecision - } - - class SlabIntrusiveList { - -head: Option~usize~ - -tail: Option~usize~ - -len: usize - } - - class SlabNode { - +addr: usize - +size_class: SizeClass - +alloc_object() Option~usize~ - +dealloc_object(ptr) bool - } - - class SlabHeader { - +magic: u32 - +size_class: u16 - +object_count: u16 - +free_count: u16 - +owner_cpu: u32 - +slab_bytes: usize - +prev: usize - +next: usize - +free_bitmap: [u64; 8] - } - - class AllocError { - <> - InvalidParam - MemoryOverlap - NoMemory - NotAllocated - } - - %% Relationships - - GlobalAllocator --> Os : uses - GlobalAllocator *-- PerCpuSlabSlot : cpu_count slots - GlobalAllocator *-- CompositePageAllocator : Mutex wrapped - GlobalAllocator ..> UsageStats : returns - - CompositePageAllocator *-- BuddyPageAllocator - CompositePageAllocator *-- CompositeBlockTracker - CompositeBlockTracker *-- CompositeBlockInfo : tracks - - BuddyPageAllocator *-- BuddySet : up to 4 zones - BuddyPageAllocator *-- GlobalNodePool : shared - BuddyPageAllocator ..> AddrTranslator : optional - BuddySet *-- PooledLinkedList : 11 free lists - PooledLinkedList ..> GlobalNodePool : allocates nodes from - PooledLinkedList ..> BuddyBlock : manages - - PerCpuSlabSlot *-- SlabByteAllocator : Mutex wrapped - SlabByteAllocator *-- SlabCache : 9 size classes - SlabByteAllocator ..> SizeClass : dispatches by - SlabCache *-- SlabIntrusiveList : 3 lists - SlabIntrusiveList ..> SlabNode : manages - SlabNode *-- SlabHeader : embedded in memory +```mermaid +flowchart TD + GA[GlobalAllocator] --> B[SpinMutex] + GA --> OS[OsImpl] + GA --> PCS[per_cpu_slabs: *mut SpinMutex[]] + + B --> PM[PageMeta[]] + B --> FL[free_lists by order] + + PCS --> SA[SlabAllocator] + SA --> SC[SlabCache x 9] + SC --> P[partial] + SC --> F[full] + SC --> E[empty] + SC --> H[SlabPageHeader] ``` -### Allocation Flow +### Allocation routing ```mermaid flowchart TD - A["GlobalAllocator::alloc(layout)"] --> B{size ≤ 2048B?} - B -- Yes --> C["Route to per-CPU SlabByteAllocator"] - C --> D{Slab has free object?} - D -- Yes --> E["Return object from SlabCache"] - D -- No --> F["Request pages from CompositePageAllocator"] - F --> G["Provide new slab to SlabCache"] + A[GlobalAllocator::alloc(layout)] --> B{size <= 2048 and align <= 2048?} + B -- Yes --> C[Use current CPU slab allocator] + C --> D{Allocated from slab?} + D -- Yes --> E[Return object pointer] + D -- No --> F[Allocate pages from buddy as a new slab] + F --> G[add_slab and retry] G --> E - B -- No --> H["Allocate pages from CompositePageAllocator"] - H --> I{Contiguous pages available?} - I -- Yes --> J["Return contiguous pages"] - I -- No --> K["Combine multiple buddy blocks (composite)"] - K --> J + B -- No --> H[Allocate pages directly from buddy] + H --> I[Return page-backed pointer] ``` -## Features +### Cross-CPU free path -- **Buddy page allocator** — power-of-2 page allocation with splitting and merging -- **Slab byte allocator** — 9 fixed size classes (8B ~ 2048B) for small objects -- **Composite page allocation** — combines non-contiguous buddy blocks for large requests -- **Multi-core support** — per-CPU slab caches with `GlobalAllocator` facade -- **Multi-zone** — up to 4 memory zones with low-memory (DMA32) preference -- **Runtime memory addition** — add memory regions after initialization -- **`no_std`** — suitable for kernel and embedded environments -- **Optional `tracking`** — compile-time statistics gathering -- **Built-in `log`** — structured logging for debugging +```mermaid +sequenceDiagram + participant CPU1 as current CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabAllocator + + CPU1->>H: remote_free(obj_addr) + Note right of H: lock-free CAS push to remote_free_head + CPU1-->>CPU1: return immediately + + CPU0->>S: next local alloc/dealloc under slab lock + S->>H: drain_remote_frees() + H-->>S: slots moved back into local bitmap +``` -## Quick Start +## Features -### Add Dependency +- Buddy page allocation with splitting and merging +- Slab allocation for 9 size classes: `8..=2048` +- Per-CPU slab caches +- Lock-free cross-CPU remote frees +- DMA32 / lowmem page allocation via `alloc_pages_lowmem` +- `no_std` friendly +- Built-in `log` integration +- Standalone `BuddyAllocator` and `SlabAllocator` usage + +## Add dependency ```toml [dependencies] buddy-slab-allocator = "0.2.0" - -# With statistics tracking -buddy-slab-allocator = { version = "0.2.0", features = ["tracking"] } ``` -### Using `GlobalAllocator` - -The `GlobalAllocator` is the recommended interface. It automatically routes small allocations to per-CPU slab caches and large allocations to the buddy page allocator. +## Using `GlobalAllocator` ```rust use buddy_slab_allocator::{GlobalAllocator, OsImpl}; @@ -242,6 +94,7 @@ use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; struct DemoOs; + impl OsImpl for DemoOs { fn current_cpu_idx(&self) -> usize { 0 } fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } @@ -260,96 +113,93 @@ unsafe { let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); -allocator.dealloc(ptr, layout); + +unsafe { + allocator.dealloc(ptr, layout); +} ``` -### Using Buddy and Slab Separately +## Using Buddy and Slab separately -For lower-level control, use the components directly: +For lower-level control, the two building blocks can be used directly. ```rust use buddy_slab_allocator::{ - CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, + BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; -let mut page_alloc = CompositePageAllocator::::new(); -page_alloc.init(0x8000_0000, 16 * 1024 * 1024); +const HEAP_SIZE: usize = 16 * 1024 * 1024; + +let heap_start = 0x8000_0000usize; +let meta_size = BuddyAllocator::::required_meta_size(HEAP_SIZE); +let meta_ptr = 0x9000_0000 as *mut u8; -let mut slab_alloc = SlabByteAllocator::::new(); +let mut buddy = BuddyAllocator::::new(); +unsafe { + buddy.init(meta_ptr, meta_size, heap_start, HEAP_SIZE, None).unwrap(); +} + +let mut slab = SlabAllocator::::new(); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = loop { - match slab_alloc.alloc(layout).unwrap() { - SlabAllocDecision::Allocated(ptr, _) => break ptr, - SlabAllocDecision::NeedsRefill { - size_class, - page_count, - slab_bytes, - } => { - let slab_base = page_alloc.alloc_pages(page_count, slab_bytes).unwrap(); - slab_alloc - .provide_slab(size_class, 0, slab_base, slab_bytes) - .unwrap(); + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => break ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); } } }; -if let SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - .. -} = slab_alloc.dealloc(ptr, layout) -{ - page_alloc.dealloc_pages(slab_base, page_count); +match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + buddy.dealloc_pages(base, pages); + } } ``` -## API Reference - -### Core Types - -- **`GlobalAllocator`** — Top-level facade combining page and slab allocation -- **`CompositePageAllocator`** — Page allocator with composite block support -- **`BuddyPageAllocator`** — Multi-zone buddy page allocator -- **`SlabByteAllocator`** — Slab allocator managing 9 size classes - -### Traits - -- **`Os`** — Provides CPU index for per-CPU slab routing -- **`AddrTranslator`** — Virtual-to-physical address translation for DMA32 - -### Allocation Decisions - -The slab allocator returns decisions rather than directly allocating pages, allowing the caller to manage page allocation: - -- **`SlabAllocDecision::Allocated(ptr, size)`** — object allocated successfully -- **`SlabAllocDecision::NeedsRefill { size_class, page_count, slab_bytes }`** — slab needs new pages - -- **`SlabDeallocDecision::Done`** — object freed within the slab -- **`SlabDeallocDecision::ReleaseSlab { slab_base, page_count, .. }`** — slab became empty, pages can be released - -## Testing & Benchmarking +## Public API summary + +- `GlobalAllocator` + High-level allocator facade that can also implement `GlobalAlloc`. +- `BuddyAllocator` + Standalone page allocator. +- `SlabAllocator` + Standalone slab allocator. +- `SizeClass` + Fixed object size classes used by slab. +- `SlabAllocResult` + `Allocated(ptr)` or `NeedsSlab { size_class, pages }`. +- `SlabDeallocResult` + `Done` or `FreeSlab { base, pages }`. +- `OsImpl` + Provides `current_cpu_idx()` and `virt_to_phys()` for per-CPU routing and lowmem selection. + +## Testing ```bash -# Run tests +# Run normal tests cargo test -cargo test --features tracking + +# Run tests serially +cargo test -- --test-threads=1 + +# Run ignored stress tests +cargo test --test stress_test -- --ignored --nocapture # Check benchmarks compile cargo check --benches -# Run all benchmarks +# Run benchmarks cargo bench - -# Run one benchmark suite -cargo bench --bench buddy_allocator -cargo bench --bench slab_allocator -cargo bench --bench global_allocator ``` -Benchmarks are built with `divan`. Detailed benchmark notes are in `benches/README_CN.md`. +More test notes are in [tests/README.md](tests/README.md). ## License diff --git a/README_CN.md b/README_CN.md index 638f578..e311b04 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,27 +1,91 @@ # buddy-slab-allocator 内存分配器 -一个面向内核和嵌入式环境的 `no_std` Buddy + Slab 分配器。 +这是一个面向内核和嵌入式环境的 `no_std` 两级内存分配器，结合了 buddy 页分配器与 per-CPU slab 小对象分配器。 -## 特性 +## 总览 + +当前实现由三层组成： + +1. `BuddyAllocator` + 管理一段连续虚拟地址区间中的页，支持按 2 的幂分裂与合并。 +2. `SlabAllocator` + 管理 `<= 2048` 字节的小对象，采用固定 size class。 +3. `GlobalAllocator` + 将两者组合起来，对小对象走 per-CPU slab，对大对象走 buddy 页分配。 + +更完整的设计细节见 [docs/design.md](docs/design.md)。 + +## 架构图 -- Buddy 作为共享页分配后端 -- Slab 作为小对象分配前端 -- 多核 `GlobalAllocator` 门面：共享 Buddy + 每 CPU 一个 Slab -- 全局 OS provider，用于获取当前 CPU -- 可选 `tracking` 统计 -- 内置 `log` 支持 +```mermaid +flowchart TD + GA[GlobalAllocator] --> B[SpinMutex] + GA --> OS[OsImpl] + GA --> PCS[per_cpu_slabs: *mut SpinMutex[]] + + B --> PM[PageMeta[]] + B --> FL[按 order 的 free_lists] + + PCS --> SA[SlabAllocator] + SA --> SC[9 个 SlabCache] + SC --> P[partial] + SC --> F[full] + SC --> E[empty] + SC --> H[SlabPageHeader] +``` + +### 分配路由 + +```mermaid +flowchart TD + A[GlobalAllocator::alloc(layout)] --> B{size <= 2048 且 align <= 2048?} + B -- 是 --> C[走当前 CPU 的 slab allocator] + C --> D{slab 直接命中?} + D -- 是 --> E[返回对象指针] + D -- 否 --> F[从 buddy 申请新 slab 页] + F --> G[add_slab 后重试] + G --> E + B -- 否 --> H[直接走 buddy 页分配] + H --> I[返回页级指针] +``` -## 快速开始 +### 跨 CPU 释放 -### 添加依赖 +```mermaid +sequenceDiagram + participant CPU1 as 当前 CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabAllocator + + CPU1->>H: remote_free(obj_addr) + Note right of H: 无锁 CAS 推入 remote_free_head + CPU1-->>CPU1: 立即返回 + + CPU0->>S: 后续本地 alloc/dealloc + S->>H: drain_remote_frees() + H-->>S: 远程释放对象回收到本地 bitmap +``` + +## 特性 + +- Buddy 页分配，支持拆分与合并 +- Slab 小对象分配，固定 9 个 size class：`8..=2048` +- per-CPU slab cache +- 跨 CPU 释放走 lock-free remote free +- 支持 `alloc_pages_lowmem` 低地址页分配 +- 适合 `no_std` +- 内置 `log` 日志支持 +- 可分别独立使用 `BuddyAllocator` 与 `SlabAllocator` + +## 添加依赖 ```toml [dependencies] -buddy-slab-allocator = "0.1.0" -buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } +buddy-slab-allocator = "0.2.0" ``` -### 使用 `GlobalAllocator` +## 使用 `GlobalAllocator` ```rust use buddy_slab_allocator::{GlobalAllocator, OsImpl}; @@ -30,91 +94,113 @@ use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; struct DemoOs; + impl OsImpl for DemoOs { - fn current_cpu_idx(&self) -> usize { - 0 - } - fn virt_to_phys(&self, vaddr: usize) -> usize { - vaddr - } - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } + fn current_cpu_idx(&self) -> usize { 0 } + fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } } static OS: DemoOs = DemoOs; let allocator = GlobalAllocator::::new(); -let region_start = 0x8000_0000; +let region_start = 0x8000_0000 as *mut u8; let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; unsafe { - allocator - .init(region_start, region_size, 1, &OS) - .unwrap(); + allocator.init(region, 1, &OS).unwrap(); } let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); -allocator.dealloc(ptr, layout); + +unsafe { + allocator.dealloc(ptr, layout); +} ``` -### 分别使用 Buddy 与 Slab +## 分别使用 Buddy 与 Slab + +如果需要更底层的控制，也可以分别使用这两个组件。 ```rust use buddy_slab_allocator::{ - CompositePageAllocator, SlabAllocDecision, SlabByteAllocator, SlabDeallocDecision, + BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; -let mut page_alloc = CompositePageAllocator::::new(); -page_alloc.init(0x8000_0000, 16 * 1024 * 1024); +const HEAP_SIZE: usize = 16 * 1024 * 1024; + +let heap_start = 0x8000_0000usize; +let meta_size = BuddyAllocator::::required_meta_size(HEAP_SIZE); +let meta_ptr = 0x9000_0000 as *mut u8; + +let mut buddy = BuddyAllocator::::new(); +unsafe { + buddy.init(meta_ptr, meta_size, heap_start, HEAP_SIZE, None).unwrap(); +} -let mut slab_alloc = SlabByteAllocator::::new(); +let mut slab = SlabAllocator::::new(); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = loop { - match slab_alloc.alloc(layout).unwrap() { - SlabAllocDecision::Allocated(ptr, _) => break ptr, - SlabAllocDecision::NeedsRefill { - size_class, - page_count, - slab_bytes, - } => { - let slab_base = page_alloc.alloc_pages(page_count, slab_bytes).unwrap(); - slab_alloc - .provide_slab(size_class, 0, slab_base, slab_bytes) - .unwrap(); + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => break ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); } } }; -if let SlabDeallocDecision::ReleaseSlab { - slab_base, - page_count, - .. -} = slab_alloc.dealloc(ptr, layout) -{ - page_alloc.dealloc_pages(slab_base, page_count); +match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + buddy.dealloc_pages(base, pages); + } } ``` +## 公开 API 摘要 + +- `GlobalAllocator` + 高层门面，也可以作为 `GlobalAlloc` 使用。 +- `BuddyAllocator` + 独立页分配器。 +- `SlabAllocator` + 独立 slab 分配器。 +- `SizeClass` + slab 使用的固定对象尺寸类。 +- `SlabAllocResult` + `Allocated(ptr)` 或 `NeedsSlab { size_class, pages }`。 +- `SlabDeallocResult` + `Done` 或 `FreeSlab { base, pages }`。 +- `OsImpl` + 提供 `current_cpu_idx()` 和 `virt_to_phys()`，用于 per-CPU 路由与 lowmem 选择。 + ## 测试 ```bash +# 常规测试 cargo test -cargo test --features tracking -``` -## Benchmark +# 串行执行，便于排查问题 +cargo test -- --test-threads=1 -```bash +# 运行忽略的压力测试 +cargo test --test stress_test -- --ignored --nocapture + +# 检查 benchmark 是否可编译 cargo check --benches + +# 运行 benchmark cargo bench -cargo bench --bench buddy_allocator -cargo bench --bench slab_allocator -cargo bench --bench global_allocator ``` -Benchmark 基于 `divan`，说明见 `benches/README_CN.md`。 +更多测试说明见 [tests/README.md](tests/README.md)。 + +## 许可证 + +采用 [Apache-2.0](LICENSE) 许可证。 diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..6191401 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,567 @@ +# buddy-slab-allocator 设计文档 + +本文档面向阅读和维护当前实现的开发者，描述这个分配器在 `buddy + slab` 两级结构下的核心数据结构、初始化过程、`alloc` / `dealloc` 路径，以及多核并发场景下的协作方式。 + +文中描述以当前仓库实现为准，重点对应以下模块： + +- `src/global.rs` +- `src/buddy/mod.rs` +- `src/slab/mod.rs` +- `src/slab/cache.rs` +- `src/slab/page.rs` + +## 1. 设计目标 + +这个分配器服务于 `no_std`、内核或嵌入式环境，目标是同时满足两类需求： + +- 页级分配：按页返回连续内存，支持 power-of-two 分裂和合并。 +- 小对象分配：对 `<= 2048` 字节的对象提供更低碎片、更低分配开销的快速路径。 + +因此整体采用两级结构： + +- 后端：`BuddyAllocator` + 负责管理一段连续虚拟地址区间中的页。 +- 前端：`SlabAllocator` + 负责管理不同 size class 的小对象。 +- 门面：`GlobalAllocator` + 统一对外暴露 `alloc` / `dealloc` / `alloc_pages` / `dealloc_pages` 等接口。 + +## 2. 总体结构 + +```mermaid +flowchart TD + GA[GlobalAllocator] --> B[BuddyAllocator] + GA --> PCS[per-CPU SlabAllocator[]] + GA --> OS[OsImpl] + + PCS --> SC8[SlabCache: 8B] + PCS --> SC64[SlabCache: 64B] + PCS --> SC2K[SlabCache: 2048B] + + SC8 --> SPH[SlabPageHeader] + SC64 --> SPH + SC2K --> SPH + + B --> PM[PageMeta[]] + B --> FL[free_lists by order] +``` + +### 2.1 地址语义 + +当前实现的主语义是“虚拟地址”： + +- `GlobalAllocator::init()` 接收的是一段可写 `&mut [u8]` 区域。 +- `BuddyAllocator` 管理的是一段连续虚拟地址区间。 +- `alloc_pages()`、`alloc()` 返回的也都是虚拟地址。 + +物理地址只在 `alloc_pages_lowmem()` 里参与判断： + +- 候选块的虚拟地址通过 `OsImpl::virt_to_phys()` 翻译成物理地址。 +- 只有物理地址低于 `4 GiB` 的块才会被当作 DMA32 候选。 + +## 3. 初始化与内存布局 + +`GlobalAllocator::init(region, cpu_count, os)` 会把调用者提供的整段区域拆成两部分： + +- 前缀：元数据区 +- 后缀：真正交给 buddy 管理的 heap 区 + +元数据区又细分为： + +- `PageMeta[]` +- `per_cpu_slabs: [SpinMutex; cpu_count]` + +```mermaid +flowchart LR + A[region start] --> B[aligned meta_start] + B --> C[Buddy PageMeta array] + C --> D[padding for alignment] + D --> E[per-CPU SlabAllocator slots] + E --> F[padding to PAGE_SIZE] + F --> G[managed_heap_start] + G --> H[managed heap managed by BuddyAllocator] +``` + +### 3.1 初始化步骤 + +`GlobalAllocator::init()` 的主要步骤如下： + +1. 根据 `region.len()` 和 `cpu_count` 计算最多能管理多少页。 +2. 为这批页预留足够的 `PageMeta[]` 和 per-CPU slab 槽位。 +3. 选择一个页对齐的 `managed_heap_start`。 +4. 调用 `BuddyAllocator::init()` 初始化后端页分配器。 +5. 原地构造每个 CPU 对应的 `SpinMutex`。 +6. 将 `per_cpu_slabs`、`cpu_count`、`os` 写入 `GlobalAllocator`。 + +### 3.2 为什么 metadata 放在 region 前缀 + +这样做有几个好处： + +- 不需要额外的启动期堆分配。 +- `BuddyAllocator` 的 `PageMeta[]` 生命周期天然和被管理区域一致。 +- `per_cpu_slabs` 的内存来源和被管理堆绑在一起，部署简单。 + +代价是： + +- metadata 会消耗一部分可分配空间。 +- `managed_heap_start` 可能晚于 `region` 起点。 + +## 4. BuddyAllocator 设计 + +`BuddyAllocator` 管理页级连续内存，是整个系统的共享后端。 + +### 4.1 核心数据结构 + +```mermaid +classDiagram + class BuddyAllocator { + +meta: *mut PageMeta + +heap_start: usize + +heap_size: usize + +free_lists: [u32; MAX_ORDER+1] + +free_pages: usize + +os: Option<&dyn OsImpl> + } + + class PageMeta { + +flags: PageFlags + +order: u8 + +prev: u32 + +next: u32 + } + + class PageFlags { + <> + Free + Allocated + Slab + } +``` + +说明： + +- `PageMeta` 是每页一项的外部元数据。 +- `free_lists[order]` 保存对应阶的空闲块链表头。 +- 链表节点不是单独分配的，而是复用 `PageMeta.prev/next`。 + +### 4.2 buddy 初始化 + +初始化时，buddy 会从低地址到高地址扫描整段 heap： + +- 每次尽量切出“当前地址自然对齐、且仍然放得下”的最大 order 块。 +- 将该块挂到对应 order 的 free list。 + +这一步的结果是： + +- 所有页都被覆盖。 +- free list 从一开始就处于可分裂、可合并的标准 buddy 形态。 + +### 4.3 页分配流程 + +```mermaid +flowchart TD + A[alloc_pages(count, align)] --> B[计算 order] + B --> C[计算 align_order] + C --> D[effective_order = max(order, align_order)] + D --> E[从 effective_order 向上找非空 free list] + E --> F{找到块?} + F -- 否 --> G[返回 NoMemory] + F -- 是 --> H[弹出大块] + H --> I[逐级 split 到目标 order] + I --> J[标记头页为 Allocated] + J --> K[返回虚拟地址] +``` + +注意点： + +- `count` 会向上取整到 2 的幂。 +- `align` 也会转成对应的 `align_order`。 +- 真正分配的块大小可能大于用户请求。 + +### 4.4 低地址页分配 + +`alloc_pages_lowmem()` 的差异在于： + +- 不只看 free list 是否有块。 +- 还要检查块的物理地址区间是否完全落在 `DMA32_LIMIT` 下方。 + +流程是： + +1. 逐个扫描候选 free list。 +2. 对每个候选块调用 `os.virt_to_phys(addr)`。 +3. 计算 `phys + block_bytes <= 4GiB` 是否成立。 +4. 找到合格块后执行和普通分配相同的拆分逻辑。 + +### 4.5 页释放流程 + +`dealloc_pages(addr, count)` 的关键点是： + +- `addr` 必须是原始返回地址。 +- 实际释放的 order 以 `PageMeta.order` 为准。 +- `count` 主要用于 debug 断言，确保调用者没有传得比真实块更大。 + +```mermaid +flowchart TD + A[dealloc_pages(addr, count)] --> B[根据 addr 算出 pfn] + B --> C[读取 PageMeta.order] + C --> D[与 buddy pfn 检查是否可合并] + D --> E{buddy 同 order 且空闲?} + E -- 是 --> F[从 free list 移除 buddy] + F --> G[合并并提升 order] + G --> D + E -- 否 --> H[将最终块挂回 free list] +``` + +## 5. SlabAllocator 设计 + +`SlabAllocator` 不直接申请页，它只管理“已经拿到的 slab page”。 + +### 5.1 size class + +当前固定 9 个 size class： + +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +- 1024 +- 2048 + +布局规则： + +- `layout.size()` 与 `layout.align()` 取较大值。 +- 选择能容纳它的最小 size class。 +- 超过 `2048` 字节则不走 slab。 + +### 5.2 SlabCache 的三条链 + +每个 size class 对应一个 `SlabCache`，内部维护三条 intrusive list： + +- `partial` + 还有空位的 slab，优先分配。 +- `full` + 本地 bitmap 没空位，但可能积累了 remote free。 +- `empty` + 所有对象都空闲的 slab。 + +实现上还有限制： + +- 每个 `SlabCache` 最多缓存 1 个空 slab。 +- 如果又出现新的空 slab，多出来的那个会还给 buddy。 + +这也是压力测试里“页数不一定精确回到初始值”的原因之一：每个 CPU、每个 size class 允许保留一个空 slab 作为缓存。 + +### 5.3 SlabPageHeader + +每个 slab page 起始位置都嵌着一个 `SlabPageHeader`。 + +它包含： + +- `magic` +- `size_class` +- `object_count` +- `local_free_count` +- `owner_cpu` +- `slab_bytes` +- `list_prev/list_next` +- `local_bitmap` +- `remote_free_head` +- `remote_free_count` + +### 5.4 本地分配 + +owner CPU 在持有对应 slab 锁时执行本地分配： + +1. 先看 `partial` 头 slab。 +2. 如果它有 remote free，先 drain 回本地 bitmap。 +3. 从 bitmap 找一个空位。 +4. 如果分完后满了，把 slab 从 `partial` 挪到 `full`。 + +如果 `partial` 不行，则： + +1. 扫 `full`，找是否有 slab 因 remote free 重新出现空位。 +2. 如果有，把它移回 `partial`。 +3. 还不行就尝试复用 `empty` 中缓存的 slab。 +4. 再不行就返回 `NeedsSlab`，要求上层先提供新页。 + +## 6. GlobalAllocator 的分配路径 + +`GlobalAllocator` 是真正对外使用的入口。 + +它把请求分成两类： + +- 小对象：走 slab +- 大对象：走 buddy + +判断条件： + +- `layout.size() <= 2048` +- `layout.align() <= 2048` + +满足才走 slab。 + +### 6.1 alloc 总流程 + +```mermaid +flowchart TD + A[alloc(layout)] --> B{已初始化?} + B -- 否 --> X[NotInitialized] + B -- 是 --> C{slab eligible?} + C -- 是 --> D[slab_alloc] + C -- 否 --> E[large_alloc] +``` + +### 6.2 小对象分配 + +```mermaid +sequenceDiagram + participant Caller + participant GA as GlobalAllocator + participant S as per-CPU SlabAllocator + participant B as BuddyAllocator + + Caller->>GA: alloc(layout) + GA->>GA: current_cpu_idx() + GA->>S: lock + alloc(layout) + alt Slab 命中 + S-->>GA: Allocated(ptr) + GA-->>Caller: ptr + else 需要新 slab + S-->>GA: NeedsSlab(size_class, pages) + GA->>S: drop lock + GA->>B: alloc_pages(pages, slab_bytes) + GA->>B: set_page_flags(addr, Slab) + GA->>S: lock + add_slab(...) + GA->>S: alloc(layout) + S-->>GA: Allocated(ptr) + GA-->>Caller: ptr + end +``` + +这里有一个很重要的锁顺序细节： + +- 当 slab 缺页时，`GlobalAllocator` 会先释放 slab 锁，再去拿 buddy 锁。 + +这样可以避免： + +- 持有 slab 锁时阻塞在 buddy 上。 +- 未来演化成更复杂锁图时出现循环等待。 + +### 6.3 大对象分配 + +大对象路径很直接： + +1. 根据 `layout.size()` 计算页数。 +2. 对齐至少是 `PAGE_SIZE`。 +3. 调用 `BuddyAllocator::alloc_pages()`。 + +大对象不会进入 slab，也不会设置 `PageFlags::Slab`。 + +## 7. dealloc 路径 + +### 7.1 大对象释放 + +大对象释放和分配对称： + +1. 根据 `layout.size()` 推导页数。 +2. 直接进入 `BuddyAllocator::dealloc_pages()`。 + +### 7.2 小对象释放：本地路径 + +如果当前 CPU 就是该 slab 的 owner： + +1. 通过对象地址反推出 slab base。 +2. 拿 owner CPU 对应的 slab 锁。 +3. 调用 `slab.dealloc(ptr, layout)`。 +4. 如果结果是 `Done`，结束。 +5. 如果结果是 `FreeSlab { base, pages }`，释放 slab 锁后把整页还给 buddy。 + +为什么空 slab 不总是立刻还给 buddy： + +- `SlabCache` 会保留一个 empty slab 作为热缓存。 +- 只有当已有一个 cached empty slab 时，新的空 slab 才会还给 buddy。 + +### 7.3 小对象释放：远程路径 + +如果当前 CPU 不是 owner CPU，不会直接拿对方的 slab 锁。 + +而是走 lock-free remote free： + +```mermaid +sequenceDiagram + participant CPU1 as current CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabCache + + CPU1->>H: remote_free(obj_addr) + Note right of H: CAS 推入 remote_free_head\n并增加 remote_free_count + CPU1-->>CPU1: 返回，不加锁 + + CPU0->>S: 下次 alloc/dealloc 时持锁进入 + S->>H: drain_remote_frees() + Note right of H: 将 remote 栈中的对象\n重新记回 local bitmap + S-->>CPU0: 对象重新可分配 +``` + +这条路径的核心特点： + +- 释放方不需要知道 owner CPU 的锁状态。 +- 释放方不抢远端锁。 +- owner CPU 在后续本地操作时，顺手把 remote free 合并回 bitmap。 + +这也是当前实现中“跨 CPU free 无锁，但重新利用对象仍由 owner CPU 控制”的关键设计。 + +## 8. 多线程与并发模型 + +### 8.1 并发边界 + +当前实现中真正共享的状态主要有两类： + +- `BuddyAllocator` + 整体包在一个 `SpinMutex` 里，页级操作串行化。 +- 每 CPU 一个 `SlabAllocator` + 每个都各自包在一个 `SpinMutex` 里。 + +因此并发模型可以概括为： + +- 小对象本地路径：只竞争本 CPU 的 slab 锁。 +- 小对象远程 free：不走锁，直接原子 CAS。 +- 缺页或空 slab 回收：会短暂进入全局 buddy 锁。 + +### 8.2 为什么是 per-CPU slab + +如果所有小对象都走单个全局 slab 锁，会有两个问题： + +- 多核下锁竞争重。 +- 小对象热点路径和页级路径容易互相干扰。 + +per-CPU slab 的好处是： + +- 本地分配/释放命中时只碰本 CPU 锁。 +- 跨 CPU free 通过 remote list 延后处理，不把释放方拖进远端临界区。 + +### 8.3 远程释放的正确性依赖 + +远程释放正确性依赖几个事实： + +- slab 页头里记录了 `owner_cpu`。 +- 远程释放只把对象地址压到无锁链表里。 +- owner CPU 持锁 drain 时，再把对象重新放回本地 bitmap。 +- remote 栈中的 next 指针写在对象自身内存开头，这要求被释放对象已经不再被用户使用。 + +### 8.4 锁顺序 + +当前实现中建议理解为以下顺序： + +- 常规 slab 本地路径：只拿 slab 锁。 +- 常规 buddy 页路径：只拿 buddy 锁。 +- slab 缺页：先释放 slab 锁，再拿 buddy 锁，再重新拿 slab 锁。 +- slab 回收空页：先释放 slab 锁，再拿 buddy 锁。 + +这样可以避免出现“持有 slab 锁后等待 buddy，同时另一路持有 buddy 锁又等待 slab”的循环。 + +## 9. 关键实现细节 + +### 9.1 `PageFlags` + +页状态分三种： + +- `Free` +- `Allocated` +- `Slab` + +含义是： + +- `Free`：在 buddy free list 中。 +- `Allocated`：普通页分配返回的块头页。 +- `Slab`：该块当前作为 slab 页使用。 + +### 9.2 buddy 只在块头记录 order + +`PageMeta.order` 只有块头页有意义。 + +因此： + +- 分配时只标记头页的 `order`。 +- 释放时调用方必须传回原始返回地址。 + +### 9.3 `dealloc_pages(count)` 为什么不完全信任 `count` + +因为分配时请求可能被扩大： + +- `count` 会按 buddy order 向上取整。 +- `align` 也可能把块提升到更高阶。 + +所以释放时真正信任的是 `PageMeta.order`，`count` 只用于做 debug 级别的合理性检查。 + +### 9.4 empty slab 缓存策略 + +每个 `SlabCache` 最多保留一个 empty slab。 + +这是一个很重要的折中： + +- 优点：后续同 size class 再分配时更快，减少 buddy 往返。 +- 代价：空闲页不会总是立刻全部回收到 buddy。 + +所以测试或观测时不能简单假设： + +- “所有对象释放后，buddy free pages 一定精确回到初始值”。 + +更准确的理解应是： + +- 除去每 CPU、每 size class 允许缓存的 empty slab 后，应当没有额外页泄漏。 + +## 10. realloc 行为 + +`GlobalAllocator` 通过 `GlobalAlloc` 实现了 `realloc`： + +1. 构造新的 `Layout` +2. 新分配一块内存 +3. 拷贝 `min(old_size, new_size)` 字节 +4. 释放旧块 + +它不是原地扩容，也没有针对 slab/buddy 做特殊优化。 + +## 11. 当前实现的优点与限制 + +### 11.1 优点 + +- 结构清晰，页级与对象级职责分离。 +- 小对象本地路径开销低。 +- 跨 CPU free 不抢远端锁。 +- 初始化不依赖额外堆分配。 +- 支持低地址页分配。 + +### 11.2 限制 + +- buddy 后端仍然是全局单锁。 +- `remote_free` 的对象复用必须等 owner CPU 后续 drain。 +- empty slab 缓存会牺牲一部分“空闲页立刻回收”的极致性。 +- `realloc` 为通用 copy 语义，不做原地优化。 + +## 12. 阅读源码建议 + +如果想快速建立整体理解，建议按下面顺序读： + +1. `src/lib.rs` + 先看公开模块和 `OsImpl`。 +2. `src/global.rs` + 先理解门面层如何在 buddy/slab 间路由。 +3. `src/buddy/mod.rs` + 看页级分配、拆分和合并。 +4. `src/slab/mod.rs` + 看 slab 与上层的接口契约。 +5. `src/slab/cache.rs` + 看 `partial/full/empty` 三链切换。 +6. `src/slab/page.rs` + 看 bitmap、remote free 栈与 owner CPU drain。 + +## 13. 一句话总结 + +这个分配器的核心思想可以概括成一句话： + +> 用 buddy 统一管理页，用 per-CPU slab 加速小对象，把跨 CPU free 变成“无锁登记、延迟回收”，从而在实现复杂度、局部性和并发开销之间取得平衡。 diff --git a/src/global.rs b/src/global.rs index 6961cec..47a8f29 100644 --- a/src/global.rs +++ b/src/global.rs @@ -345,7 +345,6 @@ impl GlobalAllocator { SlabDeallocResult::Done => {} SlabDeallocResult::FreeSlab { base, pages } => { drop(slab); - self.buddy.lock().set_page_flags(base, PageFlags::Free); self.buddy.lock().dealloc_pages(base, pages); } } diff --git a/tests/README.md b/tests/README.md index dae056b..227f624 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,8 +9,10 @@ - `dma32_pages_test.rs` 低地址页分配相关测试。 - `stress_test.rs` - 长时间随机、耗尽恢复、碎片化恢复等压力测试。 + 长时间随机、耗尽恢复、碎片化恢复，以及真实多线程跨 CPU 压力测试。 这些测试默认使用 `#[ignore]`，不会进入常规 `cargo test` 路径。 +- `common/` + 共享测试辅助模块，提供宿主堆管理、线程本地 CPU mock、固定种子 RNG 和通用初始化逻辑。 单元测试仍位于 `src/**/*.rs` 的 `#[cfg(test)]` 模块中，文档测试位于公共 API 注释中。 @@ -20,27 +22,27 @@ # 常规测试 cargo test -# 带 tracking 的常规测试 -cargo test --features tracking - # 串行执行，便于排查测试交互 cargo test -- --test-threads=1 +# 仅运行常规集成测试 +cargo test --test integration_test + # 仅运行压力测试 cargo test --test stress_test -- --ignored --nocapture -# 带 tracking 运行压力测试 -cargo test --test stress_test --features tracking -- --ignored --nocapture +# 串行运行压力测试，便于定位多线程问题 +cargo test --test stress_test -- --ignored --nocapture --test-threads=1 ``` ## 设计原则 - 常规测试应保持快速、稳定，可直接进入 CI。 -- 压力测试用于长时间 workload、耗尽恢复、碎片化恢复与统计不变量检查。 +- 压力测试用于长时间 workload、真实多线程 cross-CPU 交互、耗尽恢复、碎片化恢复与统计不变量检查。 - 性能测量移至 `benches/`，不在测试中混入 benchmark 逻辑。 ## 注意事项 1. 所有测试都使用宿主分配器申请一块测试堆。 2. 压力测试默认被忽略，需要显式运行。 -3. `tracking` 相关断言仅在启用 `tracking` feature 时编译。 +3. 多线程压力测试默认被 `#[ignore]` 标记，需要显式运行。 diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..6e62156 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,120 @@ +#![allow(dead_code)] + +use buddy_slab_allocator::{GlobalAllocator, OsImpl}; +use core::ptr::NonNull; +use rand::{rngs::StdRng, SeedableRng}; +use std::alloc::{alloc, dealloc, Layout}; +use std::cell::Cell; + +thread_local! { + static CURRENT_CPU: Cell = const { Cell::new(0) }; +} + +fn identity_map(vaddr: usize) -> usize { + vaddr +} + +fn lowmem_map(vaddr: usize) -> usize { + vaddr & 0x0FFF_FFFF +} + +pub struct ThreadAwareOs { + mapper: fn(usize) -> usize, +} + +impl ThreadAwareOs { + pub const fn new(mapper: fn(usize) -> usize) -> Self { + Self { mapper } + } +} + +impl OsImpl for ThreadAwareOs { + fn current_cpu_idx(&self) -> usize { + CURRENT_CPU.with(|cpu| cpu.get()) + } + + fn virt_to_phys(&self, vaddr: usize) -> usize { + (self.mapper)(vaddr) + } +} + +pub static TEST_OS: ThreadAwareOs = ThreadAwareOs::new(identity_map); +pub static LOWMEM_OS: ThreadAwareOs = ThreadAwareOs::new(lowmem_map); + +pub fn set_current_cpu(cpu: usize) { + CURRENT_CPU.with(|slot| slot.set(cpu)); +} + +pub fn seeded_rng(seed: u64) -> StdRng { + StdRng::seed_from_u64(seed) +} + +pub struct HostRegion { + ptr: *mut u8, + layout: Layout, +} + +impl HostRegion { + pub fn new(size: usize, align: usize) -> Self { + let layout = Layout::from_size_align(size, align).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "host alloc failed"); + Self { ptr, layout } + } + + pub fn addr(&self) -> usize { + self.ptr as usize + } + + pub fn len(&self) -> usize { + self.layout.size() + } + + pub fn as_mut_ptr(&mut self) -> *mut u8 { + self.ptr + } + + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } + } + + pub unsafe fn subslice(&mut self, offset: usize, len: usize) -> &mut [u8] { + std::slice::from_raw_parts_mut(self.ptr.add(offset), len) + } +} + +impl Drop for HostRegion { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +pub fn init_global( + allocator: &GlobalAllocator, + region: &mut HostRegion, + cpu_count: usize, + os: &'static dyn OsImpl, +) { + set_current_cpu(0); + unsafe { + allocator + .init(region.as_mut_slice(), cpu_count, os) + .unwrap() + }; +} + +pub fn count_free_pages(allocator: &GlobalAllocator) -> usize { + let mut addrs = Vec::new(); + while let Ok(addr) = allocator.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); + } + let count = addrs.len(); + for addr in addrs { + allocator.dealloc_pages(addr, 1); + } + count +} + +pub fn nonnull_from_addr(addr: usize) -> NonNull { + unsafe { NonNull::new_unchecked(addr as *mut u8) } +} diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index 77714aa..d9319aa 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -2,48 +2,23 @@ extern crate buddy_slab_allocator; -use buddy_slab_allocator::{GlobalAllocator, OsImpl}; -use core::alloc::Layout; -use std::alloc::{alloc, dealloc}; +mod common; + +use buddy_slab_allocator::GlobalAllocator; +use common::{init_global, HostRegion, LOWMEM_OS}; const PAGE_SIZE: usize = 0x1000; const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; -struct MockOs; - -impl OsImpl for MockOs { - fn current_cpu_idx(&self) -> usize { - 0 - } - fn virt_to_phys(&self, vaddr: usize) -> usize { - // Map virtual addresses below 4 GiB so lowmem checks pass. - vaddr & 0x0FFF_FFFF - } -} - -static MOCK_OS: MockOs = MockOs; - -fn host_alloc(size: usize, align: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null()); - (ptr, layout) -} - -fn host_dealloc(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; -} - -fn init_allocator(allocator: &GlobalAllocator, region: &mut [u8]) { - unsafe { allocator.init(region, 1, &MOCK_OS).unwrap() }; +fn init_allocator(allocator: &GlobalAllocator, region: &mut HostRegion) { + init_global(allocator, region, 1, &LOWMEM_OS); } #[test] fn test_lowmem_basic() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_allocator(&allocator, region); + init_allocator(&allocator, &mut region); let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -57,30 +32,24 @@ fn test_lowmem_basic() { allocator.dealloc_pages(addr1, 1); allocator.dealloc_pages(addr2, 4); - - host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_aligned() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 2); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 2); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_allocator(&allocator, region); + init_allocator(&allocator, &mut region); let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); assert_eq!((addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), 0); allocator.dealloc_pages(addr, 1); - - host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_vs_normal() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_allocator(&allocator, region); + init_allocator(&allocator, &mut region); let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); @@ -90,16 +59,13 @@ fn test_lowmem_vs_normal() { allocator.dealloc_pages(addr_low, 1); allocator.dealloc_pages(addr_normal, 1); - - host_dealloc(region_ptr, region_layout); } #[test] fn test_lowmem_stress() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_allocator(&allocator, region); + init_allocator(&allocator, &mut region); let mut addrs = Vec::new(); for _ in 0..32 { @@ -109,6 +75,4 @@ fn test_lowmem_stress() { for addr in addrs { allocator.dealloc_pages(addr, 1); } - - host_dealloc(region_ptr, region_layout); } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 62f504c..656ae68 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,62 +1,24 @@ //! Integration tests for the buddy-slab-allocator crate. extern crate buddy_slab_allocator; +mod common; use buddy_slab_allocator::{ - slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, OsImpl, SizeClass, - SlabAllocResult, SlabAllocator, SlabDeallocResult, + slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, SizeClass, SlabAllocResult, + SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; -use core::sync::atomic::{AtomicUsize, Ordering}; -use std::alloc::{alloc, dealloc}; +use core::ptr::NonNull; +use std::collections::BTreeSet; + +use common::{ + count_free_pages, init_global as init_global_allocator, set_current_cpu, HostRegion, LOWMEM_OS, + TEST_OS, +}; const PAGE_SIZE: usize = 0x1000; const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16 MiB -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -struct TestOs { - cpu: AtomicUsize, -} - -impl TestOs { - const fn new() -> Self { - Self { - cpu: AtomicUsize::new(0), - } - } - - #[allow(dead_code)] - fn set_cpu(&self, cpu: usize) { - self.cpu.store(cpu, Ordering::Relaxed); - } -} - -impl OsImpl for TestOs { - fn current_cpu_idx(&self) -> usize { - self.cpu.load(Ordering::Relaxed) - } - fn virt_to_phys(&self, vaddr: usize) -> usize { - vaddr // identity mapping - } -} - -static TEST_OS: TestOs = TestOs::new(); - -/// Allocate a region from the host allocator. -fn host_alloc(size: usize, align: usize) -> (*mut u8, Layout) { - let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "host alloc failed"); - (ptr, layout) -} - -fn host_dealloc(ptr: *mut u8, layout: Layout) { - unsafe { dealloc(ptr, layout) }; -} - fn align_up(value: usize, align: usize) -> usize { (value + align - 1) & !(align - 1) } @@ -67,15 +29,21 @@ fn align_up(value: usize, align: usize) -> usize { #[test] fn buddy_basic_alloc_dealloc() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .init( + meta.as_mut_ptr(), + meta.len(), + heap_addr, + TEST_HEAP_SIZE, + None, + ) .unwrap(); } @@ -90,24 +58,27 @@ fn buddy_basic_alloc_dealloc() { buddy.dealloc_pages(addr1, 1); buddy.dealloc_pages(addr4, 4); assert!(buddy.free_pages() > free_before); - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn buddy_alignment() { // Heap must be aligned to the highest alignment we test (PAGE_SIZE * 4) // so that PFN-based alignment maps to absolute address alignment. - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .init( + meta.as_mut_ptr(), + meta.len(), + heap_addr, + TEST_HEAP_SIZE, + None, + ) .unwrap(); } @@ -119,23 +90,20 @@ fn buddy_alignment() { buddy.dealloc_pages(addr2, 1); buddy.dealloc_pages(addr4, 1); - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn buddy_aligned_alloc_dealloc_uses_recorded_order() { let heap_size = 64 * PAGE_SIZE; - let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE * 16); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(heap_size, PAGE_SIZE * 16); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) .unwrap(); } @@ -143,23 +111,20 @@ fn buddy_aligned_alloc_dealloc_uses_recorded_order() { let addr = buddy.alloc_pages(4, PAGE_SIZE * 16).unwrap(); buddy.dealloc_pages(addr, 4); assert_eq!(buddy.free_pages(), free_before); - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn buddy_exhaust_and_recover() { let heap_size = 64 * PAGE_SIZE; // Small heap - let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(heap_size, PAGE_SIZE); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) .unwrap(); } @@ -186,23 +151,20 @@ fn buddy_exhaust_and_recover() { for a in addrs { buddy.dealloc_pages(a, 1); } - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn buddy_merge_coalescing() { let heap_size = 16 * PAGE_SIZE; - let (heap_ptr, heap_layout) = host_alloc(heap_size, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(heap_size, PAGE_SIZE); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, heap_size, None) + .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) .unwrap(); } @@ -216,9 +178,79 @@ fn buddy_merge_coalescing() { // After freeing both, free_pages should return to initial assert_eq!(buddy.free_pages(), initial_free); +} + +#[test] +fn buddy_fragmentation_blocks_high_order_then_recovers() { + let heap_size = 32 * PAGE_SIZE; + let heap = HostRegion::new(heap_size, PAGE_SIZE); + let mut meta = HostRegion::new( + BuddyAllocator::::required_meta_size(heap_size), + 16, + ); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta.as_mut_ptr(), meta.len(), heap.addr(), heap_size, None) + .unwrap(); + } + + let mut addrs = Vec::new(); + while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); + } + assert_eq!(addrs.len(), heap_size / PAGE_SIZE); + + for &addr in addrs.iter().step_by(2) { + buddy.dealloc_pages(addr, 1); + } + assert!(buddy.alloc_pages(2, PAGE_SIZE).is_err()); + + for &addr in addrs.iter().skip(1).step_by(2) { + buddy.dealloc_pages(addr, 1); + } + + let addr = buddy.alloc_pages(8, PAGE_SIZE).unwrap(); + buddy.dealloc_pages(addr, 8); + assert_eq!(buddy.free_pages(), heap_size / PAGE_SIZE); +} + +#[test] +fn buddy_high_order_full_cycle_restores_free_pages() { + let heap_size = 256 * PAGE_SIZE; + let heap = HostRegion::new(heap_size, PAGE_SIZE * 16); + let mut meta = HostRegion::new( + BuddyAllocator::::required_meta_size(heap_size), + 16, + ); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init(meta.as_mut_ptr(), meta.len(), heap.addr(), heap_size, None) + .unwrap(); + } + + let initial_free = buddy.free_pages(); + let requests = [ + (1usize, PAGE_SIZE), + (2, 2 * PAGE_SIZE), + (3, 4 * PAGE_SIZE), + (8, 8 * PAGE_SIZE), + (5, PAGE_SIZE), + (16, 16 * PAGE_SIZE), + ]; + let mut allocations = Vec::new(); + + for (count, align) in requests { + let addr = buddy.alloc_pages(count, align).unwrap(); + allocations.push((addr, count)); + } + assert!(buddy.free_pages() < initial_free); - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); + for (addr, count) in allocations.into_iter().rev() { + buddy.dealloc_pages(addr, count); + } + assert_eq!(buddy.free_pages(), initial_free); } // ====================================================================== @@ -227,15 +259,21 @@ fn buddy_merge_coalescing() { #[test] fn slab_basic() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .init( + meta.as_mut_ptr(), + meta.len(), + heap_addr, + TEST_HEAP_SIZE, + None, + ) .unwrap(); } @@ -262,22 +300,25 @@ fn slab_basic() { SlabDeallocResult::Done => {} SlabDeallocResult::FreeSlab { .. } => {} // also valid } - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn slab_many_objects() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .init( + meta.as_mut_ptr(), + meta.len(), + heap_addr, + TEST_HEAP_SIZE, + None, + ) .unwrap(); } @@ -305,22 +346,25 @@ fn slab_many_objects() { for ptr in ptrs { let _ = slab.dealloc(ptr, layout); } - - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); } #[test] fn slab_all_size_classes() { - let (heap_ptr, heap_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap_ptr as usize; + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let heap_addr = heap.addr(); let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let (meta_ptr, meta_layout) = host_alloc(meta_size, 16); + let mut meta = HostRegion::new(meta_size, 16); let mut buddy = BuddyAllocator::::new(); unsafe { buddy - .init(meta_ptr, meta_size, heap_addr, TEST_HEAP_SIZE, None) + .init( + meta.as_mut_ptr(), + meta.len(), + heap_addr, + TEST_HEAP_SIZE, + None, + ) .unwrap(); } @@ -348,26 +392,108 @@ fn slab_all_size_classes() { for (ptr, layout) in allocations { let _ = slab.dealloc(ptr, layout); } +} - host_dealloc(meta_ptr, meta_layout); - host_dealloc(heap_ptr, heap_layout); +#[test] +fn slab_reuses_freed_objects_same_size_class() { + let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut meta = HostRegion::new( + BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE), + 16, + ); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy + .init( + meta.as_mut_ptr(), + meta.len(), + heap.addr(), + TEST_HEAP_SIZE, + None, + ) + .unwrap(); + } + + let mut slab = SlabAllocator::::new(); + let layout = Layout::from_size_align(64, 8).unwrap(); + let (size_class, pages) = match slab.alloc(layout).unwrap() { + SlabAllocResult::NeedsSlab { size_class, pages } => (size_class, pages), + SlabAllocResult::Allocated(_) => panic!("should need slab first"), + }; + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + + let first = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected allocation from fresh slab"), + }; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected same slab to satisfy alloc"), + }; + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + ptrs.push(ptr); + } + assert!(matches!( + slab.alloc(layout).unwrap(), + SlabAllocResult::NeedsSlab { .. } + )); + + let freed_ptrs: Vec<_> = ptrs.iter().copied().step_by(2).collect(); + let freed_addrs: BTreeSet<_> = freed_ptrs.iter().map(|ptr| ptr.as_ptr() as usize).collect(); + for &ptr in &freed_ptrs { + assert!(matches!(slab.dealloc(ptr, layout), SlabDeallocResult::Done)); + } + + let mut reused_addrs = BTreeSet::new(); + for _ in 0..freed_addrs.len() { + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected reuse from freed slots"), + }; + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + reused_addrs.insert(ptr.as_ptr() as usize); + } + assert_eq!(reused_addrs, freed_addrs); + + for ptr in ptrs { + let addr = ptr.as_ptr() as usize; + if !freed_addrs.contains(&addr) { + let _ = slab.dealloc(ptr, layout); + } + } + for addr in reused_addrs { + let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; + let _ = slab.dealloc(ptr, layout); + } } // ====================================================================== // Global allocator tests // ====================================================================== -fn init_global(allocator: &GlobalAllocator, region: &mut [u8], cpu_count: usize) { - unsafe { allocator.init(region, cpu_count, &TEST_OS).unwrap() }; +fn init_global(allocator: &GlobalAllocator, region: &mut HostRegion, cpu_count: usize) { + init_global_allocator(allocator, region, cpu_count, &TEST_OS); } #[test] fn global_page_alloc() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let region_addr = region_ptr as usize; + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let region_addr = region.addr(); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 1); + init_global(&allocator, &mut region, 1); let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -377,44 +503,35 @@ fn global_page_alloc() { assert!(addr >= managed_start && addr < managed_end); assert_eq!(addr % PAGE_SIZE, 0); allocator.dealloc_pages(addr, 4); - - host_dealloc(region_ptr, region_layout); } #[test] fn global_small_alloc() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 1); + init_global(&allocator, &mut region, 1); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - - host_dealloc(region_ptr, region_layout); } #[test] fn global_large_alloc() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 1); + init_global(&allocator, &mut region, 1); let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - - host_dealloc(region_ptr, region_layout); } #[test] fn global_mixed_alloc() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 1); + init_global(&allocator, &mut region, 1); let sizes: &[(usize, usize)] = &[ (8, 8), @@ -432,19 +549,16 @@ fn global_mixed_alloc() { for (ptr, layout) in allocations { unsafe { allocator.dealloc(ptr, layout) }; } - - host_dealloc(region_ptr, region_layout); } #[test] fn global_cross_cpu_free() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 2); + init_global(&allocator, &mut region, 2); // Allocate on CPU 0 - TEST_OS.set_cpu(0); + set_current_cpu(0); let layout = Layout::from_size_align(64, 8).unwrap(); let mut ptrs = Vec::new(); for _ in 0..10 { @@ -452,27 +566,24 @@ fn global_cross_cpu_free() { } // Free from CPU 1 (triggers remote free path) - TEST_OS.set_cpu(1); + set_current_cpu(1); for ptr in ptrs { unsafe { allocator.dealloc(ptr, layout) }; } // Allocate on CPU 0 again — should drain remote frees and succeed - TEST_OS.set_cpu(0); + set_current_cpu(0); let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout) }; - - host_dealloc(region_ptr, region_layout); } #[test] fn global_cross_cpu_free_drains_remote_queue() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 2); + init_global(&allocator, &mut region, 2); - TEST_OS.set_cpu(0); + set_current_cpu(0); let layout = Layout::from_size_align(64, 8).unwrap(); let ptr = allocator.alloc(layout).unwrap(); @@ -483,32 +594,49 @@ fn global_cross_cpu_free_drains_remote_queue() { let base = SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); let hdr = unsafe { &*(base as *const SlabPageHeader) }; assert_eq!(hdr.owner_cpu, 0); - assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); - TEST_OS.set_cpu(1); + set_current_cpu(1); unsafe { allocator.dealloc(ptr, layout) }; - assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 1); - assert_ne!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 1 + ); + assert_ne!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); - TEST_OS.set_cpu(0); + set_current_cpu(0); let ptr2 = allocator.alloc(layout).unwrap(); - assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); - assert_eq!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + assert_eq!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); unsafe { allocator.dealloc(ptr2, layout) }; - - host_dealloc(region_ptr, region_layout); } #[test] fn global_cross_cpu_free_multiple_rounds_same_slab() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - init_global(&allocator, region, 2); + init_global(&allocator, &mut region, 2); let layout = Layout::from_size_align(64, 8).unwrap(); - TEST_OS.set_cpu(0); + set_current_cpu(0); let first = allocator.alloc(layout).unwrap(); let slab_bytes = SizeClass::from_layout(layout) .unwrap() @@ -527,66 +655,155 @@ fn global_cross_cpu_free_multiple_rounds_same_slab() { ptrs.push(ptr); } - TEST_OS.set_cpu(1); + set_current_cpu(1); for &ptr in &ptrs { unsafe { allocator.dealloc(ptr, layout) }; } assert_eq!( - hdr.remote_free_count.load(Ordering::Relaxed) as usize, + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed) as usize, object_count ); - TEST_OS.set_cpu(0); + set_current_cpu(0); let mut drained = Vec::with_capacity(object_count); for _ in 0..object_count { drained.push(allocator.alloc(layout).unwrap()); } - assert_eq!(hdr.remote_free_count.load(Ordering::Relaxed), 0); - assert_eq!(hdr.remote_free_head.load(Ordering::Relaxed), 0); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + assert_eq!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); for ptr in drained { unsafe { allocator.dealloc(ptr, layout) }; } +} + +#[test] +fn global_small_object_churn_then_large_alloc() { + const REGION_SIZE: usize = 2 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, 1); - host_dealloc(region_ptr, region_layout); + let small_layout = Layout::from_size_align(2048, 8).unwrap(); + let warmup = allocator.alloc(small_layout).unwrap(); + unsafe { allocator.dealloc(warmup, small_layout) }; + let baseline = count_free_pages(&allocator); + let mut ptrs = Vec::new(); + while let Ok(ptr) = allocator.alloc(small_layout) { + ptrs.push(ptr); + } + assert!(!ptrs.is_empty()); + + for ptr in ptrs { + unsafe { allocator.dealloc(ptr, small_layout) }; + } + + let large_layout = Layout::from_size_align(16 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(large_layout).unwrap(); + unsafe { allocator.dealloc(ptr, large_layout) }; + assert_eq!(count_free_pages(&allocator), baseline); } #[test] -fn global_lowmem_pages() { - // Use a custom OsImpl that maps virt addresses below 4 GiB so lowmem - // allocation can succeed on 64-bit hosts where heap addresses are > 4 GiB. - struct LowmemOs; - impl OsImpl for LowmemOs { - fn current_cpu_idx(&self) -> usize { - 0 - } - fn virt_to_phys(&self, vaddr: usize) -> usize { - vaddr & 0x0FFF_FFFF - } +fn global_cross_cpu_free_all_objects_recovers_backend_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, 2); + + let layout = Layout::from_size_align(64, 8).unwrap(); + set_current_cpu(0); + let warmup = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(warmup, layout) }; + let baseline = count_free_pages(&allocator); + + let first = allocator.alloc(layout).unwrap(); + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + ptrs.push(allocator.alloc(layout).unwrap()); + } + + set_current_cpu(1); + for &ptr in &ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + + set_current_cpu(0); + let mut drained = Vec::with_capacity(object_count); + for _ in 0..object_count { + drained.push(allocator.alloc(layout).unwrap()); + } + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; } - static LOWMEM_OS: LowmemOs = LowmemOs; - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE, PAGE_SIZE * 4); + assert_eq!(count_free_pages(&allocator), baseline); +} + +#[test] +fn global_lowmem_fragmentation_recovery() { + const REGION_SIZE: usize = 512 * PAGE_SIZE; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, TEST_HEAP_SIZE) }; - unsafe { allocator.init(region, 1, &LOWMEM_OS).unwrap() }; + init_global_allocator(&allocator, &mut region, 1, &LOWMEM_OS); + + let mut addrs = Vec::new(); + while let Ok(addr) = allocator.alloc_pages_lowmem(1, PAGE_SIZE) { + addrs.push(addr); + } + assert!(addrs.len() > 8); + + for &addr in addrs.iter().step_by(2) { + allocator.dealloc_pages(addr, 1); + } + assert!(allocator.alloc_pages_lowmem(2, 2 * PAGE_SIZE).is_err()); + + for &addr in addrs.iter().skip(1).step_by(2) { + allocator.dealloc_pages(addr, 1); + } + + let addr = allocator.alloc_pages_lowmem(2, 2 * PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 2); +} + +#[test] +fn global_lowmem_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global_allocator(&allocator, &mut region, 1, &LOWMEM_OS); let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); assert!(addr >= allocator.managed_heap_start()); allocator.dealloc_pages(addr, 1); - - host_dealloc(region_ptr, region_layout); } #[test] fn global_unaligned_region_start() { - let (region_ptr, region_layout) = host_alloc(TEST_HEAP_SIZE + PAGE_SIZE, PAGE_SIZE * 4); - let region_start = region_ptr as usize + 1; + let mut region = HostRegion::new(TEST_HEAP_SIZE + PAGE_SIZE, PAGE_SIZE * 4); + let region_start = region.addr() + 1; let region_size = TEST_HEAP_SIZE; let allocator = GlobalAllocator::::new(); - let region = - unsafe { core::slice::from_raw_parts_mut(region_ptr.wrapping_add(1), region_size) }; - init_global(&allocator, region, 1); + let unaligned_region = unsafe { region.subslice(1, region_size) }; + unsafe { allocator.init(unaligned_region, 1, &TEST_OS).unwrap() }; let managed_start = allocator.managed_heap_start(); let managed_end = managed_start + allocator.managed_heap_size(); @@ -598,8 +815,6 @@ fn global_unaligned_region_start() { let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); assert!(addr >= managed_start && addr < managed_end); allocator.dealloc_pages(addr, 1); - - host_dealloc(region_ptr, region_layout); } #[test] @@ -609,12 +824,9 @@ fn global_rejects_region_without_one_managed_page() { let slab_offset = align_up(buddy_meta_size, slab_align); let slab_size = core::mem::size_of::>>(); let region_size = PAGE_SIZE + slab_offset + slab_size - 1; - let (region_ptr, region_layout) = host_alloc(region_size, PAGE_SIZE); + let mut region = HostRegion::new(region_size, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - let region = unsafe { core::slice::from_raw_parts_mut(region_ptr, region_size) }; - let err = unsafe { allocator.init(region, 1, &TEST_OS) }.unwrap_err(); + let err = unsafe { allocator.init(region.as_mut_slice(), 1, &TEST_OS) }.unwrap_err(); assert_eq!(err, AllocError::InvalidParam); - - host_dealloc(region_ptr, region_layout); } diff --git a/tests/stress_test.rs b/tests/stress_test.rs index 10d0d2a..84c0fc7 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -1,84 +1,51 @@ //! Stress tests for allocator stability. -use buddy_slab_allocator::{GlobalAllocator, OsImpl}; -use rand::{RngExt, SeedableRng}; -use std::alloc::{alloc, dealloc, Layout}; -use std::sync::atomic::{AtomicUsize, Ordering}; +mod common; + +use buddy_slab_allocator::{GlobalAllocator, SizeClass}; +use common::{ + count_free_pages, init_global, nonnull_from_addr, seeded_rng, set_current_cpu, HostRegion, + TEST_OS, +}; +use rand::RngExt; +use std::alloc::Layout; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Barrier, Mutex}; +use std::thread; const PAGE_SIZE: usize = 0x1000; const HEAP_SIZE: usize = 64 * 1024 * 1024; +const WORKERS: usize = 4; -struct MockOs { - cpu: AtomicUsize, -} - -impl MockOs { - const fn new() -> Self { - Self { - cpu: AtomicUsize::new(0), - } - } - - fn set_cpu(&self, cpu: usize) { - self.cpu.store(cpu, Ordering::Relaxed); - } -} - -impl OsImpl for MockOs { - fn current_cpu_idx(&self) -> usize { - self.cpu.load(Ordering::Relaxed) - } - fn virt_to_phys(&self, vaddr: usize) -> usize { - vaddr - } -} - -static MOCK_OS: MockOs = MockOs::new(); - -struct TestHeap { - ptr: *mut u8, - layout: Layout, -} - -impl TestHeap { - fn new(size: usize) -> Self { - let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "failed to allocate test heap"); - Self { ptr, layout } - } - - fn as_region(&mut self) -> &mut [u8] { - unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } - } -} - -impl Drop for TestHeap { - fn drop(&mut self) { - unsafe { dealloc(self.ptr, self.layout) }; - } -} - -fn init_allocator(allocator: &GlobalAllocator, heap: &mut TestHeap, cpu_count: usize) { - MOCK_OS.set_cpu(0); - unsafe { - allocator - .init(heap.as_region(), cpu_count, &MOCK_OS) - .unwrap() - }; +fn assert_recovered_with_cached_slabs( + allocator: &GlobalAllocator, + baseline: usize, + cpu_count: usize, + cached_classes: &[SizeClass], +) { + let recovered = count_free_pages(allocator); + let retained_pages = cached_classes + .iter() + .map(|sc| sc.slab_pages(PAGE_SIZE)) + .sum::() + * cpu_count; + assert!( + recovered + retained_pages >= baseline, + "recovered {recovered} pages, baseline {baseline}, retained allowance {retained_pages}", + ); } #[test] #[ignore = "stress test"] fn stress_random_mixed_alloc_free() { - let mut heap = TestHeap::new(HEAP_SIZE); + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &mut heap, 2); - let mut rng = rand::rngs::StdRng::from_seed([0; 32]); - let mut allocated: Vec<(core::ptr::NonNull, Layout)> = Vec::new(); + init_global(&allocator, &mut region, 2, &TEST_OS); + let mut rng = seeded_rng(0); + let mut allocated: Vec<(usize, Layout)> = Vec::new(); for i in 0..10_000 { - MOCK_OS.set_cpu(i % 2); + set_current_cpu(i % 2); if allocated.is_empty() || rng.random_bool(0.65) { let size: usize = rng.random_range(8..8193); let layout = if size <= 2048 { @@ -89,35 +56,35 @@ fn stress_random_mixed_alloc_free() { }; if let Ok(ptr) = allocator.alloc(layout) { - allocated.push((ptr, layout)); + allocated.push((ptr.as_ptr() as usize, layout)); } } else { let idx = rng.random_range(0..allocated.len()); - let (ptr, layout) = allocated.swap_remove(idx); - unsafe { allocator.dealloc(ptr, layout) }; + let (addr, layout) = allocated.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; } } - for (ptr, layout) in allocated { - unsafe { allocator.dealloc(ptr, layout) }; + for (addr, layout) in allocated { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; } } #[test] #[ignore = "stress test"] fn stress_exhaustion_recovery() { - let mut heap = TestHeap::new(HEAP_SIZE); + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &mut heap, 1); + init_global(&allocator, &mut region, 1, &TEST_OS); let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); let mut allocated = Vec::new(); while let Ok(ptr) = allocator.alloc(layout) { - allocated.push(ptr); + allocated.push(ptr.as_ptr() as usize); } - for ptr in allocated.drain(..allocated.len() / 4) { - unsafe { allocator.dealloc(ptr, layout) }; + for addr in allocated.drain(..allocated.len() / 4) { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; } let recovered = allocator.alloc(layout); @@ -127,39 +94,330 @@ fn stress_exhaustion_recovery() { unsafe { allocator.dealloc(ptr, layout) }; } - for ptr in allocated { - unsafe { allocator.dealloc(ptr, layout) }; + for addr in allocated { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; } } #[test] #[ignore = "stress test"] fn stress_fragmentation_recovery() { - let mut heap = TestHeap::new(HEAP_SIZE); + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); - init_allocator(&allocator, &mut heap, 2); + init_global(&allocator, &mut region, 2, &TEST_OS); let small_layout = Layout::from_size_align(64, 8).unwrap(); let mut small_ptrs = Vec::new(); for i in 0..4000 { - MOCK_OS.set_cpu(i % 2); + set_current_cpu(i % 2); if let Ok(ptr) = allocator.alloc(small_layout) { - small_ptrs.push(ptr); + small_ptrs.push(ptr.as_ptr() as usize); } } for i in (0..small_ptrs.len()).step_by(2) { - unsafe { allocator.dealloc(small_ptrs[i], small_layout) }; + unsafe { allocator.dealloc(nonnull_from_addr(small_ptrs[i]), small_layout) }; } let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap(); let large = allocator.alloc(large_layout); - for ptr in small_ptrs.into_iter().skip(1).step_by(2) { - unsafe { allocator.dealloc(ptr, small_layout) }; + for addr in small_ptrs.into_iter().skip(1).step_by(2) { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; } if let Ok(ptr) = large { unsafe { allocator.dealloc(ptr, large_layout) }; } } + +#[test] +#[ignore = "stress test"] +fn stress_multithread_mixed_alloc_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, WORKERS, &TEST_OS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x1000 + cpu as u64); + let mut live: Vec<(usize, Layout)> = Vec::new(); + for _ in 0..4_000 { + if live.is_empty() || rng.random_bool(0.65) { + let layout = if rng.random_bool(0.7) { + let size: usize = rng.random_range(8..=2048); + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let page_counts = [1usize, 2, 4, 8]; + let pages = page_counts[rng.random_range(0..page_counts.len())]; + Layout::from_size_align(pages * PAGE_SIZE, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + live.push((ptr.as_ptr() as usize, layout)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, layout) = live.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + } + + for (addr, layout) in live { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + }); + } + }); + + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &SizeClass::ALL); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_remote_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, WORKERS, &TEST_OS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let layout = Layout::from_size_align(64, 8).unwrap(); + let queues: Vec<_> = (0..WORKERS) + .map(|_| Mutex::new(Vec::::new())) + .collect(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let queues = &queues; + scope.spawn(move || { + set_current_cpu(cpu); + let mut local = Vec::new(); + for _ in 0..256 { + local.push(allocator.alloc(layout).unwrap().as_ptr() as usize); + } + + let target = (cpu + 1) % WORKERS; + queues[target].lock().unwrap().extend(local); + barrier.wait(); + + let remote = { + let mut queue = queues[cpu].lock().unwrap(); + queue.drain(..).collect::>() + }; + for addr in remote { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + barrier.wait(); + + let mut drained = Vec::new(); + for _ in 0..256 { + drained.push(allocator.alloc(layout).unwrap()); + } + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; + } + barrier.wait(); + }); + } + }); + + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &[SizeClass::Bytes64]); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_page_alloc_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, WORKERS, &TEST_OS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x2000 + cpu as u64); + let page_counts = [1usize, 2, 4, 8]; + let alignments = [PAGE_SIZE, 2 * PAGE_SIZE, 4 * PAGE_SIZE, 8 * PAGE_SIZE]; + let mut live = Vec::new(); + + for _ in 0..2_000 { + if live.is_empty() || rng.random_bool(0.6) { + let count = page_counts[rng.random_range(0..page_counts.len())]; + let align = alignments[rng.random_range(0..alignments.len())]; + if let Ok(addr) = allocator.alloc_pages(count, align.max(PAGE_SIZE)) { + live.push((addr, count)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, count) = live.swap_remove(idx); + allocator.dealloc_pages(addr, count); + } + } + + for (addr, count) in live { + allocator.dealloc_pages(addr, count); + } + }); + } + }); + + assert_eq!(count_free_pages(allocator), baseline); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_fragmentation_recovery() { + const REGION_SIZE: usize = 4 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, WORKERS, &TEST_OS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let before_cleanup_failed = AtomicBool::new(false); + let partial_cleanup_failed = AtomicBool::new(false); + let large_layout = Layout::from_size_align(32 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let small_layout = Layout::from_size_align(64, 8).unwrap(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let before_cleanup_failed = &before_cleanup_failed; + let partial_cleanup_failed = &partial_cleanup_failed; + scope.spawn(move || { + set_current_cpu(cpu); + let mut live = Vec::new(); + while let Ok(ptr) = allocator.alloc(small_layout) { + live.push(ptr.as_ptr() as usize); + } + + barrier.wait(); + if cpu == 0 { + match allocator.alloc(large_layout) { + Ok(ptr) => unsafe { + allocator.dealloc(ptr, large_layout); + }, + Err(_) => before_cleanup_failed.store(true, Ordering::Relaxed), + } + } + + barrier.wait(); + let mut retained = Vec::new(); + for (idx, addr) in live.into_iter().enumerate() { + if idx % 2 == 0 { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; + } else { + retained.push(addr); + } + } + + barrier.wait(); + if cpu == 0 { + match allocator.alloc(large_layout) { + Ok(ptr) => unsafe { + allocator.dealloc(ptr, large_layout); + }, + Err(_) => partial_cleanup_failed.store(true, Ordering::Relaxed), + } + } + + barrier.wait(); + for addr in retained { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; + } + + barrier.wait(); + if cpu == 0 { + let ptr = allocator.alloc(large_layout).unwrap(); + unsafe { allocator.dealloc(ptr, large_layout) }; + } + barrier.wait(); + }); + } + }); + + assert!(before_cleanup_failed.load(Ordering::Relaxed)); + assert!(partial_cleanup_failed.load(Ordering::Relaxed)); + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &[SizeClass::Bytes64]); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_exhaustion_recovery() { + const REGION_SIZE: usize = 8 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, WORKERS, &TEST_OS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let exhausted = AtomicBool::new(false); + let recovered = AtomicBool::new(false); + let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let exhausted = &exhausted; + let recovered = &recovered; + scope.spawn(move || { + set_current_cpu(cpu); + let mut live = Vec::new(); + while let Ok(ptr) = allocator.alloc(layout) { + live.push(ptr.as_ptr() as usize); + } + + barrier.wait(); + if cpu == 0 { + exhausted.store(allocator.alloc(layout).is_err(), Ordering::Relaxed); + } + + barrier.wait(); + let mut retained = Vec::new(); + for (idx, addr) in live.into_iter().enumerate() { + if idx % 4 == 0 { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } else { + retained.push(addr); + } + } + + barrier.wait(); + if cpu == 0 { + if let Ok(ptr) = allocator.alloc(layout) { + recovered.store(true, Ordering::Relaxed); + unsafe { allocator.dealloc(ptr, layout) }; + } + } + + barrier.wait(); + for addr in retained { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + barrier.wait(); + }); + } + }); + + assert!(exhausted.load(Ordering::Relaxed)); + assert!(recovered.load(Ordering::Relaxed)); + assert_eq!(count_free_pages(allocator), baseline); +} From d7e0770de990044eb1ecdafbec1e2ca03ddf4701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 10:50:46 +0800 Subject: [PATCH 13/19] Refactor GlobalAllocator to support multiple managed sections - Introduced `BuddySection` and `ManagedSection` to manage multiple memory regions. - Updated `GlobalAllocator` to compute and manage metadata for multiple sections. - Modified allocation and deallocation methods to work with sections. - Enhanced tests to validate behavior with multiple regions, including allocation, deallocation, and memory management. - Added stress tests for concurrent allocation and region addition. --- README.md | 43 ++- README_CN.md | 42 ++- docs/design.md | 136 +++++--- src/buddy/mod.rs | 647 ++++++++++++++++++++++++++++---------- src/error.rs | 6 + src/global.rs | 169 +++++----- src/lib.rs | 2 +- tests/dma32_pages_test.rs | 14 +- tests/integration_test.rs | 515 ++++++++++++++++++++---------- tests/stress_test.rs | 58 ++++ 10 files changed, 1147 insertions(+), 485 deletions(-) diff --git a/README.md b/README.md index a89586c..5772995 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A `no_std` two-level allocator for kernel and embedded environments, combining a The current implementation is built from three layers: 1. `BuddyAllocator` - Manages a contiguous virtual heap in page units, with power-of-two splitting and merging. + Manages one or more virtual memory sections in page units, with power-of-two splitting and merging. 2. `SlabAllocator` Manages small objects up to 2048 bytes with fixed size classes. 3. `GlobalAllocator` @@ -70,6 +70,7 @@ sequenceDiagram ## Features - Buddy page allocation with splitting and merging +- Dynamic hot-add of managed regions via `add_region` - Slab allocation for 9 size classes: `8..=2048` - Per-CPU slab caches - Lock-free cross-CPU remote frees @@ -117,6 +118,17 @@ let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout); } + +// More memory can be added later. +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + allocator.add_region(extra_region).unwrap(); +} ``` ## Using Buddy and Slab separately @@ -130,15 +142,13 @@ use buddy_slab_allocator::{ use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 16 * 1024 * 1024; - -let heap_start = 0x8000_0000usize; -let meta_size = BuddyAllocator::::required_meta_size(HEAP_SIZE); -let meta_ptr = 0x9000_0000 as *mut u8; +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; let mut buddy = BuddyAllocator::::new(); unsafe { - buddy.init(meta_ptr, meta_size, heap_start, HEAP_SIZE, None).unwrap(); + buddy.init(region, None).unwrap(); } let mut slab = SlabAllocator::::new(); @@ -161,14 +171,26 @@ match slab.dealloc(ptr, layout) { buddy.dealloc_pages(base, pages); } } + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + buddy.add_region(extra_region).unwrap(); +} ``` ## Public API summary - `GlobalAllocator` - High-level allocator facade that can also implement `GlobalAlloc`. + High-level allocator facade that can also implement `GlobalAlloc`, and supports `add_region`, `managed_section_count`, `managed_section`, `managed_bytes`, and `allocated_bytes`. - `BuddyAllocator` - Standalone page allocator. + Standalone multi-section page allocator, supporting `init`, `add_region`, section queries, `managed_bytes`, and `allocated_bytes`. +- `ManagedSection` + Read-only summary for one managed section. - `SlabAllocator` Standalone slab allocator. - `SizeClass` @@ -180,6 +202,9 @@ match slab.dealloc(ptr, layout) { - `OsImpl` Provides `current_cpu_idx()` and `virt_to_phys()` for per-CPU routing and lowmem selection. +`managed_bytes` counts only allocatable heap bytes and excludes region-prefix metadata. +`allocated_bytes` is backend page occupancy, not the exact sum of requested `layout.size()`. + ## Testing ```bash diff --git a/README_CN.md b/README_CN.md index e311b04..1c46fd2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -7,7 +7,7 @@ 当前实现由三层组成： 1. `BuddyAllocator` - 管理一段连续虚拟地址区间中的页，支持按 2 的幂分裂与合并。 + 管理一个或多个虚拟内存 section，支持按 2 的幂分裂与合并。 2. `SlabAllocator` 管理 `<= 2048` 字节的小对象，采用固定 size class。 3. `GlobalAllocator` @@ -70,6 +70,7 @@ sequenceDiagram ## 特性 - Buddy 页分配，支持拆分与合并 +- 支持通过 `add_region` 动态追加可管理 region - Slab 小对象分配，固定 9 个 size class：`8..=2048` - per-CPU slab cache - 跨 CPU 释放走 lock-free remote free @@ -117,6 +118,16 @@ let ptr = allocator.alloc(layout).unwrap(); unsafe { allocator.dealloc(ptr, layout); } + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + allocator.add_region(extra_region).unwrap(); +} ``` ## 分别使用 Buddy 与 Slab @@ -130,15 +141,13 @@ use buddy_slab_allocator::{ use core::alloc::Layout; const PAGE_SIZE: usize = 0x1000; -const HEAP_SIZE: usize = 16 * 1024 * 1024; - -let heap_start = 0x8000_0000usize; -let meta_size = BuddyAllocator::::required_meta_size(HEAP_SIZE); -let meta_ptr = 0x9000_0000 as *mut u8; +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; let mut buddy = BuddyAllocator::::new(); unsafe { - buddy.init(meta_ptr, meta_size, heap_start, HEAP_SIZE, None).unwrap(); + buddy.init(region, None).unwrap(); } let mut slab = SlabAllocator::::new(); @@ -161,14 +170,26 @@ match slab.dealloc(ptr, layout) { buddy.dealloc_pages(base, pages); } } + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + buddy.add_region(extra_region).unwrap(); +} ``` ## 公开 API 摘要 - `GlobalAllocator` - 高层门面，也可以作为 `GlobalAlloc` 使用。 + 高层门面，也可以作为 `GlobalAlloc` 使用，并支持 `add_region`、`managed_section_count`、`managed_section`、`managed_bytes` 与 `allocated_bytes`。 - `BuddyAllocator` - 独立页分配器。 + 独立多 section 页分配器，支持 `init`、`add_region`、section 查询、`managed_bytes` 与 `allocated_bytes`。 +- `ManagedSection` + 单个 managed section 的只读摘要。 - `SlabAllocator` 独立 slab 分配器。 - `SizeClass` @@ -180,6 +201,9 @@ match slab.dealloc(ptr, layout) { - `OsImpl` 提供 `current_cpu_idx()` 和 `virt_to_phys()`，用于 per-CPU 路由与 lowmem 选择。 +`managed_bytes` 只统计可分配 heap，不包含 region 前缀 metadata。 +`allocated_bytes` 表示后端页占用，不是用户请求的 `layout.size()` 精确求和。 + ## 测试 ```bash diff --git a/docs/design.md b/docs/design.md index 6191401..a53caa8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,6 +1,6 @@ # buddy-slab-allocator 设计文档 -本文档面向阅读和维护当前实现的开发者，描述这个分配器在 `buddy + slab` 两级结构下的核心数据结构、初始化过程、`alloc` / `dealloc` 路径，以及多核并发场景下的协作方式。 +本文档面向阅读和维护当前实现的开发者，描述这个分配器在 `buddy + slab` 两级结构下的核心数据结构、初始化过程、`add_region` 热添加流程、`alloc` / `dealloc` 路径，以及多核并发场景下的协作方式。 文中描述以当前仓库实现为准，重点对应以下模块： @@ -20,7 +20,7 @@ 因此整体采用两级结构： - 后端：`BuddyAllocator` - 负责管理一段连续虚拟地址区间中的页。 + 负责管理一个或多个 section 中的页。 - 前端：`SlabAllocator` 负责管理不同 size class 的小对象。 - 门面：`GlobalAllocator` @@ -42,8 +42,9 @@ flowchart TD SC64 --> SPH SC2K --> SPH - B --> PM[PageMeta[]] - B --> FL[free_lists by order] + B --> SH[BuddySection intrusive list] + SH --> PM[PageMeta[]] + SH --> FL[free_lists by order] ``` ### 2.1 地址语义 @@ -59,75 +60,117 @@ flowchart TD - 候选块的虚拟地址通过 `OsImpl::virt_to_phys()` 翻译成物理地址。 - 只有物理地址低于 `4 GiB` 的块才会被当作 DMA32 候选。 -## 3. 初始化与内存布局 +### 2.2 容量统计语义 -`GlobalAllocator::init(region, cpu_count, os)` 会把调用者提供的整段区域拆成两部分： +当前公开了两类整体容量统计： -- 前缀：元数据区 -- 后缀：真正交给 buddy 管理的 heap 区 +- `managed_bytes` + 所有 section 中可分配 heap 的总字节数，不包含 region 前缀 metadata。 +- `allocated_bytes` + 后端页占用字节数，按 `managed_bytes - free_pages * PAGE_SIZE` 计算。 -元数据区又细分为： +因此 `allocated_bytes` 的口径是“页后端已经占住多少字节”，不是用户请求的 `layout.size()` 精确累加。 +它会包含： +- slab 页 +- empty slab cache 保留页 +- 对齐放大 +- buddy / slab 的内部碎片 + +## 3. 初始化、追加 Region 与内存布局 + +当前实现支持两种 region 进入 allocator 的方式： + +- `GlobalAllocator::init(region, cpu_count, os)` + 注册首个 region，并初始化唯一一份 per-CPU slab 槽位。 +- `GlobalAllocator::add_region(region)` + 在运行时追加新的 region，只扩展 buddy 后端，不再重复创建 per-CPU slab。 + +对 buddy 而言，每个 region 都会被拆成： + +- 前缀：region 自带元数据 +- 后缀：该 section 的 managed heap + +首个 global region 的元数据前缀包含： + +- `BuddySection` - `PageMeta[]` - `per_cpu_slabs: [SpinMutex; cpu_count]` +后续追加 region 的元数据前缀只包含： + +- `BuddySection` +- `PageMeta[]` + ```mermaid flowchart LR - A[region start] --> B[aligned meta_start] - B --> C[Buddy PageMeta array] - C --> D[padding for alignment] - D --> E[per-CPU SlabAllocator slots] - E --> F[padding to PAGE_SIZE] - F --> G[managed_heap_start] - G --> H[managed heap managed by BuddyAllocator] + A[region start] --> B[aligned section_start] + B --> C[BuddySection] + C --> D[PageMeta array] + D --> E{first global region?} + E -- Yes --> F[per-CPU SlabAllocator slots] + E -- No --> G[no slab slots here] + F --> H[padding to PAGE_SIZE] + G --> H + H --> I[section heap start] + I --> J[managed heap of this section] ``` ### 3.1 初始化步骤 `GlobalAllocator::init()` 的主要步骤如下： -1. 根据 `region.len()` 和 `cpu_count` 计算最多能管理多少页。 -2. 为这批页预留足够的 `PageMeta[]` 和 per-CPU slab 槽位。 -3. 选择一个页对齐的 `managed_heap_start`。 -4. 调用 `BuddyAllocator::init()` 初始化后端页分配器。 +1. 根据 `region.len()` 和 `cpu_count` 计算首个 section 最多能管理多少页。 +2. 在 region 前缀预留 `BuddySection + PageMeta[] + per_cpu_slabs`。 +3. 选择一个页对齐的 section heap 起点。 +4. 将首个 section 注册进 `BuddyAllocator`。 5. 原地构造每个 CPU 对应的 `SpinMutex`。 6. 将 `per_cpu_slabs`、`cpu_count`、`os` 写入 `GlobalAllocator`。 +`GlobalAllocator::add_region()` 的步骤更简单： + +1. 校验 allocator 已初始化。 +2. 在新 region 前缀预留 `BuddySection + PageMeta[]`。 +3. 计算该 section 的可管理 heap。 +4. 将该 section 追加到 buddy 的 intrusive list 尾部。 + ### 3.2 为什么 metadata 放在 region 前缀 这样做有几个好处： - 不需要额外的启动期堆分配。 -- `BuddyAllocator` 的 `PageMeta[]` 生命周期天然和被管理区域一致。 +- `BuddySection` 与 `PageMeta[]` 都跟随 region 自身生命周期。 - `per_cpu_slabs` 的内存来源和被管理堆绑在一起，部署简单。 +- 不需要预先声明最大 section 数量。 代价是： - metadata 会消耗一部分可分配空间。 -- `managed_heap_start` 可能晚于 `region` 起点。 +- 每个 section 的 heap 起点都可能晚于各自的 region 起点。 +- section 查询与地址归属定位当前采用线性扫描。 ## 4. BuddyAllocator 设计 -`BuddyAllocator` 管理页级连续内存，是整个系统的共享后端。 +`BuddyAllocator` 管理页级内存，是整个系统的共享后端。它的基本单位不再是单一 heap，而是一个按注册顺序串起来的 section 链表。 ### 4.1 核心数据结构 ```mermaid classDiagram class BuddyAllocator { + +sections_head: *mut BuddySection + +section_count: usize + +os: Option<&dyn OsImpl> + } + + class BuddySection { + +next: *mut BuddySection +meta: *mut PageMeta +heap_start: usize +heap_size: usize +free_lists: [u32; MAX_ORDER+1] +free_pages: usize - +os: Option<&dyn OsImpl> - } - - class PageMeta { - +flags: PageFlags - +order: u8 - +prev: u32 - +next: u32 + +total_pages: usize } class PageFlags { @@ -140,21 +183,22 @@ classDiagram 说明： -- `PageMeta` 是每页一项的外部元数据。 -- `free_lists[order]` 保存对应阶的空闲块链表头。 -- 链表节点不是单独分配的，而是复用 `PageMeta.prev/next`。 +- `BuddyAllocator` 自身只维护 section 链表头和 section 数量。 +- 每个 `BuddySection` 都拥有自己的一组 `PageMeta[]` 和 `free_lists`。 +- `PageMeta` 仍然是每页一项的外部元数据。 +- 同一 section 内按 buddy 规则拆分和合并，section 之间不跨界合并。 ### 4.2 buddy 初始化 -初始化时，buddy 会从低地址到高地址扫描整段 heap： +初始化某个 section 时，buddy 会从低地址到高地址扫描该 section 的整段 heap： - 每次尽量切出“当前地址自然对齐、且仍然放得下”的最大 order 块。 - 将该块挂到对应 order 的 free list。 这一步的结果是： -- 所有页都被覆盖。 -- free list 从一开始就处于可分裂、可合并的标准 buddy 形态。 +- 该 section 中的所有页都被覆盖。 +- 该 section 的 free list 从一开始就处于可分裂、可合并的标准 buddy 形态。 ### 4.3 页分配流程 @@ -163,13 +207,17 @@ flowchart TD A[alloc_pages(count, align)] --> B[计算 order] B --> C[计算 align_order] C --> D[effective_order = max(order, align_order)] - D --> E[从 effective_order 向上找非空 free list] - E --> F{找到块?} - F -- 否 --> G[返回 NoMemory] - F -- 是 --> H[弹出大块] - H --> I[逐级 split 到目标 order] - I --> J[标记头页为 Allocated] - J --> K[返回虚拟地址] + D --> E[按注册顺序扫描 section] + E --> F[在 section 内从 effective_order 向上找非空 free list] + F --> G{找到块?} + G -- 否 --> H[尝试下一个 section] + H --> I{还有 section?} + I -- 否 --> J[返回 NoMemory] + I -- 是 --> F + G -- 是 --> K[弹出大块] + K --> L[逐级 split 到目标 order] + L --> M[标记头页为 Allocated] + M --> N[返回虚拟地址] ``` 注意点： diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index 9fa8266..ea3d5ba 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -1,132 +1,201 @@ //! Buddy page allocator — page-metadata-based with intrusive free lists. //! -//! The allocator manages a contiguous virtual address range ("the heap"). -//! An external metadata region stores one [`PageMeta`] per page frame, -//! enabling O(1) free-list operations without any dynamic allocation. +//! The allocator manages one or more contiguous virtual address ranges ("sections"). +//! Each section stores its own [`BuddySection`] descriptor and [`PageMeta`] array +//! in the caller-provided region prefix, enabling O(1) free-list operations +//! without any dynamic allocation. pub mod page_meta; pub use page_meta::{PageFlags, PageMeta, PFN_NONE}; +use core::ptr; + use crate::error::{AllocError, AllocResult}; -use crate::{is_aligned, OsImpl}; +use crate::{align_up, is_aligned, OsImpl}; use page_meta::{free_list_pop, free_list_push, free_list_remove}; -/// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks. +/// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks. pub const MAX_ORDER: usize = 20; /// DMA32 zone upper bound (4 GiB physical). const DMA32_LIMIT: usize = 0x1_0000_0000; -/// Page-metadata-based buddy allocator. -/// -/// `PAGE_SIZE` must be a power of two (commonly 0x1000 = 4 KiB). -pub struct BuddyAllocator { - /// Pointer to the `PageMeta` array in the external metadata region. - meta: *mut PageMeta, - /// Maximum number of pages the metadata can describe. - max_pages: usize, - /// Virtual start address of the managed heap region. - heap_start: usize, - /// Size of the managed heap in bytes. - heap_size: usize, - /// Free-list head PFN per order (`PFN_NONE` = empty). - free_lists: [u32; MAX_ORDER + 1], - /// Number of currently free pages. - free_pages: usize, - /// Total number of pages in the heap. - total_pages: usize, - /// Optional OS interface for DMA32/lowmem address translation. - os: Option<&'static dyn OsImpl>, +struct RegionLayout { + section_start: usize, + meta_start: usize, + managed_heap_start: usize, + managed_heap_size: usize, } -// SAFETY: The allocator is designed to be wrapped in a SpinMutex. -// Raw pointer field `meta` points to the externally supplied metadata region -// whose lifetime is managed by the caller. -unsafe impl Send for BuddyAllocator {} +/// Public read-only summary of a managed section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ManagedSection { + pub start: usize, + pub size: usize, + pub free_pages: usize, + pub total_pages: usize, +} -impl BuddyAllocator { - // ------------------------------------------------------------------ - // Construction helpers - // ------------------------------------------------------------------ +/// Per-region buddy state stored in the region prefix. +#[repr(C)] +pub(crate) struct BuddySection { + pub(crate) next: *mut BuddySection, + pub(crate) region_start: usize, + pub(crate) region_size: usize, + pub(crate) meta: *mut PageMeta, + pub(crate) max_pages: usize, + pub(crate) heap_start: usize, + pub(crate) heap_size: usize, + pub(crate) free_lists: [u32; MAX_ORDER + 1], + pub(crate) free_pages: usize, + pub(crate) total_pages: usize, +} - /// Calculate the metadata-region size (in bytes) required for `heap_size` bytes. - pub const fn required_meta_size(heap_size: usize) -> usize { - let pages = heap_size / PAGE_SIZE; - pages * core::mem::size_of::() +impl BuddySection { + const fn metadata_align() -> usize { + let section_align = core::mem::align_of::(); + let meta_align = core::mem::align_of::(); + if section_align > meta_align { + section_align + } else { + meta_align + } } - /// Create an uninitialised allocator. Call [`init`](Self::init) before use. - pub const fn new() -> Self { - Self { - meta: core::ptr::null_mut(), - max_pages: 0, - heap_start: 0, - heap_size: 0, - free_lists: [PFN_NONE; MAX_ORDER + 1], - free_pages: 0, - total_pages: 0, - os: None, + fn metadata_layout_for_pages(pages: usize) -> Option<(usize, usize)> { + let meta_offset = align_up( + core::mem::size_of::(), + core::mem::align_of::(), + ); + let page_meta_size = pages.checked_mul(core::mem::size_of::())?; + let meta_size = meta_offset.checked_add(page_meta_size)?; + Some((meta_offset, meta_size)) + } + + fn available_heap_pages( + region_end: usize, + section_start: usize, + meta_size: usize, + ) -> Option { + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE); + if managed_heap_start > region_end { + return Some(0); } + Some((region_end - managed_heap_start) / PAGE_SIZE) } -} -impl Default for BuddyAllocator { - fn default() -> Self { - Self::new() + fn can_manage_pages( + region_end: usize, + section_start: usize, + pages: usize, + ) -> bool { + let Some((_, meta_size)) = Self::metadata_layout_for_pages(pages) else { + return false; + }; + let Some(available_pages) = + Self::available_heap_pages::(region_end, section_start, meta_size) + else { + return false; + }; + available_pages >= pages } -} -impl BuddyAllocator { - /// Initialise the allocator over a heap region, using `meta` for page metadata. - /// - /// # Safety - /// - `meta` must point to a writeable region of at least - /// [`required_meta_size`](Self::required_meta_size)`(heap_size)` bytes. - /// - `heap_start` must be page-aligned and the region `[heap_start, heap_start + heap_size)` - /// must not overlap with the metadata region. - /// - The caller is responsible for ensuring the metadata region lives as long as this allocator. - pub unsafe fn init( - &mut self, - meta: *mut u8, + fn compute_region_layout( + region_start: usize, + region_size: usize, + ) -> Option { + if region_size == 0 || !PAGE_SIZE.is_power_of_two() { + return None; + } + + let region_end = region_start.checked_add(region_size)?; + let section_start = align_up(region_start, Self::metadata_align()); + if section_start >= region_end { + return None; + } + + let heap_search_start = align_up( + section_start.checked_add(core::mem::size_of::())?, + PAGE_SIZE, + ); + let max_pages = if heap_search_start >= region_end { + 0 + } else { + (region_end - heap_search_start) / PAGE_SIZE + }; + + let mut low = 0usize; + let mut high = max_pages; + while low < high { + let mid = low + (high - low).div_ceil(2); + if Self::can_manage_pages::(region_end, section_start, mid) { + low = mid; + } else { + high = mid - 1; + } + } + + if low == 0 { + return None; + } + + let (meta_offset, meta_size) = Self::metadata_layout_for_pages(low)?; + let meta_start = section_start.checked_add(meta_offset)?; + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE); + let managed_heap_size = low.checked_mul(PAGE_SIZE)?; + + Some(RegionLayout { + section_start, + meta_start, + managed_heap_start, + managed_heap_size, + }) + } + + unsafe fn init_at( + section_ptr: *mut BuddySection, + region_start: usize, + region_size: usize, + meta_ptr: *mut u8, meta_size: usize, heap_start: usize, heap_size: usize, - os: Option<&'static dyn OsImpl>, ) -> AllocResult { - // Validate if !PAGE_SIZE.is_power_of_two() { return Err(AllocError::InvalidParam); } if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 { return Err(AllocError::InvalidParam); } + let total_pages = heap_size / PAGE_SIZE; - let required = Self::required_meta_size(heap_size); + let required = BuddyAllocator::::required_meta_size(heap_size); if meta_size < required { return Err(AllocError::InvalidParam); } - let meta_ptr = meta as *mut PageMeta; - - // Zero-init all page metadata + let meta = meta_ptr as *mut PageMeta; for i in 0..total_pages { - meta_ptr.add(i).write(PageMeta::new()); + meta.add(i).write(PageMeta::new()); } - self.meta = meta_ptr; - self.max_pages = total_pages; - self.heap_start = heap_start; - self.heap_size = heap_size; - self.free_lists = [PFN_NONE; MAX_ORDER + 1]; - self.free_pages = 0; - self.total_pages = total_pages; - self.os = os; + section_ptr.write(BuddySection { + next: ptr::null_mut(), + region_start, + region_size, + meta, + max_pages: total_pages, + heap_start, + heap_size, + free_lists: [PFN_NONE; MAX_ORDER + 1], + free_pages: 0, + total_pages, + }); - // Build free lists: greedily carve the largest possible blocks. + let section = &mut *section_ptr; let mut pfn: usize = 0; while pfn < total_pages { - // Find highest order block that (a) fits and (b) is naturally aligned. let mut order = MAX_ORDER; loop { let block_pages = 1usize << order; @@ -139,58 +208,242 @@ impl BuddyAllocator { order -= 1; } let block_pages = 1usize << order; - let m = &mut *self.meta.add(pfn); + let m = &mut *section.meta.add(pfn); m.flags = PageFlags::Free; m.order = order as u8; - free_list_push(self.meta, &mut self.free_lists, pfn as u32, order); - self.free_pages += block_pages; + free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); + section.free_pages += block_pages; pfn += block_pages; } + Ok(()) + } + + #[inline] + fn contains_heap_addr(&self, addr: usize) -> bool { + addr >= self.heap_start && addr < self.heap_start + self.heap_size + } + + #[inline] + fn summary(&self) -> ManagedSection { + ManagedSection { + start: self.heap_start, + size: self.heap_size, + free_pages: self.free_pages, + total_pages: self.total_pages, + } + } +} + +/// Page-metadata-based buddy allocator. +/// +/// `PAGE_SIZE` must be a power of two (commonly 0x1000 = 4 KiB). +pub struct BuddyAllocator { + sections_head: *mut BuddySection, + sections_tail: *mut BuddySection, + section_count: usize, + os: Option<&'static dyn OsImpl>, +} + +// SAFETY: The allocator is designed to be wrapped in a SpinMutex. +// All section pointers point into caller-provided regions whose lifetime is managed externally. +unsafe impl Send for BuddyAllocator {} + +impl BuddyAllocator { + /// Calculate the metadata-region size (in bytes) required for `heap_size` bytes. + pub const fn required_meta_size(heap_size: usize) -> usize { + let pages = heap_size / PAGE_SIZE; + pages * core::mem::size_of::() + } + + /// Create an uninitialised allocator. Call [`init`](Self::init) before use. + pub const fn new() -> Self { + Self { + sections_head: ptr::null_mut(), + sections_tail: ptr::null_mut(), + section_count: 0, + os: None, + } + } +} + +impl Default for BuddyAllocator { + fn default() -> Self { + Self::new() + } +} + +impl BuddyAllocator { + pub(crate) fn reset(&mut self, os: Option<&'static dyn OsImpl>) { + self.sections_head = ptr::null_mut(); + self.sections_tail = ptr::null_mut(); + self.section_count = 0; + self.os = os; + } + + /// Initialise the allocator over the first section. + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - Bytes consumed by metadata become unavailable for allocation. + pub unsafe fn init( + &mut self, + region: &mut [u8], + os: Option<&'static dyn OsImpl>, + ) -> AllocResult { + self.reset(os); + self.add_region(region) + } + + /// Add a new managed region after initialisation. + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - The region must not overlap any existing managed region. + pub unsafe fn add_region(&mut self, region: &mut [u8]) -> AllocResult { + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); + let layout = BuddySection::compute_region_layout::(region_start, region_size) + .ok_or(AllocError::InvalidParam)?; + self.add_region_raw( + region_start, + region_size, + layout.section_start as *mut BuddySection, + layout.meta_start as *mut u8, + Self::required_meta_size(layout.managed_heap_size), + layout.managed_heap_start, + layout.managed_heap_size, + ) + } + + pub(crate) unsafe fn add_region_raw( + &mut self, + region_start: usize, + region_size: usize, + section_ptr: *mut BuddySection, + meta_ptr: *mut u8, + meta_size: usize, + heap_start: usize, + heap_size: usize, + ) -> AllocResult { + let region_end = region_start + .checked_add(region_size) + .ok_or(AllocError::InvalidParam)?; + let heap_end = heap_start + .checked_add(heap_size) + .ok_or(AllocError::InvalidParam)?; + if heap_end > region_end { + return Err(AllocError::InvalidParam); + } + + let mut section = self.sections_head; + while !section.is_null() { + let existing = &*section; + let existing_end = existing + .region_start + .checked_add(existing.region_size) + .ok_or(AllocError::InvalidParam)?; + if region_start < existing_end && existing.region_start < region_end { + return Err(AllocError::MemoryOverlap); + } + section = existing.next; + } + + BuddySection::init_at::( + section_ptr, + region_start, + region_size, + meta_ptr, + meta_size, + heap_start, + heap_size, + )?; + + if self.sections_head.is_null() { + self.sections_head = section_ptr; + } else { + (*self.sections_tail).next = section_ptr; + } + self.sections_tail = section_ptr; + self.section_count += 1; + log::debug!( - "BuddyAllocator: heap {:#x}..{:#x}, {} pages, {} free", + "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages", + region_start, + region_size, heap_start, - heap_start + heap_size, - total_pages, - self.free_pages, + heap_end, + heap_size / PAGE_SIZE, ); Ok(()) } - // ------------------------------------------------------------------ - // Queries - // ------------------------------------------------------------------ + /// Number of managed sections. + pub fn section_count(&self) -> usize { + self.section_count + } - /// Total number of pages managed. - pub fn total_pages(&self) -> usize { - self.total_pages + /// Read-only summary for a managed section by registration order. + pub fn section(&self, index: usize) -> Option { + let mut current = self.sections_head; + let mut i = 0usize; + while !current.is_null() { + if i == index { + return Some(unsafe { (&*current).summary() }); + } + current = unsafe { (*current).next }; + i += 1; + } + None } - /// Number of currently free pages. - pub fn free_pages(&self) -> usize { - self.free_pages + /// Total number of pages managed across all sections. + pub fn total_pages(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).total_pages }; + current = unsafe { (*current).next }; + } + total } - /// Start address of the managed heap. - pub fn heap_start(&self) -> usize { - self.heap_start + /// Total managed heap bytes across all sections. + /// + /// This counts only bytes in allocatable heaps, excluding region-prefix metadata. + pub fn managed_bytes(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).heap_size }; + current = unsafe { (*current).next }; + } + total } - /// Size of the managed heap in bytes. - pub fn heap_size(&self) -> usize { - self.heap_size + /// Number of currently free pages across all sections. + pub fn free_pages(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).free_pages }; + current = unsafe { (*current).next }; + } + total } - // ------------------------------------------------------------------ - // Allocation - // ------------------------------------------------------------------ + /// Allocated backend bytes across all sections. + /// + /// This is computed as managed heap bytes minus currently free page bytes. + /// It reflects page-level occupancy, so it includes slab pages, alignment + /// amplification, and internal fragmentation. + pub fn allocated_bytes(&self) -> usize { + self.managed_bytes() + .saturating_sub(self.free_pages().saturating_mul(PAGE_SIZE)) + } /// Allocate `count` contiguous pages, returning the virtual address. - /// - /// `count` is rounded up to the nearest power of two. - /// Optional `align` must be a power of two (in bytes) and ≥ `PAGE_SIZE`; 0 means - /// default page alignment. pub fn alloc_pages(&mut self, count: usize, align: usize) -> AllocResult { if count == 0 { return Err(AllocError::InvalidParam); @@ -205,19 +458,24 @@ impl BuddyAllocator { return Err(AllocError::InvalidParam); } - // Alignment order: how many pages the alignment corresponds to. let align_order = (align / PAGE_SIZE).trailing_zeros() as usize; let effective_order = order.max(align_order); - self.alloc_order(effective_order) + let mut section = self.sections_head; + while !section.is_null() { + if let Ok(addr) = unsafe { Self::alloc_from_section(&mut *section, effective_order) } { + return Ok(addr); + } + section = unsafe { (*section).next }; + } + + Err(AllocError::NoMemory) } - /// Low-level allocation by buddy order. - fn alloc_order(&mut self, order: usize) -> AllocResult { - // Search upwards for the first non-empty free list + fn alloc_from_section(section: &mut BuddySection, order: usize) -> AllocResult { let mut found_order = order; while found_order <= MAX_ORDER { - if self.free_lists[found_order] != PFN_NONE { + if section.free_lists[found_order] != PFN_NONE { break; } found_order += 1; @@ -226,43 +484,37 @@ impl BuddyAllocator { return Err(AllocError::NoMemory); } - // Pop from found_order - let pfn = unsafe { free_list_pop(self.meta, &mut self.free_lists, found_order) }; + let pfn = unsafe { free_list_pop(section.meta, &mut section.free_lists, found_order) }; debug_assert_ne!(pfn, PFN_NONE); - // Split down to target order let mut current_order = found_order; while current_order > order { current_order -= 1; let buddy_pfn = pfn as usize + (1 << current_order); unsafe { - let bm = &mut *self.meta.add(buddy_pfn); + let bm = &mut *section.meta.add(buddy_pfn); bm.flags = PageFlags::Free; bm.order = current_order as u8; free_list_push( - self.meta, - &mut self.free_lists, + section.meta, + &mut section.free_lists, buddy_pfn as u32, current_order, ); } } - // Mark head page as allocated unsafe { - let m = &mut *self.meta.add(pfn as usize); + let m = &mut *section.meta.add(pfn as usize); m.flags = PageFlags::Allocated; m.order = order as u8; } - self.free_pages -= 1 << order; - let addr = self.heap_start + (pfn as usize) * PAGE_SIZE; - Ok(addr) + section.free_pages -= 1 << order; + Ok(section.heap_start + (pfn as usize) * PAGE_SIZE) } /// Allocate pages whose *physical* address is below 4 GiB (DMA32 zone). - /// - /// Requires [`OsImpl`] to have been set during init. pub fn alloc_pages_lowmem(&mut self, count: usize, align: usize) -> AllocResult { let os = self.os.ok_or(AllocError::InvalidParam)?; @@ -281,31 +533,51 @@ impl BuddyAllocator { return Err(AllocError::InvalidParam); } - // Walk free lists looking for a block below DMA32_LIMIT + let mut section = self.sections_head; + while !section.is_null() { + if let Ok(addr) = + unsafe { Self::alloc_lowmem_from_section(&mut *section, effective_order, os) } + { + return Ok(addr); + } + section = unsafe { (*section).next }; + } + + Err(AllocError::NoMemory) + } + + fn alloc_lowmem_from_section( + section: &mut BuddySection, + effective_order: usize, + os: &'static dyn OsImpl, + ) -> AllocResult { for search_order in effective_order..=MAX_ORDER { - let mut pfn_u32 = self.free_lists[search_order]; + let mut pfn_u32 = section.free_lists[search_order]; while pfn_u32 != PFN_NONE { - let addr = self.heap_start + (pfn_u32 as usize) * PAGE_SIZE; + let addr = section.heap_start + (pfn_u32 as usize) * PAGE_SIZE; let phys = os.virt_to_phys(addr); let block_bytes = (1usize << search_order) * PAGE_SIZE; if phys + block_bytes <= DMA32_LIMIT { - // Found! Remove from list, then split. unsafe { - free_list_remove(self.meta, &mut self.free_lists, pfn_u32, search_order); + free_list_remove( + section.meta, + &mut section.free_lists, + pfn_u32, + search_order, + ); } - // Split let mut current_order = search_order; while current_order > effective_order { current_order -= 1; let buddy_pfn = pfn_u32 as usize + (1 << current_order); unsafe { - let bm = &mut *self.meta.add(buddy_pfn); + let bm = &mut *section.meta.add(buddy_pfn); bm.flags = PageFlags::Free; bm.order = current_order as u8; free_list_push( - self.meta, - &mut self.free_lists, + section.meta, + &mut section.free_lists, buddy_pfn as u32, current_order, ); @@ -313,37 +585,40 @@ impl BuddyAllocator { } unsafe { - let m = &mut *self.meta.add(pfn_u32 as usize); + let m = &mut *section.meta.add(pfn_u32 as usize); m.flags = PageFlags::Allocated; m.order = effective_order as u8; } - self.free_pages -= 1 << effective_order; + section.free_pages -= 1 << effective_order; return Ok(addr); } - pfn_u32 = unsafe { (*self.meta.add(pfn_u32 as usize)).next }; + pfn_u32 = unsafe { (*section.meta.add(pfn_u32 as usize)).next }; } } Err(AllocError::NoMemory) } - // ------------------------------------------------------------------ - // Deallocation - // ------------------------------------------------------------------ - /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). /// /// `addr` must be the exact address returned by alloc. The allocator frees /// the full block size recorded in page metadata, which may be larger than /// `count` if the original allocation was rounded up for buddy order or alignment. pub fn dealloc_pages(&mut self, addr: usize, count: usize) { + let Some(section) = self.find_section_by_addr_mut(addr) else { + debug_assert!( + false, + "dealloc_pages called with address outside all sections" + ); + return; + }; + debug_assert!(is_aligned(addr, PAGE_SIZE)); - debug_assert!(addr >= self.heap_start); debug_assert!(count > 0); - let pfn = (addr - self.heap_start) / PAGE_SIZE; - debug_assert!(pfn < self.max_pages); - let stored = unsafe { &*self.meta.add(pfn) }; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + debug_assert!(pfn < section.max_pages); + let stored = unsafe { &*section.meta.add(pfn) }; debug_assert!( stored.flags == PageFlags::Allocated || stored.flags == PageFlags::Slab, "dealloc_pages called on non-allocated block" @@ -355,53 +630,85 @@ impl BuddyAllocator { expected_order <= order, "dealloc_pages count implies larger order than the allocated block" ); - self.dealloc_order(pfn, order); + Self::dealloc_in_section(section, pfn, order); } - /// Mark the page at `pfn` with the given flags (used by slab to tag pages). + /// Mark the page at `addr` with the given flags (used by slab to tag pages). /// /// # Safety - /// The caller must ensure `pfn` is valid and properly allocated. - pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) { - let pfn = (addr - self.heap_start) / PAGE_SIZE; - (*self.meta.add(pfn)).flags = flags; + /// The caller must ensure `addr` is valid and properly allocated. + pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) -> AllocResult { + let section = self + .find_section_by_addr_mut(addr) + .ok_or(AllocError::NotFound)?; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + (*section.meta.add(pfn)).flags = flags; + Ok(()) } /// Read the flags of the page containing `addr`. - pub fn page_flags(&self, addr: usize) -> PageFlags { - let pfn = (addr - self.heap_start) / PAGE_SIZE; - unsafe { (*self.meta.add(pfn)).flags } + pub fn page_flags(&self, addr: usize) -> AllocResult { + let section = self + .find_section_by_addr(addr) + .ok_or(AllocError::NotFound)?; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + Ok(unsafe { (*section.meta.add(pfn)).flags }) } - /// Internal: free a block at `pfn` of the given `order`, merging buddies. - fn dealloc_order(&mut self, mut pfn: usize, mut order: usize) { - // Only the originally freed block's pages are newly freed; - // merged buddies were already counted as free. + fn dealloc_in_section(section: &mut BuddySection, mut pfn: usize, mut order: usize) { let freed_pages = 1usize << order; while order < MAX_ORDER { let buddy_pfn = pfn ^ (1 << order); - if buddy_pfn >= self.max_pages { + if buddy_pfn >= section.max_pages { break; } - let buddy = unsafe { &*self.meta.add(buddy_pfn) }; + let buddy = unsafe { &*section.meta.add(buddy_pfn) }; if buddy.flags != PageFlags::Free || buddy.order as usize != order { break; } - // Merge: remove buddy from its free list unsafe { - free_list_remove(self.meta, &mut self.free_lists, buddy_pfn as u32, order); + free_list_remove( + section.meta, + &mut section.free_lists, + buddy_pfn as u32, + order, + ); } pfn = pfn.min(buddy_pfn); order += 1; } unsafe { - let m = &mut *self.meta.add(pfn); + let m = &mut *section.meta.add(pfn); m.flags = PageFlags::Free; m.order = order as u8; - free_list_push(self.meta, &mut self.free_lists, pfn as u32, order); + free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); + } + section.free_pages += freed_pages; + } + + fn find_section_by_addr(&self, addr: usize) -> Option<&BuddySection> { + let mut section = self.sections_head; + while !section.is_null() { + let current = unsafe { &*section }; + if current.contains_heap_addr(addr) { + return Some(current); + } + section = current.next; + } + None + } + + fn find_section_by_addr_mut(&mut self, addr: usize) -> Option<&mut BuddySection> { + let mut section = self.sections_head; + while !section.is_null() { + let current = unsafe { &mut *section }; + if current.contains_heap_addr(addr) { + return Some(current); + } + section = current.next; } - self.free_pages += freed_pages; + None } } diff --git a/src/error.rs b/src/error.rs index cc2d849..e48693d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -5,21 +5,27 @@ use core::fmt; pub enum AllocError { /// Invalid size, alignment, or other input parameter. InvalidParam, + /// A region overlaps with an existing managed region. + MemoryOverlap, /// Not enough memory is available to satisfy the request. NoMemory, /// Attempted to deallocate memory that was not allocated. NotAllocated, /// The allocator has not been initialized. NotInitialized, + /// The requested address or entity was not found in any managed region. + NotFound, } impl fmt::Display for AllocError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::InvalidParam => write!(f, "invalid parameter"), + Self::MemoryOverlap => write!(f, "memory regions overlap"), Self::NoMemory => write!(f, "out of memory"), Self::NotAllocated => write!(f, "memory not allocated"), Self::NotInitialized => write!(f, "allocator not initialized"), + Self::NotFound => write!(f, "not found"), } } } diff --git a/src/global.rs b/src/global.rs index 47a8f29..a9697a4 100644 --- a/src/global.rs +++ b/src/global.rs @@ -8,14 +8,15 @@ use core::sync::atomic::{AtomicBool, Ordering}; use spin::Mutex as SpinMutex; -use crate::buddy::{BuddyAllocator, PageFlags}; +use crate::buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags}; use crate::error::{AllocError, AllocResult}; use crate::slab::page::{SlabPageHeader, SLAB_MAGIC}; use crate::slab::size_class::{SizeClass, SLAB_MAX_SIZE}; use crate::slab::{SlabAllocResult, SlabAllocator, SlabDeallocResult}; use crate::{align_up, OsImpl}; -struct RegionLayout { +struct InitialRegionLayout { + section_start: usize, meta_start: usize, meta_size: usize, buddy_meta_size: usize, @@ -35,42 +36,43 @@ pub struct GlobalAllocator { // SAFETY: All mutable state is behind SpinMutex or AtomicBool. // `per_cpu_slabs` is a raw pointer into the reserved metadata prefix of the -// caller-provided region (same lifetime model as BuddyAllocator::meta). +// caller-provided region. unsafe impl Sync for GlobalAllocator {} unsafe impl Send for GlobalAllocator {} impl GlobalAllocator { - // ------------------------------------------------------------------ - // Metadata sizing - // ------------------------------------------------------------------ - - /// Required alignment for the reserved metadata prefix. const fn metadata_align() -> usize { - let a1 = core::mem::align_of::(); - let a2 = core::mem::align_of::>>(); - if a1 > a2 { - a1 + let a1 = core::mem::align_of::(); + let a2 = core::mem::align_of::(); + let a3 = core::mem::align_of::>>(); + let m = if a1 > a2 { a1 } else { a2 }; + if m > a3 { + m } else { - a2 + a3 } } fn metadata_layout_for_pages(pages: usize, cpu_count: usize) -> Option<(usize, usize, usize)> { + let meta_offset = align_up( + core::mem::size_of::(), + core::mem::align_of::(), + ); let buddy_meta_size = pages.checked_mul(core::mem::size_of::())?; let slab_align = core::mem::align_of::>>(); - let slab_offset = align_up(buddy_meta_size, slab_align); + let slab_offset = align_up(meta_offset.checked_add(buddy_meta_size)?, slab_align); let slab_size = core::mem::size_of::>>().checked_mul(cpu_count)?; let meta_size = slab_offset.checked_add(slab_size)?; - Some((buddy_meta_size, slab_offset, meta_size)) + Some((meta_offset, buddy_meta_size, meta_size)) } fn available_heap_pages( region_end: usize, - meta_start: usize, + section_start: usize, meta_size: usize, ) -> Option { - let managed_heap_start = align_up(meta_start.checked_add(meta_size)?, PAGE_SIZE); + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE); if managed_heap_start > region_end { return Some(0); } @@ -79,36 +81,40 @@ impl GlobalAllocator { fn can_manage_pages( region_end: usize, - meta_start: usize, + section_start: usize, cpu_count: usize, pages: usize, ) -> bool { let Some((_, _, meta_size)) = Self::metadata_layout_for_pages(pages, cpu_count) else { return false; }; - let Some(available_pages) = Self::available_heap_pages(region_end, meta_start, meta_size) + let Some(available_pages) = + Self::available_heap_pages(region_end, section_start, meta_size) else { return false; }; available_pages >= pages } - fn compute_region_layout( + fn compute_initial_region_layout( region_start: usize, region_size: usize, cpu_count: usize, - ) -> Option { + ) -> Option { if cpu_count == 0 || region_size == 0 || !PAGE_SIZE.is_power_of_two() { return None; } let region_end = region_start.checked_add(region_size)?; - let meta_start = align_up(region_start, Self::metadata_align()); - if meta_start >= region_end { + let section_start = align_up(region_start, Self::metadata_align()); + if section_start >= region_end { return None; } - let heap_search_start = align_up(region_start, PAGE_SIZE); + let heap_search_start = align_up( + section_start.checked_add(core::mem::size_of::())?, + PAGE_SIZE, + ); let max_pages = if heap_search_start >= region_end { 0 } else { @@ -119,7 +125,7 @@ impl GlobalAllocator { let mut high = max_pages; while low < high { let mid = low + (high - low).div_ceil(2); - if Self::can_manage_pages(region_end, meta_start, cpu_count, mid) { + if Self::can_manage_pages(region_end, section_start, cpu_count, mid) { low = mid; } else { high = mid - 1; @@ -130,12 +136,18 @@ impl GlobalAllocator { return None; } - let (buddy_meta_size, slab_offset, meta_size) = + let (meta_offset, buddy_meta_size, meta_size) = Self::metadata_layout_for_pages(low, cpu_count)?; - let managed_heap_start = align_up(meta_start.checked_add(meta_size)?, PAGE_SIZE); + let meta_start = section_start.checked_add(meta_offset)?; + let slab_offset = align_up( + meta_offset.checked_add(buddy_meta_size)?, + core::mem::align_of::>>(), + ); + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE); let managed_heap_size = low.checked_mul(PAGE_SIZE)?; - Some(RegionLayout { + Some(InitialRegionLayout { + section_start, meta_start, meta_size, buddy_meta_size, @@ -145,10 +157,6 @@ impl GlobalAllocator { }) } - // ------------------------------------------------------------------ - // Construction / initialisation - // ------------------------------------------------------------------ - /// Create an uninitialised global allocator. pub const fn new() -> Self { Self { @@ -168,13 +176,7 @@ impl Default for GlobalAllocator { } impl GlobalAllocator { - /// Initialise the allocator. - /// - /// # Arguments - /// - `region` — total writable region to reserve for this allocator. - /// The allocator carves metadata from the region prefix and manages the remaining tail. - /// - `cpu_count` — number of CPUs (≥ 1). - /// - `os` — platform abstraction. + /// Initialise the allocator over the first region. /// /// # Safety /// - `region` must be writable and remain valid for the lifetime of this allocator. @@ -187,29 +189,31 @@ impl GlobalAllocator { ) -> AllocResult { let region_start = region.as_mut_ptr() as usize; let region_size = region.len(); - let layout = Self::compute_region_layout(region_start, region_size, cpu_count) + let layout = Self::compute_initial_region_layout(region_start, region_size, cpu_count) .ok_or(AllocError::InvalidParam)?; + let section_ptr = layout.section_start as *mut BuddySection; let meta_ptr = layout.meta_start as *mut u8; - let slab_ptr = meta_ptr.add(layout.slab_offset) as *mut SpinMutex>; + let slab_ptr = + (layout.section_start + layout.slab_offset) as *mut SpinMutex>; - // --- Init buddy --- let mut buddy = self.buddy.lock(); - buddy.init( + buddy.reset(Some(os)); + buddy.add_region_raw( + region_start, + region_size, + section_ptr, meta_ptr, layout.buddy_meta_size, layout.managed_heap_start, layout.managed_heap_size, - Some(os), )?; drop(buddy); - // --- Init per-CPU slabs --- for i in 0..cpu_count { let slot = slab_ptr.add(i); slot.write(SpinMutex::new(SlabAllocator::new())); } - // --- Write fields via interior mutability (self is &self for GlobalAlloc compat) --- let self_mut = self as *const Self as *mut Self; (*self_mut).per_cpu_slabs = slab_ptr; (*self_mut).cpu_count = cpu_count; @@ -217,11 +221,11 @@ impl GlobalAllocator { self.initialized.store(true, Ordering::Release); log::debug!( - "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, managed heap {:#x}+{:#x}", + "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, first heap {:#x}+{:#x}", cpu_count, region_start, region_size, - layout.meta_start, + layout.section_start, layout.meta_size, layout.managed_heap_start, layout.managed_heap_size, @@ -230,21 +234,44 @@ impl GlobalAllocator { Ok(()) } - /// Start address of the heap range actually managed by the buddy allocator. - pub fn managed_heap_start(&self) -> usize { - self.buddy.lock().heap_start() + /// Add a new managed region after [`init`](Self::init). + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - The region must not overlap any already managed region. + pub unsafe fn add_region(&self, region: &mut [u8]) -> AllocResult { + if !self.initialized.load(Ordering::Acquire) { + return Err(AllocError::NotInitialized); + } + self.buddy.lock().add_region(region) + } + + /// Number of managed sections. + pub fn managed_section_count(&self) -> usize { + self.buddy.lock().section_count() } - /// Size in bytes of the heap range actually managed by the buddy allocator. - pub fn managed_heap_size(&self) -> usize { - self.buddy.lock().heap_size() + /// Read-only summary for a managed section. + pub fn managed_section(&self, index: usize) -> Option { + self.buddy.lock().section(index) + } + + /// Total managed heap bytes across all sections. + /// + /// This excludes region-prefix metadata such as `BuddySection`, `PageMeta[]`, + /// and per-CPU slab slots. + pub fn managed_bytes(&self) -> usize { + self.buddy.lock().managed_bytes() } - // ------------------------------------------------------------------ - // Public page-level API (forwarded to buddy) - // ------------------------------------------------------------------ + /// Allocated backend bytes across all sections. + /// + /// This is page-level occupancy, not the exact sum of requested layout sizes. + pub fn allocated_bytes(&self) -> usize { + self.buddy.lock().allocated_bytes() + } - /// Allocate contiguous pages. Returns the virtual start address. + /// Allocate contiguous pages. Returns the virtual start address. pub fn alloc_pages(&self, count: usize, align: usize) -> AllocResult { self.buddy.lock().alloc_pages(count, align) } @@ -259,11 +286,7 @@ impl GlobalAllocator { self.buddy.lock().alloc_pages_lowmem(count, align) } - // ------------------------------------------------------------------ - // Object-level API (slab or buddy fallback) - // ------------------------------------------------------------------ - - /// Allocate memory for `layout`. Returns a pointer on success. + /// Allocate memory for `layout`. Returns a pointer on success. pub fn alloc(&self, layout: Layout) -> AllocResult> { if !self.initialized.load(Ordering::Acquire) { return Err(AllocError::NotInitialized); @@ -288,10 +311,6 @@ impl GlobalAllocator { } } - // ------------------------------------------------------------------ - // Internal: slab path - // ------------------------------------------------------------------ - #[inline] fn is_slab_eligible(&self, layout: &Layout) -> bool { layout.size() <= SLAB_MAX_SIZE && layout.align() <= SLAB_MAX_SIZE @@ -308,12 +327,11 @@ impl GlobalAllocator { match slab.alloc(layout)? { SlabAllocResult::Allocated(ptr) => Ok(ptr), SlabAllocResult::NeedsSlab { size_class, pages } => { - drop(slab); // release slab lock before locking buddy + drop(slab); let bytes = pages * PAGE_SIZE; let addr = self.buddy.lock().alloc_pages(pages, bytes)?; - // Tag the page as SLAB in buddy metadata. unsafe { - self.buddy.lock().set_page_flags(addr, PageFlags::Slab); + self.buddy.lock().set_page_flags(addr, PageFlags::Slab)?; } let mut slab = slab_lock.lock(); slab.add_slab(size_class, addr, bytes, cpu as u16); @@ -338,7 +356,6 @@ impl GlobalAllocator { let current_cpu = os.current_cpu_idx(); if owner_cpu == current_cpu { - // Local free path (under lock). let slab_lock = &*self.per_cpu_slabs.add(current_cpu); let mut slab = slab_lock.lock(); match slab.dealloc(ptr, layout) { @@ -349,20 +366,14 @@ impl GlobalAllocator { } } } else { - // Remote free path (lock-free CAS). hdr.remote_free(ptr.as_ptr() as usize); } } - // ------------------------------------------------------------------ - // Internal: large allocation (buddy pages) - // ------------------------------------------------------------------ - fn large_alloc(&self, layout: Layout) -> AllocResult> { let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE; let align = layout.align().max(PAGE_SIZE); let addr = self.buddy.lock().alloc_pages(pages, align)?; - // SAFETY: buddy returns non-null, page-aligned addresses. Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) }) } @@ -374,10 +385,6 @@ impl GlobalAllocator { } } -// --------------------------------------------------------------------------- -// GlobalAlloc implementation -// --------------------------------------------------------------------------- - unsafe impl GlobalAlloc for GlobalAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { match self.alloc(layout) { diff --git a/src/lib.rs b/src/lib.rs index 98d8b5c..5bb26b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ mod error; pub use error::{AllocError, AllocResult}; pub mod buddy; -pub use buddy::BuddyAllocator; +pub use buddy::{BuddyAllocator, ManagedSection}; pub mod slab; pub use slab::{SizeClass, SlabAllocResult, SlabAllocator, SlabDeallocResult}; diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index d9319aa..cbc0447 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -19,8 +19,9 @@ fn test_lowmem_basic() { let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); let allocator = GlobalAllocator::::new(); init_allocator(&allocator, &mut region); - let managed_start = allocator.managed_heap_start(); - let managed_end = managed_start + allocator.managed_heap_size(); + let section = allocator.managed_section(0).unwrap(); + let managed_start = section.start; + let managed_end = managed_start + section.size; let addr1 = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr2 = allocator.alloc_pages_lowmem(4, PAGE_SIZE).unwrap(); @@ -41,7 +42,10 @@ fn test_lowmem_aligned() { init_allocator(&allocator, &mut region); let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); - assert_eq!((addr - allocator.managed_heap_start()) % (2 * PAGE_SIZE), 0); + assert_eq!( + (addr - allocator.managed_section(0).unwrap().start) % (2 * PAGE_SIZE), + 0 + ); allocator.dealloc_pages(addr, 1); } @@ -54,8 +58,8 @@ fn test_lowmem_vs_normal() { let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); - assert!(addr_low >= allocator.managed_heap_start()); - assert!(addr_normal >= allocator.managed_heap_start()); + assert!(addr_low >= allocator.managed_section(0).unwrap().start); + assert!(addr_normal >= allocator.managed_section(0).unwrap().start); allocator.dealloc_pages(addr_low, 1); allocator.dealloc_pages(addr_normal, 1); diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 656ae68..bf366ae 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -4,8 +4,8 @@ extern crate buddy_slab_allocator; mod common; use buddy_slab_allocator::{ - slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, SizeClass, SlabAllocResult, - SlabAllocator, SlabDeallocResult, + slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, ManagedSection, SizeClass, + SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; use core::ptr::NonNull; @@ -19,8 +19,21 @@ use common::{ const PAGE_SIZE: usize = 0x1000; const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16 MiB -fn align_up(value: usize, align: usize) -> usize { - (value + align - 1) & !(align - 1) +fn buddy_region_size(heap_size: usize) -> usize { + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4 +} + +fn init_buddy( + buddy: &mut BuddyAllocator, + region: &mut HostRegion, + os: Option<&'static dyn buddy_slab_allocator::OsImpl>, +) -> ManagedSection { + unsafe { buddy.init(region.as_mut_slice(), os).unwrap() }; + buddy.section(0).unwrap() +} + +fn primary_section(allocator: &GlobalAllocator) -> ManagedSection { + allocator.managed_section(0).unwrap() } // ====================================================================== @@ -29,26 +42,12 @@ fn align_up(value: usize, align: usize) -> usize { #[test] fn buddy_basic_alloc_dealloc() { - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap_addr, - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let section = init_buddy(&mut buddy, &mut region, None); let addr1 = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); - assert!(addr1 >= heap_addr && addr1 < heap_addr + TEST_HEAP_SIZE); + assert!(addr1 >= section.start && addr1 < section.start + section.size); assert_eq!(addr1 % PAGE_SIZE, 0); let addr4 = buddy.alloc_pages(4, PAGE_SIZE).unwrap(); @@ -63,30 +62,15 @@ fn buddy_basic_alloc_dealloc() { #[test] fn buddy_alignment() { // Heap must be aligned to the highest alignment we test (PAGE_SIZE * 4) - // so that PFN-based alignment maps to absolute address alignment. - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE * 4); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap_addr, - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let section = init_buddy(&mut buddy, &mut region, None); let addr2 = buddy.alloc_pages(1, PAGE_SIZE * 2).unwrap(); - assert_eq!(addr2 % (PAGE_SIZE * 2), 0); + assert_eq!((addr2 - section.start) % (PAGE_SIZE * 2), 0); let addr4 = buddy.alloc_pages(1, PAGE_SIZE * 4).unwrap(); - assert_eq!(addr4 % (PAGE_SIZE * 4), 0); + assert_eq!((addr4 - section.start) % (PAGE_SIZE * 4), 0); buddy.dealloc_pages(addr2, 1); buddy.dealloc_pages(addr4, 1); @@ -95,17 +79,9 @@ fn buddy_alignment() { #[test] fn buddy_aligned_alloc_dealloc_uses_recorded_order() { let heap_size = 64 * PAGE_SIZE; - let heap = HostRegion::new(heap_size, PAGE_SIZE * 16); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE * 16); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let free_before = buddy.free_pages(); let addr = buddy.alloc_pages(4, PAGE_SIZE * 16).unwrap(); @@ -116,17 +92,9 @@ fn buddy_aligned_alloc_dealloc_uses_recorded_order() { #[test] fn buddy_exhaust_and_recover() { let heap_size = 64 * PAGE_SIZE; // Small heap - let heap = HostRegion::new(heap_size, PAGE_SIZE); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let mut addrs = Vec::new(); while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { @@ -156,17 +124,9 @@ fn buddy_exhaust_and_recover() { #[test] fn buddy_merge_coalescing() { let heap_size = 16 * PAGE_SIZE; - let heap = HostRegion::new(heap_size, PAGE_SIZE); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(heap_size); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init(meta.as_mut_ptr(), meta.len(), heap_addr, heap_size, None) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let initial_free = buddy.free_pages(); @@ -183,23 +143,15 @@ fn buddy_merge_coalescing() { #[test] fn buddy_fragmentation_blocks_high_order_then_recovers() { let heap_size = 32 * PAGE_SIZE; - let heap = HostRegion::new(heap_size, PAGE_SIZE); - let mut meta = HostRegion::new( - BuddyAllocator::::required_meta_size(heap_size), - 16, - ); + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init(meta.as_mut_ptr(), meta.len(), heap.addr(), heap_size, None) - .unwrap(); - } + let section = init_buddy(&mut buddy, &mut region, None); let mut addrs = Vec::new(); while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { addrs.push(addr); } - assert_eq!(addrs.len(), heap_size / PAGE_SIZE); + assert_eq!(addrs.len(), section.total_pages); for &addr in addrs.iter().step_by(2) { buddy.dealloc_pages(addr, 1); @@ -212,23 +164,15 @@ fn buddy_fragmentation_blocks_high_order_then_recovers() { let addr = buddy.alloc_pages(8, PAGE_SIZE).unwrap(); buddy.dealloc_pages(addr, 8); - assert_eq!(buddy.free_pages(), heap_size / PAGE_SIZE); + assert_eq!(buddy.free_pages(), section.total_pages); } #[test] fn buddy_high_order_full_cycle_restores_free_pages() { let heap_size = 256 * PAGE_SIZE; - let heap = HostRegion::new(heap_size, PAGE_SIZE * 16); - let mut meta = HostRegion::new( - BuddyAllocator::::required_meta_size(heap_size), - 16, - ); + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE * 16); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init(meta.as_mut_ptr(), meta.len(), heap.addr(), heap_size, None) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let initial_free = buddy.free_pages(); let requests = [ @@ -253,29 +197,175 @@ fn buddy_high_order_full_cycle_restores_free_pages() { assert_eq!(buddy.free_pages(), initial_free); } +#[test] +fn buddy_add_region_enables_second_section_allocation() { + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first, None); + + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + assert_eq!(buddy.free_pages(), 0); + + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + assert_eq!(buddy.section_count(), 2); + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + assert!(addr < first_section.start || addr >= first_section.start + first_section.size); +} + +#[test] +fn buddy_add_region_overlap_rejected() { + let mut region = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region, None); + + let overlap = unsafe { region.subslice(1, region.len() - 1) }; + let err = unsafe { buddy.add_region(overlap) }.unwrap_err(); + assert_eq!(err, AllocError::MemoryOverlap); +} + +#[test] +fn buddy_alloc_pages_first_fit_by_registration_order() { + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first, None); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= first_section.start && addr < first_section.start + first_section.size); +} + +#[test] +fn buddy_lowmem_scans_across_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first, Some(&LOWMEM_OS)); + + while buddy.alloc_pages_lowmem(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); +} + +#[test] +fn buddy_dealloc_pages_finds_correct_section() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first, None); + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let baseline = buddy.free_pages(); + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + buddy.dealloc_pages(addr, 1); + + assert_eq!(buddy.free_pages(), baseline); +} + +#[test] +fn buddy_total_and_free_pages_are_aggregated() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first, None); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + assert_eq!( + buddy.total_pages(), + first_section.total_pages + second_section.total_pages + ); + assert_eq!( + buddy.free_pages(), + first_section.free_pages + second_section.free_pages + ); +} + +#[test] +fn buddy_managed_bytes_matches_all_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first, None); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + assert_eq!( + buddy.managed_bytes(), + first_section.size + second_section.size + ); +} + +#[test] +fn buddy_allocated_bytes_changes_with_page_alloc_free() { + let mut region = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region, None); + + assert_eq!(buddy.allocated_bytes(), 0); + + let a = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert_eq!(buddy.allocated_bytes(), PAGE_SIZE); + + let b = buddy.alloc_pages(4, PAGE_SIZE).unwrap(); + assert_eq!(buddy.allocated_bytes(), 5 * PAGE_SIZE); + + buddy.dealloc_pages(a, 1); + assert_eq!(buddy.allocated_bytes(), 4 * PAGE_SIZE); + + buddy.dealloc_pages(b, 4); + assert_eq!(buddy.allocated_bytes(), 0); +} + +#[test] +fn buddy_allocated_bytes_zero_when_all_free() { + let mut region = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region, None); + + assert_eq!(buddy.allocated_bytes(), 0); +} + +#[test] +fn buddy_allocated_bytes_aggregates_across_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first, None); + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + + let addr = buddy.alloc_pages(8, PAGE_SIZE).unwrap(); + assert_eq!( + buddy.allocated_bytes(), + buddy.managed_bytes() - buddy.free_pages() * PAGE_SIZE + ); + buddy.dealloc_pages(addr, 8); + assert_eq!( + buddy.allocated_bytes(), + buddy.managed_bytes() - buddy.free_pages() * PAGE_SIZE + ); +} + // ====================================================================== // Slab allocator (standalone) tests // ====================================================================== #[test] fn slab_basic() { - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap_addr, - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let mut slab = SlabAllocator::::new(); @@ -304,23 +394,9 @@ fn slab_basic() { #[test] fn slab_many_objects() { - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE * 4); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap_addr, - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let mut slab = SlabAllocator::::new(); let layout = Layout::from_size_align(32, 8).unwrap(); @@ -350,23 +426,9 @@ fn slab_many_objects() { #[test] fn slab_all_size_classes() { - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let heap_addr = heap.addr(); - let meta_size = BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE); - let mut meta = HostRegion::new(meta_size, 16); - + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE * 4); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap_addr, - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let mut slab = SlabAllocator::::new(); let mut allocations = Vec::new(); @@ -396,23 +458,9 @@ fn slab_all_size_classes() { #[test] fn slab_reuses_freed_objects_same_size_class() { - let heap = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); - let mut meta = HostRegion::new( - BuddyAllocator::::required_meta_size(TEST_HEAP_SIZE), - 16, - ); + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE * 4); let mut buddy = BuddyAllocator::::new(); - unsafe { - buddy - .init( - meta.as_mut_ptr(), - meta.len(), - heap.addr(), - TEST_HEAP_SIZE, - None, - ) - .unwrap(); - } + let _section = init_buddy(&mut buddy, &mut region, None); let mut slab = SlabAllocator::::new(); let layout = Layout::from_size_align(64, 8).unwrap(); @@ -495,8 +543,9 @@ fn global_page_alloc() { let allocator = GlobalAllocator::::new(); init_global(&allocator, &mut region, 1); - let managed_start = allocator.managed_heap_start(); - let managed_end = managed_start + allocator.managed_heap_size(); + let section = primary_section(&allocator); + let managed_start = section.start; + let managed_end = managed_start + section.size; let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); assert!(managed_start > region_addr); @@ -792,7 +841,7 @@ fn global_lowmem_pages() { init_global_allocator(&allocator, &mut region, 1, &LOWMEM_OS); let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); - assert!(addr >= allocator.managed_heap_start()); + assert!(addr >= primary_section(&allocator).start); allocator.dealloc_pages(addr, 1); } @@ -805,8 +854,9 @@ fn global_unaligned_region_start() { let unaligned_region = unsafe { region.subslice(1, region_size) }; unsafe { allocator.init(unaligned_region, 1, &TEST_OS).unwrap() }; - let managed_start = allocator.managed_heap_start(); - let managed_end = managed_start + allocator.managed_heap_size(); + let section = primary_section(&allocator); + let managed_start = section.start; + let managed_end = managed_start + section.size; assert_eq!(managed_start % PAGE_SIZE, 0); assert!(managed_start >= region_start); @@ -819,14 +869,147 @@ fn global_unaligned_region_start() { #[test] fn global_rejects_region_without_one_managed_page() { - let buddy_meta_size = BuddyAllocator::::required_meta_size(PAGE_SIZE); - let slab_align = core::mem::align_of::>>(); - let slab_offset = align_up(buddy_meta_size, slab_align); - let slab_size = core::mem::size_of::>>(); - let region_size = PAGE_SIZE + slab_offset + slab_size - 1; + let region_size = PAGE_SIZE - 1; let mut region = HostRegion::new(region_size, PAGE_SIZE); let allocator = GlobalAllocator::::new(); let err = unsafe { allocator.init(region.as_mut_slice(), 1, &TEST_OS) }.unwrap_err(); assert_eq!(err, AllocError::InvalidParam); } + +#[test] +fn global_add_region_after_init_expands_capacity() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(512 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + + let before = count_free_pages(&allocator); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let after = count_free_pages(&allocator); + + assert!(after > before); + assert_eq!(allocator.managed_section_count(), 2); +} + +#[test] +fn global_add_region_supports_discontiguous_regions() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(512 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + + while allocator.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let second_section = allocator.managed_section(1).unwrap(); + + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); +} + +#[test] +fn global_large_alloc_can_come_from_added_region() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + + while allocator.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let second_section = allocator.managed_section(1).unwrap(); + + let layout = Layout::from_size_align(8 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + let addr = ptr.as_ptr() as usize; + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_managed_section_queries_report_all_sections() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(512 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + + assert_eq!(allocator.managed_section_count(), 2); + let first_section = allocator.managed_section(0).unwrap(); + let second_section = allocator.managed_section(1).unwrap(); + assert!(first_section.size > 0); + assert!(second_section.size > 0); +} + +#[test] +fn global_add_region_overlap_rejected() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + + let overlap = unsafe { first.subslice(1, first.len() - 1) }; + let err = unsafe { allocator.add_region(overlap) }.unwrap_err(); + assert_eq!(err, AllocError::MemoryOverlap); +} + +#[test] +fn global_managed_bytes_matches_all_sections() { + let mut first = HostRegion::new(256 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(512 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, 1); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + + let expected = (0..allocator.managed_section_count()) + .map(|i| allocator.managed_section(i).unwrap().size) + .sum::(); + assert_eq!(allocator.managed_bytes(), expected); +} + +#[test] +fn global_allocated_bytes_changes_with_large_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, 1); + + assert_eq!(allocator.allocated_bytes(), 0); + + let layout = Layout::from_size_align(3 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert_eq!(allocator.allocated_bytes(), 4 * PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; + assert_eq!(allocator.allocated_bytes(), 0); +} + +#[test] +fn global_allocated_bytes_reflects_slab_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, 1); + + assert_eq!(allocator.allocated_bytes(), 0); + + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert!(allocator.allocated_bytes() >= PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_allocated_bytes_not_zero_until_cached_empty_slab_released() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut region, 1); + + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + let allocated_after_refill = allocator.allocated_bytes(); + assert!(allocated_after_refill >= PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; + + // One empty slab may remain cached, so backend occupancy need not drop to zero. + assert!(allocator.allocated_bytes() <= allocated_after_refill); + assert!(allocator.allocated_bytes() >= PAGE_SIZE); +} diff --git a/tests/stress_test.rs b/tests/stress_test.rs index 84c0fc7..c853002 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -421,3 +421,61 @@ fn stress_multithread_exhaustion_recovery() { assert!(recovered.load(Ordering::Relaxed)); assert_eq!(count_free_pages(allocator), baseline); } + +#[test] +#[ignore = "stress test"] +fn stress_add_region_then_multithread_alloc_free() { + let mut first = HostRegion::new(2 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut third = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + init_global(&allocator, &mut first, WORKERS, &TEST_OS); + unsafe { + allocator.add_region(second.as_mut_slice()).unwrap(); + allocator.add_region(third.as_mut_slice()).unwrap(); + } + assert_eq!(allocator.managed_section_count(), 3); + + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x3000 + cpu as u64); + let mut live: Vec<(usize, Layout)> = Vec::new(); + for _ in 0..5_000 { + if live.is_empty() || rng.random_bool(0.65) { + let layout = if rng.random_bool(0.75) { + let size: usize = rng.random_range(8..=2048); + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let pages = [1usize, 2, 4, 8][rng.random_range(0..4)]; + Layout::from_size_align(pages * PAGE_SIZE, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + live.push((ptr.as_ptr() as usize, layout)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, layout) = live.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + } + + for (addr, layout) in live { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + }); + } + }); + + assert_eq!(allocator.managed_section_count(), 3); + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &SizeClass::ALL); +} From a33a996820e7d3aeaf2393191e2c73a1427e3913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 10:56:34 +0800 Subject: [PATCH 14/19] refactor: streamline region initialization by introducing SectionInitSpec for better clarity and maintainability --- benches/common.rs | 77 +++++++++++------------------------------------ src/buddy/mod.rs | 74 ++++++++++++++++++++++++--------------------- src/global.rs | 12 ++++---- 3 files changed, 63 insertions(+), 100 deletions(-) diff --git a/benches/common.rs b/benches/common.rs index 1d0d925..489c1b2 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -1,9 +1,6 @@ #![allow(dead_code)] -use buddy_slab_allocator::{ - buddy::PageMeta, BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, - SlabDeallocResult, -}; +use buddy_slab_allocator::{BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, SlabDeallocResult}; use core::alloc::Layout; use rand::{rngs::StdRng, SeedableRng}; use std::alloc::{alloc, dealloc}; @@ -32,39 +29,13 @@ impl HostRegion { pub fn addr(&self) -> usize { self.ptr as usize } -} - -impl Drop for HostRegion { - fn drop(&mut self) { - unsafe { dealloc(self.ptr, self.layout) }; - } -} - -pub struct MetadataBuf { - ptr: *mut u8, - layout: Layout, -} - -impl MetadataBuf { - pub fn for_heap(heap_size: usize) -> Self { - let size = BuddyAllocator::::required_meta_size(heap_size); - let align = core::mem::align_of::().max(8); - let layout = Layout::from_size_align(size, align).unwrap(); - let ptr = unsafe { alloc(layout) }; - assert!(!ptr.is_null(), "failed to allocate metadata"); - Self { ptr, layout } - } - pub fn ptr(&self) -> *mut u8 { - self.ptr - } - - pub fn size(&self) -> usize { - self.layout.size() + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } } } -impl Drop for MetadataBuf { +impl Drop for HostRegion { fn drop(&mut self) { unsafe { dealloc(self.ptr, self.layout) }; } @@ -94,10 +65,6 @@ impl OsImpl for MockOs { fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr } - - fn phys_to_virt(&self, paddr: usize) -> usize { - paddr - } } pub static MOCK_OS: MockOs = MockOs::new(); @@ -107,49 +74,43 @@ pub fn seeded_rng() -> StdRng { } pub struct BuddyHarness { - _heap: HostRegion, - _meta: MetadataBuf, + _region: HostRegion, pub allocator: BuddyAllocator, } impl BuddyHarness { pub fn new(heap_size: usize) -> Self { - let heap = HostRegion::new(heap_size); - let meta = MetadataBuf::for_heap(heap_size); + let region_size = + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4; + let mut region = HostRegion::new(region_size); let mut allocator = BuddyAllocator::::new(); unsafe { - allocator - .init(meta.ptr(), meta.size(), heap.addr(), heap_size, None) - .unwrap(); + allocator.init(region.as_mut_slice(), None).unwrap(); } Self { - _heap: heap, - _meta: meta, + _region: region, allocator, } } } pub struct SlabHarness { - _heap: HostRegion, - _meta: MetadataBuf, + _region: HostRegion, buddy: BuddyAllocator, slab: SlabAllocator, } impl SlabHarness { pub fn new(heap_size: usize) -> Self { - let heap = HostRegion::new(heap_size); - let meta = MetadataBuf::for_heap(heap_size); + let region_size = + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4; + let mut region = HostRegion::new(region_size); let mut buddy = BuddyAllocator::::new(); unsafe { - buddy - .init(meta.ptr(), meta.size(), heap.addr(), heap_size, None) - .unwrap(); + buddy.init(region.as_mut_slice(), None).unwrap(); } Self { - _heap: heap, - _meta: meta, + _region: region, buddy, slab: SlabAllocator::new(), } @@ -185,13 +146,11 @@ pub struct GlobalHarness { impl GlobalHarness { pub fn new(region_size: usize, cpu_count: usize) -> Self { - let region = HostRegion::new(region_size); + let mut region = HostRegion::new(region_size); let allocator = GlobalAllocator::::new(); MOCK_OS.set_cpu(0); unsafe { - allocator - .init(region.addr(), region_size, cpu_count, &MOCK_OS) - .unwrap(); + allocator.init(region.as_mut_slice(), cpu_count, &MOCK_OS).unwrap(); } Self { _region: region, diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index ea3d5ba..97c0cd1 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -28,6 +28,16 @@ struct RegionLayout { managed_heap_size: usize, } +pub(crate) struct SectionInitSpec { + pub(crate) region_start: usize, + pub(crate) region_size: usize, + pub(crate) section_ptr: *mut BuddySection, + pub(crate) meta_ptr: *mut u8, + pub(crate) meta_size: usize, + pub(crate) heap_start: usize, + pub(crate) heap_size: usize, +} + /// Public read-only summary of a managed section. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ManagedSection { @@ -305,32 +315,26 @@ impl BuddyAllocator { let region_size = region.len(); let layout = BuddySection::compute_region_layout::(region_start, region_size) .ok_or(AllocError::InvalidParam)?; - self.add_region_raw( + self.add_region_raw(SectionInitSpec { region_start, region_size, - layout.section_start as *mut BuddySection, - layout.meta_start as *mut u8, - Self::required_meta_size(layout.managed_heap_size), - layout.managed_heap_start, - layout.managed_heap_size, - ) + section_ptr: layout.section_start as *mut BuddySection, + meta_ptr: layout.meta_start as *mut u8, + meta_size: Self::required_meta_size(layout.managed_heap_size), + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + }) } - pub(crate) unsafe fn add_region_raw( - &mut self, - region_start: usize, - region_size: usize, - section_ptr: *mut BuddySection, - meta_ptr: *mut u8, - meta_size: usize, - heap_start: usize, - heap_size: usize, - ) -> AllocResult { - let region_end = region_start + pub(crate) unsafe fn add_region_raw(&mut self, spec: SectionInitSpec) -> AllocResult { + let region_size = spec.region_size; + let region_end = spec + .region_start .checked_add(region_size) .ok_or(AllocError::InvalidParam)?; - let heap_end = heap_start - .checked_add(heap_size) + let heap_end = spec + .heap_start + .checked_add(spec.heap_size) .ok_or(AllocError::InvalidParam)?; if heap_end > region_end { return Err(AllocError::InvalidParam); @@ -343,37 +347,37 @@ impl BuddyAllocator { .region_start .checked_add(existing.region_size) .ok_or(AllocError::InvalidParam)?; - if region_start < existing_end && existing.region_start < region_end { + if spec.region_start < existing_end && existing.region_start < region_end { return Err(AllocError::MemoryOverlap); } section = existing.next; } BuddySection::init_at::( - section_ptr, - region_start, - region_size, - meta_ptr, - meta_size, - heap_start, - heap_size, + spec.section_ptr, + spec.region_start, + spec.region_size, + spec.meta_ptr, + spec.meta_size, + spec.heap_start, + spec.heap_size, )?; if self.sections_head.is_null() { - self.sections_head = section_ptr; + self.sections_head = spec.section_ptr; } else { - (*self.sections_tail).next = section_ptr; + (*self.sections_tail).next = spec.section_ptr; } - self.sections_tail = section_ptr; + self.sections_tail = spec.section_ptr; self.section_count += 1; log::debug!( "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages", - region_start, - region_size, - heap_start, + spec.region_start, + spec.region_size, + spec.heap_start, heap_end, - heap_size / PAGE_SIZE, + spec.heap_size / PAGE_SIZE, ); Ok(()) diff --git a/src/global.rs b/src/global.rs index a9697a4..64095b7 100644 --- a/src/global.rs +++ b/src/global.rs @@ -8,7 +8,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use spin::Mutex as SpinMutex; -use crate::buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags}; +use crate::buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags, SectionInitSpec}; use crate::error::{AllocError, AllocResult}; use crate::slab::page::{SlabPageHeader, SLAB_MAGIC}; use crate::slab::size_class::{SizeClass, SLAB_MAX_SIZE}; @@ -198,15 +198,15 @@ impl GlobalAllocator { let mut buddy = self.buddy.lock(); buddy.reset(Some(os)); - buddy.add_region_raw( + buddy.add_region_raw(SectionInitSpec { region_start, region_size, section_ptr, meta_ptr, - layout.buddy_meta_size, - layout.managed_heap_start, - layout.managed_heap_size, - )?; + meta_size: layout.buddy_meta_size, + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + })?; drop(buddy); for i in 0..cpu_count { From 3827fe4c9cd8be55449b538755bc2702fd25643f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 10:58:11 +0800 Subject: [PATCH 15/19] style: format code for better readability in common.rs --- benches/common.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/benches/common.rs b/benches/common.rs index 489c1b2..408c0e2 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -1,6 +1,8 @@ #![allow(dead_code)] -use buddy_slab_allocator::{BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, SlabDeallocResult}; +use buddy_slab_allocator::{ + BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, SlabDeallocResult, +}; use core::alloc::Layout; use rand::{rngs::StdRng, SeedableRng}; use std::alloc::{alloc, dealloc}; @@ -150,7 +152,9 @@ impl GlobalHarness { let allocator = GlobalAllocator::::new(); MOCK_OS.set_cpu(0); unsafe { - allocator.init(region.as_mut_slice(), cpu_count, &MOCK_OS).unwrap(); + allocator + .init(region.as_mut_slice(), cpu_count, &MOCK_OS) + .unwrap(); } Self { _region: region, From 4714c90bd922d8c4a8f043906f447b705f07fae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 11:03:15 +0800 Subject: [PATCH 16/19] chore: update changelog for version 0.2.1 and modify Cargo.toml for version bump to 0.3.0 refactor: improve import order and formatting across benchmark and test files --- CHANGELOG.md | 23 +++++++++++++++++++++++ Cargo.toml | 4 ++-- benches/buddy_allocator.rs | 4 ++-- benches/common.rs | 2 +- benches/global_allocator.rs | 2 +- benches/slab_allocator.rs | 4 ++-- src/buddy/mod.rs | 4 ++-- src/global.rs | 12 ++++-------- src/slab/size_class.rs | 6 +----- tests/common/mod.rs | 4 ++-- tests/dma32_pages_test.rs | 2 +- tests/integration_test.rs | 8 ++++---- tests/stress_test.rs | 4 ++-- 13 files changed, 47 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0b51d..f9500db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Comprehensive edge case coverage ## [Unreleased]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...HEAD + +## [0.2.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.2.1) - 2026-04-09 + +### Added + +- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation +- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator +- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic +- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples +- enhance logging support by integrating log crate and updating documentation + +### Other + +- format code for better readability in common.rs +- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability +- Refactor GlobalAllocator to support multiple managed sections +- Refactor stress tests for allocator stability +- update allocator initialization to use a mutable slice instead of separate start and size parameters +- Refactor slab allocator benchmarks and improve global allocator initialization +- Refactor integration and stress tests for buddy-slab-allocator +- Refactor integration and stress tests for allocator +- Refactor benchmarks and remove stability tests +- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10 ## [0.2.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.1...v0.2.0 ## [0.1.1]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.0...v0.1.1 ## [0.1.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/releases/tag/v0.1.0 diff --git a/Cargo.toml b/Cargo.toml index 5aeccfb..1d0725e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,13 @@ autobenches = false categories = ["memory-management", "no-std", "embedded"] description = "Memory allocator with Buddy and Slab allocation" documentation = "https://docs.rs/buddy-slab-allocator" -edition = "2021" +edition = "2024" keywords = ["buddy", "slab", "allocator"] license = "Apache-2.0" name = "buddy-slab-allocator" readme = "README.md" repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator" -version = "0.2.0" +version = "0.3.0" [dependencies] log = "0.4" diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs index 19ffe5b..82c745b 100644 --- a/benches/buddy_allocator.rs +++ b/benches/buddy_allocator.rs @@ -3,9 +3,9 @@ mod common; use common::{ - seeded_rng, BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE, + BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE, seeded_rng, }; -use divan::{black_box, Bencher}; +use divan::{Bencher, black_box}; use rand::RngExt; fn main() { diff --git a/benches/common.rs b/benches/common.rs index 408c0e2..075f203 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -4,7 +4,7 @@ use buddy_slab_allocator::{ BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, SlabDeallocResult, }; use core::alloc::Layout; -use rand::{rngs::StdRng, SeedableRng}; +use rand::{SeedableRng, rngs::StdRng}; use std::alloc::{alloc, dealloc}; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs index 42f6109..a639ff8 100644 --- a/benches/global_allocator.rs +++ b/benches/global_allocator.rs @@ -4,7 +4,7 @@ mod common; use common::{GlobalHarness, HEAP_SIZE, MOCK_OS, OPERATIONS_PER_BATCH, PAGE_SIZE}; use core::alloc::Layout; -use divan::{black_box, Bencher}; +use divan::{Bencher, black_box}; fn main() { divan::main(); diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs index 773f45c..9197103 100644 --- a/benches/slab_allocator.rs +++ b/benches/slab_allocator.rs @@ -2,9 +2,9 @@ mod common; -use common::{SlabHarness, HEAP_SIZE, OPERATIONS_PER_BATCH}; +use common::{HEAP_SIZE, OPERATIONS_PER_BATCH, SlabHarness}; use core::alloc::Layout; -use divan::{black_box, Bencher}; +use divan::{Bencher, black_box}; fn main() { divan::main(); diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index 97c0cd1..67eafc0 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -7,12 +7,12 @@ pub mod page_meta; -pub use page_meta::{PageFlags, PageMeta, PFN_NONE}; +pub use page_meta::{PFN_NONE, PageFlags, PageMeta}; use core::ptr; use crate::error::{AllocError, AllocResult}; -use crate::{align_up, is_aligned, OsImpl}; +use crate::{OsImpl, align_up, is_aligned}; use page_meta::{free_list_pop, free_list_push, free_list_remove}; /// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks. diff --git a/src/global.rs b/src/global.rs index 64095b7..159a2ee 100644 --- a/src/global.rs +++ b/src/global.rs @@ -10,10 +10,10 @@ use spin::Mutex as SpinMutex; use crate::buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags, SectionInitSpec}; use crate::error::{AllocError, AllocResult}; -use crate::slab::page::{SlabPageHeader, SLAB_MAGIC}; -use crate::slab::size_class::{SizeClass, SLAB_MAX_SIZE}; +use crate::slab::page::{SLAB_MAGIC, SlabPageHeader}; +use crate::slab::size_class::{SLAB_MAX_SIZE, SizeClass}; use crate::slab::{SlabAllocResult, SlabAllocator, SlabDeallocResult}; -use crate::{align_up, OsImpl}; +use crate::{OsImpl, align_up}; struct InitialRegionLayout { section_start: usize, @@ -46,11 +46,7 @@ impl GlobalAllocator { let a2 = core::mem::align_of::(); let a3 = core::mem::align_of::>>(); let m = if a1 > a2 { a1 } else { a2 }; - if m > a3 { - m - } else { - a3 - } + if m > a3 { m } else { a3 } } fn metadata_layout_for_pages(pages: usize, cpu_count: usize) -> Option<(usize, usize, usize)> { diff --git a/src/slab/size_class.rs b/src/slab/size_class.rs index b3d5015..423c42c 100644 --- a/src/slab/size_class.rs +++ b/src/slab/size_class.rs @@ -85,11 +85,7 @@ impl SizeClass { // 2048-byte objects: 4 pages → header + room for objects let v = 16 * page_size / (obj_size * 8); let v = if v < 4 { v } else { 4 }; - if v < 1 { - 1 - } else { - v - } + if v < 1 { 1 } else { v } } } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6e62156..179a348 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -2,8 +2,8 @@ use buddy_slab_allocator::{GlobalAllocator, OsImpl}; use core::ptr::NonNull; -use rand::{rngs::StdRng, SeedableRng}; -use std::alloc::{alloc, dealloc, Layout}; +use rand::{SeedableRng, rngs::StdRng}; +use std::alloc::{Layout, alloc, dealloc}; use std::cell::Cell; thread_local! { diff --git a/tests/dma32_pages_test.rs b/tests/dma32_pages_test.rs index cbc0447..ee6e268 100644 --- a/tests/dma32_pages_test.rs +++ b/tests/dma32_pages_test.rs @@ -5,7 +5,7 @@ extern crate buddy_slab_allocator; mod common; use buddy_slab_allocator::GlobalAllocator; -use common::{init_global, HostRegion, LOWMEM_OS}; +use common::{HostRegion, LOWMEM_OS, init_global}; const PAGE_SIZE: usize = 0x1000; const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; diff --git a/tests/integration_test.rs b/tests/integration_test.rs index bf366ae..4713a6e 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -4,16 +4,16 @@ extern crate buddy_slab_allocator; mod common; use buddy_slab_allocator::{ - slab::SlabPageHeader, AllocError, BuddyAllocator, GlobalAllocator, ManagedSection, SizeClass, - SlabAllocResult, SlabAllocator, SlabDeallocResult, + AllocError, BuddyAllocator, GlobalAllocator, ManagedSection, SizeClass, SlabAllocResult, + SlabAllocator, SlabDeallocResult, slab::SlabPageHeader, }; use core::alloc::Layout; use core::ptr::NonNull; use std::collections::BTreeSet; use common::{ - count_free_pages, init_global as init_global_allocator, set_current_cpu, HostRegion, LOWMEM_OS, - TEST_OS, + HostRegion, LOWMEM_OS, TEST_OS, count_free_pages, init_global as init_global_allocator, + set_current_cpu, }; const PAGE_SIZE: usize = 0x1000; diff --git a/tests/stress_test.rs b/tests/stress_test.rs index c853002..1f58be6 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -4,8 +4,8 @@ mod common; use buddy_slab_allocator::{GlobalAllocator, SizeClass}; use common::{ - count_free_pages, init_global, nonnull_from_addr, seeded_rng, set_current_cpu, HostRegion, - TEST_OS, + HostRegion, TEST_OS, count_free_pages, init_global, nonnull_from_addr, seeded_rng, + set_current_cpu, }; use rand::RngExt; use std::alloc::Layout; From 324129e1a51dfcfde6c2b4f437040ba943524bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 11:03:39 +0800 Subject: [PATCH 17/19] refactor: add unsafe blocks for improved safety checks in allocator methods --- src/buddy/mod.rs | 249 +++++++++++++++++++++-------------------- src/buddy/page_meta.rs | 74 ++++++------ src/global.rs | 182 ++++++++++++++++-------------- src/slab/cache.rs | 58 +++++----- src/slab/page.rs | 24 ++-- 5 files changed, 312 insertions(+), 275 deletions(-) diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs index 67eafc0..e760969 100644 --- a/src/buddy/mod.rs +++ b/src/buddy/mod.rs @@ -172,61 +172,63 @@ impl BuddySection { heap_start: usize, heap_size: usize, ) -> AllocResult { - if !PAGE_SIZE.is_power_of_two() { - return Err(AllocError::InvalidParam); - } - if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 { - return Err(AllocError::InvalidParam); - } + unsafe { + if !PAGE_SIZE.is_power_of_two() { + return Err(AllocError::InvalidParam); + } + if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 { + return Err(AllocError::InvalidParam); + } - let total_pages = heap_size / PAGE_SIZE; - let required = BuddyAllocator::::required_meta_size(heap_size); - if meta_size < required { - return Err(AllocError::InvalidParam); - } + let total_pages = heap_size / PAGE_SIZE; + let required = BuddyAllocator::::required_meta_size(heap_size); + if meta_size < required { + return Err(AllocError::InvalidParam); + } - let meta = meta_ptr as *mut PageMeta; - for i in 0..total_pages { - meta.add(i).write(PageMeta::new()); - } - - section_ptr.write(BuddySection { - next: ptr::null_mut(), - region_start, - region_size, - meta, - max_pages: total_pages, - heap_start, - heap_size, - free_lists: [PFN_NONE; MAX_ORDER + 1], - free_pages: 0, - total_pages, - }); - - let section = &mut *section_ptr; - let mut pfn: usize = 0; - while pfn < total_pages { - let mut order = MAX_ORDER; - loop { - let block_pages = 1usize << order; - if block_pages <= total_pages - pfn && (pfn & (block_pages - 1)) == 0 { - break; - } - if order == 0 { - break; + let meta = meta_ptr as *mut PageMeta; + for i in 0..total_pages { + meta.add(i).write(PageMeta::new()); + } + + section_ptr.write(BuddySection { + next: ptr::null_mut(), + region_start, + region_size, + meta, + max_pages: total_pages, + heap_start, + heap_size, + free_lists: [PFN_NONE; MAX_ORDER + 1], + free_pages: 0, + total_pages, + }); + + let section = &mut *section_ptr; + let mut pfn: usize = 0; + while pfn < total_pages { + let mut order = MAX_ORDER; + loop { + let block_pages = 1usize << order; + if block_pages <= total_pages - pfn && (pfn & (block_pages - 1)) == 0 { + break; + } + if order == 0 { + break; + } + order -= 1; } - order -= 1; + let block_pages = 1usize << order; + let m = &mut *section.meta.add(pfn); + m.flags = PageFlags::Free; + m.order = order as u8; + free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); + section.free_pages += block_pages; + pfn += block_pages; } - let block_pages = 1usize << order; - let m = &mut *section.meta.add(pfn); - m.flags = PageFlags::Free; - m.order = order as u8; - free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); - section.free_pages += block_pages; - pfn += block_pages; - } - Ok(()) + Ok(()) + } } #[inline] @@ -301,8 +303,10 @@ impl BuddyAllocator { region: &mut [u8], os: Option<&'static dyn OsImpl>, ) -> AllocResult { - self.reset(os); - self.add_region(region) + unsafe { + self.reset(os); + self.add_region(region) + } } /// Add a new managed region after initialisation. @@ -311,76 +315,81 @@ impl BuddyAllocator { /// - `region` must be writable and remain valid for the lifetime of this allocator. /// - The region must not overlap any existing managed region. pub unsafe fn add_region(&mut self, region: &mut [u8]) -> AllocResult { - let region_start = region.as_mut_ptr() as usize; - let region_size = region.len(); - let layout = BuddySection::compute_region_layout::(region_start, region_size) - .ok_or(AllocError::InvalidParam)?; - self.add_region_raw(SectionInitSpec { - region_start, - region_size, - section_ptr: layout.section_start as *mut BuddySection, - meta_ptr: layout.meta_start as *mut u8, - meta_size: Self::required_meta_size(layout.managed_heap_size), - heap_start: layout.managed_heap_start, - heap_size: layout.managed_heap_size, - }) + unsafe { + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); + let layout = + BuddySection::compute_region_layout::(region_start, region_size) + .ok_or(AllocError::InvalidParam)?; + self.add_region_raw(SectionInitSpec { + region_start, + region_size, + section_ptr: layout.section_start as *mut BuddySection, + meta_ptr: layout.meta_start as *mut u8, + meta_size: Self::required_meta_size(layout.managed_heap_size), + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + }) + } } pub(crate) unsafe fn add_region_raw(&mut self, spec: SectionInitSpec) -> AllocResult { - let region_size = spec.region_size; - let region_end = spec - .region_start - .checked_add(region_size) - .ok_or(AllocError::InvalidParam)?; - let heap_end = spec - .heap_start - .checked_add(spec.heap_size) - .ok_or(AllocError::InvalidParam)?; - if heap_end > region_end { - return Err(AllocError::InvalidParam); - } - - let mut section = self.sections_head; - while !section.is_null() { - let existing = &*section; - let existing_end = existing + unsafe { + let region_size = spec.region_size; + let region_end = spec .region_start - .checked_add(existing.region_size) + .checked_add(region_size) .ok_or(AllocError::InvalidParam)?; - if spec.region_start < existing_end && existing.region_start < region_end { - return Err(AllocError::MemoryOverlap); + let heap_end = spec + .heap_start + .checked_add(spec.heap_size) + .ok_or(AllocError::InvalidParam)?; + if heap_end > region_end { + return Err(AllocError::InvalidParam); } - section = existing.next; - } - BuddySection::init_at::( - spec.section_ptr, - spec.region_start, - spec.region_size, - spec.meta_ptr, - spec.meta_size, - spec.heap_start, - spec.heap_size, - )?; + let mut section = self.sections_head; + while !section.is_null() { + let existing = &*section; + let existing_end = existing + .region_start + .checked_add(existing.region_size) + .ok_or(AllocError::InvalidParam)?; + if spec.region_start < existing_end && existing.region_start < region_end { + return Err(AllocError::MemoryOverlap); + } + section = existing.next; + } - if self.sections_head.is_null() { - self.sections_head = spec.section_ptr; - } else { - (*self.sections_tail).next = spec.section_ptr; - } - self.sections_tail = spec.section_ptr; - self.section_count += 1; - - log::debug!( - "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages", - spec.region_start, - spec.region_size, - spec.heap_start, - heap_end, - spec.heap_size / PAGE_SIZE, - ); + BuddySection::init_at::( + spec.section_ptr, + spec.region_start, + spec.region_size, + spec.meta_ptr, + spec.meta_size, + spec.heap_start, + spec.heap_size, + )?; + + if self.sections_head.is_null() { + self.sections_head = spec.section_ptr; + } else { + (*self.sections_tail).next = spec.section_ptr; + } + self.sections_tail = spec.section_ptr; + self.section_count += 1; + + log::debug!( + "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages", + spec.region_start, + spec.region_size, + spec.heap_start, + heap_end, + spec.heap_size / PAGE_SIZE, + ); - Ok(()) + Ok(()) + } } /// Number of managed sections. @@ -642,12 +651,14 @@ impl BuddyAllocator { /// # Safety /// The caller must ensure `addr` is valid and properly allocated. pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) -> AllocResult { - let section = self - .find_section_by_addr_mut(addr) - .ok_or(AllocError::NotFound)?; - let pfn = (addr - section.heap_start) / PAGE_SIZE; - (*section.meta.add(pfn)).flags = flags; - Ok(()) + unsafe { + let section = self + .find_section_by_addr_mut(addr) + .ok_or(AllocError::NotFound)?; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + (*section.meta.add(pfn)).flags = flags; + Ok(()) + } } /// Read the flags of the page containing `addr`. diff --git a/src/buddy/page_meta.rs b/src/buddy/page_meta.rs index 30c27f8..8f49110 100644 --- a/src/buddy/page_meta.rs +++ b/src/buddy/page_meta.rs @@ -69,14 +69,16 @@ impl PageMeta { /// `pfn` must not already be in any free list. #[inline] pub unsafe fn free_list_push(meta: *mut PageMeta, free_lists: &mut [u32], pfn: u32, order: usize) { - let old_head = free_lists[order]; - let m = &mut *meta.add(pfn as usize); - m.prev = PFN_NONE; - m.next = old_head; - if old_head != PFN_NONE { - (*meta.add(old_head as usize)).prev = pfn; + unsafe { + let old_head = free_lists[order]; + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = old_head; + if old_head != PFN_NONE { + (*meta.add(old_head as usize)).prev = pfn; + } + free_lists[order] = pfn; } - free_lists[order] = pfn; } /// Pop the first PFN from `free_lists[order]`, returning `PFN_NONE` if empty. @@ -85,19 +87,21 @@ pub unsafe fn free_list_push(meta: *mut PageMeta, free_lists: &mut [u32], pfn: u /// `meta` must be a valid metadata array. #[inline] pub unsafe fn free_list_pop(meta: *mut PageMeta, free_lists: &mut [u32], order: usize) -> u32 { - let head = free_lists[order]; - if head == PFN_NONE { - return PFN_NONE; - } - let m = &mut *meta.add(head as usize); - let next = m.next; - m.prev = PFN_NONE; - m.next = PFN_NONE; - if next != PFN_NONE { - (*meta.add(next as usize)).prev = PFN_NONE; + unsafe { + let head = free_lists[order]; + if head == PFN_NONE { + return PFN_NONE; + } + let m = &mut *meta.add(head as usize); + let next = m.next; + m.prev = PFN_NONE; + m.next = PFN_NONE; + if next != PFN_NONE { + (*meta.add(next as usize)).prev = PFN_NONE; + } + free_lists[order] = next; + head } - free_lists[order] = next; - head } /// Remove `pfn` from the free list at `order`. @@ -111,21 +115,23 @@ pub unsafe fn free_list_remove( pfn: u32, order: usize, ) { - let m = &*meta.add(pfn as usize); - let prev = m.prev; - let next = m.next; + unsafe { + let m = &*meta.add(pfn as usize); + let prev = m.prev; + let next = m.next; - if prev != PFN_NONE { - (*meta.add(prev as usize)).next = next; - } else { - // pfn was the head - free_lists[order] = next; - } - if next != PFN_NONE { - (*meta.add(next as usize)).prev = prev; - } + if prev != PFN_NONE { + (*meta.add(prev as usize)).next = next; + } else { + // pfn was the head + free_lists[order] = next; + } + if next != PFN_NONE { + (*meta.add(next as usize)).prev = prev; + } - let m = &mut *meta.add(pfn as usize); - m.prev = PFN_NONE; - m.next = PFN_NONE; + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = PFN_NONE; + } } diff --git a/src/global.rs b/src/global.rs index 159a2ee..ead8131 100644 --- a/src/global.rs +++ b/src/global.rs @@ -183,51 +183,53 @@ impl GlobalAllocator { cpu_count: usize, os: &'static dyn OsImpl, ) -> AllocResult { - let region_start = region.as_mut_ptr() as usize; - let region_size = region.len(); - let layout = Self::compute_initial_region_layout(region_start, region_size, cpu_count) - .ok_or(AllocError::InvalidParam)?; - let section_ptr = layout.section_start as *mut BuddySection; - let meta_ptr = layout.meta_start as *mut u8; - let slab_ptr = - (layout.section_start + layout.slab_offset) as *mut SpinMutex>; - - let mut buddy = self.buddy.lock(); - buddy.reset(Some(os)); - buddy.add_region_raw(SectionInitSpec { - region_start, - region_size, - section_ptr, - meta_ptr, - meta_size: layout.buddy_meta_size, - heap_start: layout.managed_heap_start, - heap_size: layout.managed_heap_size, - })?; - drop(buddy); - - for i in 0..cpu_count { - let slot = slab_ptr.add(i); - slot.write(SpinMutex::new(SlabAllocator::new())); - } - - let self_mut = self as *const Self as *mut Self; - (*self_mut).per_cpu_slabs = slab_ptr; - (*self_mut).cpu_count = cpu_count; - (*self_mut).os = Some(os); - self.initialized.store(true, Ordering::Release); - - log::debug!( - "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, first heap {:#x}+{:#x}", - cpu_count, - region_start, - region_size, - layout.section_start, - layout.meta_size, - layout.managed_heap_start, - layout.managed_heap_size, - ); + unsafe { + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); + let layout = Self::compute_initial_region_layout(region_start, region_size, cpu_count) + .ok_or(AllocError::InvalidParam)?; + let section_ptr = layout.section_start as *mut BuddySection; + let meta_ptr = layout.meta_start as *mut u8; + let slab_ptr = (layout.section_start + layout.slab_offset) + as *mut SpinMutex>; + + let mut buddy = self.buddy.lock(); + buddy.reset(Some(os)); + buddy.add_region_raw(SectionInitSpec { + region_start, + region_size, + section_ptr, + meta_ptr, + meta_size: layout.buddy_meta_size, + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + })?; + drop(buddy); + + for i in 0..cpu_count { + let slot = slab_ptr.add(i); + slot.write(SpinMutex::new(SlabAllocator::new())); + } - Ok(()) + let self_mut = self as *const Self as *mut Self; + (*self_mut).per_cpu_slabs = slab_ptr; + (*self_mut).cpu_count = cpu_count; + (*self_mut).os = Some(os); + self.initialized.store(true, Ordering::Release); + + log::debug!( + "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, first heap {:#x}+{:#x}", + cpu_count, + region_start, + region_size, + layout.section_start, + layout.meta_size, + layout.managed_heap_start, + layout.managed_heap_size, + ); + + Ok(()) + } } /// Add a new managed region after [`init`](Self::init). @@ -236,10 +238,12 @@ impl GlobalAllocator { /// - `region` must be writable and remain valid for the lifetime of this allocator. /// - The region must not overlap any already managed region. pub unsafe fn add_region(&self, region: &mut [u8]) -> AllocResult { - if !self.initialized.load(Ordering::Acquire) { - return Err(AllocError::NotInitialized); + unsafe { + if !self.initialized.load(Ordering::Acquire) { + return Err(AllocError::NotInitialized); + } + self.buddy.lock().add_region(region) } - self.buddy.lock().add_region(region) } /// Number of managed sections. @@ -300,10 +304,12 @@ impl GlobalAllocator { /// # Safety /// `ptr` must have been returned by a prior `alloc` with the same `layout`. pub unsafe fn dealloc(&self, ptr: NonNull, layout: Layout) { - if self.is_slab_eligible(&layout) { - self.slab_dealloc(ptr, layout); - } else { - self.large_dealloc(ptr, layout); + unsafe { + if self.is_slab_eligible(&layout) { + self.slab_dealloc(ptr, layout); + } else { + self.large_dealloc(ptr, layout); + } } } @@ -340,29 +346,31 @@ impl GlobalAllocator { } unsafe fn slab_dealloc(&self, ptr: NonNull, layout: Layout) { - let os = self.os.expect("not initialized"); - let sc = SizeClass::from_layout(layout).expect("layout exceeds slab"); - let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE; - let base = - SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); - let hdr = &*(base as *const SlabPageHeader); - debug_assert_eq!(hdr.magic, SLAB_MAGIC); - - let owner_cpu = hdr.owner_cpu as usize; - let current_cpu = os.current_cpu_idx(); - - if owner_cpu == current_cpu { - let slab_lock = &*self.per_cpu_slabs.add(current_cpu); - let mut slab = slab_lock.lock(); - match slab.dealloc(ptr, layout) { - SlabDeallocResult::Done => {} - SlabDeallocResult::FreeSlab { base, pages } => { - drop(slab); - self.buddy.lock().dealloc_pages(base, pages); + unsafe { + let os = self.os.expect("not initialized"); + let sc = SizeClass::from_layout(layout).expect("layout exceeds slab"); + let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE; + let base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + let hdr = &*(base as *const SlabPageHeader); + debug_assert_eq!(hdr.magic, SLAB_MAGIC); + + let owner_cpu = hdr.owner_cpu as usize; + let current_cpu = os.current_cpu_idx(); + + if owner_cpu == current_cpu { + let slab_lock = &*self.per_cpu_slabs.add(current_cpu); + let mut slab = slab_lock.lock(); + match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + drop(slab); + self.buddy.lock().dealloc_pages(base, pages); + } } + } else { + hdr.remote_free(ptr.as_ptr() as usize); } - } else { - hdr.remote_free(ptr.as_ptr() as usize); } } @@ -390,23 +398,27 @@ unsafe impl GlobalAlloc for GlobalAllocator { } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - if let Some(nn) = NonNull::new(ptr) { - self.dealloc(nn, layout); + unsafe { + if let Some(nn) = NonNull::new(ptr) { + self.dealloc(nn, layout); + } } } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - let new_layout = match Layout::from_size_align(new_size, layout.align()) { - Ok(l) => l, - Err(_) => return ptr::null_mut(), - }; - - let new_ptr = ::alloc(self, new_layout); - if !new_ptr.is_null() { - let copy_size = layout.size().min(new_size); - ptr::copy_nonoverlapping(ptr, new_ptr, copy_size); - ::dealloc(self, ptr, layout); + unsafe { + let new_layout = match Layout::from_size_align(new_size, layout.align()) { + Ok(l) => l, + Err(_) => return ptr::null_mut(), + }; + + let new_ptr = ::alloc(self, new_layout); + if !new_ptr.is_null() { + let copy_size = layout.size().min(new_size); + ptr::copy_nonoverlapping(ptr, new_ptr, copy_size); + ::dealloc(self, ptr, layout); + } + new_ptr } - new_ptr } } diff --git a/src/slab/cache.rs b/src/slab/cache.rs index 3473fa3..ba72e28 100644 --- a/src/slab/cache.rs +++ b/src/slab/cache.rs @@ -27,14 +27,16 @@ impl ListHead { /// # Safety /// `base` must point to a valid `SlabPageHeader`. unsafe fn push_front(&mut self, base: usize) { - let hdr = &mut *(base as *mut SlabPageHeader); - hdr.list_prev = 0; - hdr.list_next = self.first; - if self.first != 0 { - let old = &mut *(self.first as *mut SlabPageHeader); - old.list_prev = base; + unsafe { + let hdr = &mut *(base as *mut SlabPageHeader); + hdr.list_prev = 0; + hdr.list_next = self.first; + if self.first != 0 { + let old = &mut *(self.first as *mut SlabPageHeader); + old.list_prev = base; + } + self.first = base; } - self.first = base; } /// Remove a slab page from this list. @@ -42,32 +44,36 @@ impl ListHead { /// # Safety /// `base` must be in this list. unsafe fn remove(&mut self, base: usize) { - let hdr = &*(base as *const SlabPageHeader); - let prev = hdr.list_prev; - let next = hdr.list_next; + unsafe { + let hdr = &*(base as *const SlabPageHeader); + let prev = hdr.list_prev; + let next = hdr.list_next; - if prev != 0 { - (*(prev as *mut SlabPageHeader)).list_next = next; - } else { - self.first = next; - } - if next != 0 { - (*(next as *mut SlabPageHeader)).list_prev = prev; + if prev != 0 { + (*(prev as *mut SlabPageHeader)).list_next = next; + } else { + self.first = next; + } + if next != 0 { + (*(next as *mut SlabPageHeader)).list_prev = prev; + } + // Clear links + let hdr = &mut *(base as *mut SlabPageHeader); + hdr.list_prev = 0; + hdr.list_next = 0; } - // Clear links - let hdr = &mut *(base as *mut SlabPageHeader); - hdr.list_prev = 0; - hdr.list_next = 0; } /// Pop the first page from the list. Returns 0 if empty. unsafe fn pop_front(&mut self) -> usize { - if self.first == 0 { - return 0; + unsafe { + if self.first == 0 { + return 0; + } + let base = self.first; + self.remove(base); + base } - let base = self.first; - self.remove(base); - base } } diff --git a/src/slab/page.rs b/src/slab/page.rs index 12a7a4c..3771609 100644 --- a/src/slab/page.rs +++ b/src/slab/page.rs @@ -198,17 +198,19 @@ impl SlabPageHeader { /// - The object's first `size_of::()` bytes will be overwritten with /// the next-pointer. pub unsafe fn remote_free(&self, obj_addr: usize) { - loop { - let old_head = self.remote_free_head.load(Ordering::Acquire); - // Store "next" pointer inside the freed object. - (obj_addr as *mut usize).write(old_head); - if self - .remote_free_head - .compare_exchange_weak(old_head, obj_addr, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - self.remote_free_count.fetch_add(1, Ordering::Relaxed); - return; + unsafe { + loop { + let old_head = self.remote_free_head.load(Ordering::Acquire); + // Store "next" pointer inside the freed object. + (obj_addr as *mut usize).write(old_head); + if self + .remote_free_head + .compare_exchange_weak(old_head, obj_addr, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.remote_free_count.fetch_add(1, Ordering::Relaxed); + return; + } } } } From e861fd3e374a8e111815b599d4a17f51b3ab34fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 11:04:03 +0800 Subject: [PATCH 18/19] chore: update changelog for version 0.3.0 with new features and improvements --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9500db..1a8f699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...HEAD +## [0.3.0](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.3.0) - 2026-04-09 + +### Added + +- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation +- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator +- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic +- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples +- enhance logging support by integrating log crate and updating documentation + +### Other + +- add unsafe blocks for improved safety checks in allocator methods +- update changelog for version 0.2.1 and modify Cargo.toml for version bump to 0.3.0 +- format code for better readability in common.rs +- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability +- Refactor GlobalAllocator to support multiple managed sections +- Refactor stress tests for allocator stability +- update allocator initialization to use a mutable slice instead of separate start and size parameters +- Refactor slab allocator benchmarks and improve global allocator initialization +- Refactor integration and stress tests for buddy-slab-allocator +- Refactor integration and stress tests for allocator +- Refactor benchmarks and remove stability tests +- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10 + ## [0.2.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.2.1) - 2026-04-09 ### Added From 9d655878a56c824759feb8664544330155ce8f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Thu, 9 Apr 2026 11:07:37 +0800 Subject: [PATCH 19/19] refactor: improve unsafe block usage in subslice and stress test for clarity --- tests/common/mod.rs | 2 +- tests/stress_test.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 179a348..2f79297 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -79,7 +79,7 @@ impl HostRegion { } pub unsafe fn subslice(&mut self, offset: usize, len: usize) -> &mut [u8] { - std::slice::from_raw_parts_mut(self.ptr.add(offset), len) + unsafe { std::slice::from_raw_parts_mut(self.ptr.add(offset), len) } } } diff --git a/tests/stress_test.rs b/tests/stress_test.rs index 1f58be6..0c15554 100644 --- a/tests/stress_test.rs +++ b/tests/stress_test.rs @@ -401,11 +401,11 @@ fn stress_multithread_exhaustion_recovery() { } barrier.wait(); - if cpu == 0 { - if let Ok(ptr) = allocator.alloc(layout) { - recovered.store(true, Ordering::Relaxed); - unsafe { allocator.dealloc(ptr, layout) }; - } + if cpu == 0 + && let Ok(ptr) = allocator.alloc(layout) + { + recovered.store(true, Ordering::Relaxed); + unsafe { allocator.dealloc(ptr, layout) }; } barrier.wait();