diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 4cc8d5c..bc2347e 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
@@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
@@ -37,14 +37,14 @@ jobs:
uses: Swatinem/rust-cache@v2
- name: Run clippy
- run: cargo clippy --features log,tracking -- -D warnings
+ run: cargo clippy --all-targets -- -D warnings
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
@@ -55,14 +55,14 @@ jobs:
uses: Swatinem/rust-cache@v2
- name: Build library
- run: cargo build --features log,tracking
+ run: cargo build
doc:
name: Documentation
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
@@ -73,4 +73,22 @@ jobs:
uses: Swatinem/rust-cache@v2
- name: Build documentation
- run: cargo doc --no-deps --features log,tracking
+ run: cargo doc --no-deps
+
+ bench-check:
+ name: Bench Compile Check
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@master
+ with:
+ toolchain: nightly-2026-02-25
+
+ - name: Rust Cache
+ uses: Swatinem/rust-cache@v2
+
+ - name: Check benchmarks
+ run: cargo check --benches
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..f08e078
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,20 @@
+name: CI
+
+on:
+ pull_request:
+ push:
+ branches-ignore:
+ - main
+ tags-ignore:
+ - "*"
+
+permissions:
+ contents: read
+
+jobs:
+ check:
+ uses: ./.github/workflows/check.yml
+
+ test:
+ uses: ./.github/workflows/test.yml
+ needs: check
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
deleted file mode 100644
index d3850e2..0000000
--- a/.github/workflows/deploy.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-name: Deploy
-
-on:
- push:
- branches:
- - "**"
- tags-ignore:
- - "v*"
- - "v*-pre.*"
- pull_request:
-
-permissions:
- contents: read
- pages: write
- id-token: write
-
-concurrency:
- group: "pages"
- cancel-in-progress: false
-
-jobs:
- check:
- uses: ./.github/workflows/check.yml
-
- test:
- uses: ./.github/workflows/test.yml
- needs: check
-
- benchmark:
- name: Benchmark check
- runs-on: ubuntu-latest
- needs: check
- steps:
- - name: Checkout code
- uses: actions/checkout@v4
-
- - name: Install Rust
- uses: dtolnay/rust-toolchain@master
- with:
- toolchain: nightly-2026-02-25
-
- - name: Rust Cache
- uses: Swatinem/rust-cache@v2
-
- - name: Build benchmarks
- run: cargo bench --no-run
-
- build-doc:
- name: Build documentation
- runs-on: ubuntu-latest
- needs: test
- steps:
- - name: Checkout code
- uses: actions/checkout@v4
-
- - name: Install Rust toolchain
- uses: dtolnay/rust-toolchain@master
- with:
- toolchain: nightly-2026-02-25
-
- - name: Rust Cache
- uses: Swatinem/rust-cache@v2
-
- - name: Build docs
- run: cargo doc --no-deps --features log,tracking
-
- - name: Create index redirect
- run: |
- printf '' > target/doc/index.html
-
- - name: Upload artifact
- uses: actions/upload-pages-artifact@v3
- with:
- path: target/doc
-
- deploy-doc:
- name: Deploy to GitHub Pages
- environment:
- name: github-pages
- url: ${{ steps.deployment.outputs.page_url }}
- runs-on: ubuntu-latest
- needs: build-doc
- if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
- steps:
- - name: Deploy to GitHub Pages
- id: deployment
- uses: actions/deploy-pages@v4
diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml
new file mode 100644
index 0000000..0d36a95
--- /dev/null
+++ b/.github/workflows/release-plz.yml
@@ -0,0 +1,72 @@
+name: Release-plz
+
+on:
+ push:
+ branches:
+ - main
+
+permissions:
+ contents: read
+
+jobs:
+ check:
+ uses: ./.github/workflows/check.yml
+
+ test:
+ uses: ./.github/workflows/test.yml
+ needs: check
+
+ release-plz-release:
+ name: Release-plz release
+ runs-on: ubuntu-latest
+ needs: [check, test]
+ permissions:
+ contents: write
+ pull-requests: read
+ id-token: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Run release-plz
+ uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128
+ with:
+ command: release
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # If you switch to crates.io trusted publishing, remove this line
+ # and keep `id-token: write` above.
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+ release-plz-pr:
+ name: Release-plz PR
+ runs-on: ubuntu-latest
+ needs: [check, test]
+ permissions:
+ contents: write
+ pull-requests: write
+ concurrency:
+ group: release-plz-${{ github.ref }}
+ cancel-in-progress: false
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Run release-plz
+ uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128
+ with:
+ command: release-pr
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index e807c93..0000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,128 +0,0 @@
-name: Release
-
-on:
- push:
- tags:
- - "v*.*.*"
- - "v*.*.*-pre.*"
-
-permissions:
- contents: write
-
-jobs:
- check:
- uses: ./.github/workflows/check.yml
-
- test:
- uses: ./.github/workflows/test.yml
- needs: check
-
- create-release:
- name: Create GitHub Release
- runs-on: ubuntu-latest
- needs: [check, test]
-
- steps:
- - name: Checkout code
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Validate tag and branch (HEAD-based)
- shell: bash
- run: |
- set -e
-
- TAG="${{ github.ref_name }}"
- TAG_COMMIT=$(git rev-list -n 1 "$TAG")
-
- git fetch origin main dev
-
- MAIN_HEAD=$(git rev-parse origin/main)
- DEV_HEAD=$(git rev-parse origin/dev)
-
- echo "Tag: $TAG"
- echo "Tag commit: $TAG_COMMIT"
- echo "main HEAD: $MAIN_HEAD"
- echo "dev HEAD: $DEV_HEAD"
-
- if [[ "$TAG" == *-pre.* ]]; then
- if [ "$TAG_COMMIT" != "$DEV_HEAD" ]; then
- echo "❌ prerelease tag must be created from dev HEAD"
- exit 1
- fi
- echo "✅ prerelease tag validated on dev"
- else
- if [ "$TAG_COMMIT" != "$MAIN_HEAD" ]; then
- echo "❌ stable release tag must be created from main HEAD"
- exit 1
- fi
- echo "✅ stable release tag validated on main"
- fi
-
- - name: Create GitHub Release
- uses: softprops/action-gh-release@v2
- with:
- draft: false
- prerelease: ${{ contains(github.ref_name, '-pre.') }}
- body: |
- ## ${{ github.ref_name }}
-
- - [Documentation](https://docs.rs/buddy-slab-allocator)
- - [crates.io](https://crates.io/crates/buddy-slab-allocator)
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
- publish-crates:
- name: Publish to crates.io
- runs-on: ubuntu-latest
- needs: test
-
- steps:
- - name: Checkout code
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Validate tag and branch (HEAD-based)
- shell: bash
- run: |
- set -e
-
- TAG="${{ github.ref_name }}"
- TAG_COMMIT=$(git rev-list -n 1 "$TAG")
-
- git fetch origin main dev
-
- MAIN_HEAD=$(git rev-parse origin/main)
- DEV_HEAD=$(git rev-parse origin/dev)
-
- echo "Tag: $TAG"
- echo "Tag commit: $TAG_COMMIT"
- echo "main HEAD: $MAIN_HEAD"
- echo "dev HEAD: $DEV_HEAD"
-
- if [[ "$TAG" == *-pre.* ]]; then
- if [ "$TAG_COMMIT" != "$DEV_HEAD" ]; then
- echo "❌ prerelease tag must be created from dev HEAD"
- exit 1
- fi
- echo "✅ prerelease tag validated on dev"
- else
- if [ "$TAG_COMMIT" != "$MAIN_HEAD" ]; then
- echo "❌ stable release tag must be created from main HEAD"
- exit 1
- fi
- echo "✅ stable release tag validated on main"
- fi
-
- - name: Install Rust toolchain
- uses: dtolnay/rust-toolchain@master
- with:
- toolchain: nightly-2026-02-25
-
- - name: Rust Cache
- uses: Swatinem/rust-cache@v2
-
- - name: Publish to crates.io
- run: cargo publish --no-verify --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8b1b763..45ecba0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,15 +9,21 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v6 # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
with:
- toolchain: nightly-2026-02-25
+ toolchain: stable
- name: Rust Cache
uses: Swatinem/rust-cache@v2
- name: Run all tests
- run: cargo test --features log,tracking
+ run: cargo test
+
+ - name: Run tests serially
+ run: cargo test -- --test-threads=1
+
+ - name: Run ignored stress tests
+ run: cargo test --test stress_test -- --ignored --nocapture
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f0d7c65..1a8f699 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,18 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Changed
+- Removed the external allocator-interface dependency and restored crate-local `AllocError` / `AllocResult`
+- Stopped exporting the generic allocator traits `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator`
+- Public allocator APIs now favor concrete allocator methods directly instead of requiring trait imports
+- Removed `alloc_pages_at` because the current buddy/slab architecture does not stably support fixed-address allocation
+
+### Removed
+- Removed the external allocator-interface crate from `[dependencies]`
+
+### Migration Notes
+- Replace imports of `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` with direct method calls on `BuddyPageAllocator`, `CompositePageAllocator`, `SlabByteAllocator`, and `GlobalAllocator`
+- `PageAllocatorForSlab` remains available for wiring `SlabByteAllocator` to a page allocator
+
## [0.2.0] - 2026-03-05
### Added
-- Added `axallocator = "0.2"` as a dependency
+- Added an external allocator-interface dependency
### Changed
-- `AllocError`, `AllocResult`, `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` are now re-exported from `axallocator` instead of being defined locally
+- `AllocError`, `AllocResult`, `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` are now re-exported from the external allocator-interface crate instead of being defined locally
- Updated Rust toolchain to `nightly-2026-02-25`
- Benchmarks no longer require `--features bench`; `criterion` and `rand` moved to `[dev-dependencies]`
### Removed
-- Removed locally defined allocator trait and error type definitions (now provided by `axallocator`)
+- Removed locally defined allocator trait and error type definitions (now provided by the external allocator-interface crate)
- Removed the deprecated `bench` feature flag
## [0.1.1] - 2026-02-06
@@ -69,6 +82,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Comprehensive edge case coverage
## [Unreleased]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...HEAD
+
+## [0.3.0](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.3.0) - 2026-04-09
+
+### Added
+
+- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation
+- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator
+- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic
+- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples
+- enhance logging support by integrating log crate and updating documentation
+
+### Other
+
+- add unsafe blocks for improved safety checks in allocator methods
+- update changelog for version 0.2.1 and modify Cargo.toml for version bump to 0.3.0
+- format code for better readability in common.rs
+- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability
+- Refactor GlobalAllocator to support multiple managed sections
+- Refactor stress tests for allocator stability
+- update allocator initialization to use a mutable slice instead of separate start and size parameters
+- Refactor slab allocator benchmarks and improve global allocator initialization
+- Refactor integration and stress tests for buddy-slab-allocator
+- Refactor integration and stress tests for allocator
+- Refactor benchmarks and remove stability tests
+- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10
+
+## [0.2.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.2.1) - 2026-04-09
+
+### Added
+
+- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation
+- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator
+- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic
+- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples
+- enhance logging support by integrating log crate and updating documentation
+
+### Other
+
+- format code for better readability in common.rs
+- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability
+- Refactor GlobalAllocator to support multiple managed sections
+- Refactor stress tests for allocator stability
+- update allocator initialization to use a mutable slice instead of separate start and size parameters
+- Refactor slab allocator benchmarks and improve global allocator initialization
+- Refactor integration and stress tests for buddy-slab-allocator
+- Refactor integration and stress tests for allocator
+- Refactor benchmarks and remove stability tests
+- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10
## [0.2.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.1...v0.2.0
## [0.1.1]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.0...v0.1.1
## [0.1.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/releases/tag/v0.1.0
diff --git a/Cargo.toml b/Cargo.toml
index ee1f184..1d0725e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,42 +1,33 @@
[package]
-name = "buddy-slab-allocator"
-version = "0.2.0"
-edition = "2021"
-authors = ["Song Zhiyong "]
+authors = ["Song Zhiyong ", "周睿 "]
+autobenches = false
+categories = ["memory-management", "no-std", "embedded"]
description = "Memory allocator with Buddy and Slab allocation"
documentation = "https://docs.rs/buddy-slab-allocator"
-repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator"
-readme = "README.md"
-license = "Apache-2.0"
+edition = "2024"
keywords = ["buddy", "slab", "allocator"]
-categories = ["memory-management", "no-std", "embedded"]
-
-[features]
-default = []
-log = ["dep:log"]
-tracking = []
+license = "Apache-2.0"
+name = "buddy-slab-allocator"
+readme = "README.md"
+repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator"
+version = "0.3.0"
[dependencies]
-axallocator = "0.2"
-cfg-if = "1.0"
-log = { version = "0.4", optional = true }
+log = "0.4"
+spin = "0.10"
[dev-dependencies]
-criterion = { version = "0.4", features = ["html_reports"] }
-rand = { version = "0.8", features = ["small_rng"] }
+divan = "0.1.21"
+rand = {version = "0.10", features = ["std_rng"]}
[[bench]]
-name = "global_allocator"
harness = false
-
-[[bench]]
name = "buddy_allocator"
-harness = false
[[bench]]
-name = "slab_allocator"
harness = false
+name = "slab_allocator"
[[bench]]
-name = "stability"
harness = false
+name = "global_allocator"
diff --git a/README.md b/README.md
index e3ceb89..5772995 100644
--- a/README.md
+++ b/README.md
@@ -1,195 +1,231 @@
# buddy-slab-allocator
-A high-performance page-level and byte-level memory allocator designed for embedded/kernel environments.
+A `no_std` two-level allocator for kernel and embedded environments, combining a buddy page allocator with per-CPU slab allocators.
-## Features
-
-- **Buddy Page Allocator**: Page-level memory allocation
-- **Slab Byte Allocator**: Small object allocation
-- **Composite Page Allocator**: Unified multi-region page allocation interface
-- **Global Allocator**: Coordinates page and byte allocators with unified allocation interface
-- **Zero `std` Dependency**: Fully `#![no_std]`, suitable for embedded and kernel environments
-- **Conditional Logging**: Support `log` feature for logging, no dependencies by default
-- **Memory Tracking**: Support `tracking` feature for detailed statistics
-
-## Quick Start
+## Overview
-### Add Dependency
+The current implementation is built from three layers:
-Add to your `Cargo.toml`:
-
-```toml
-[dependencies]
-buddy-slab-allocator = "0.1.0"
+1. `BuddyAllocator`
+ Manages one or more virtual memory sections in page units, with power-of-two splitting and merging.
+2. `SlabAllocator`
+ Manages small objects up to 2048 bytes with fixed size classes.
+3. `GlobalAllocator`
+ Combines the two, routing small allocations to per-CPU slab caches and large allocations to buddy pages.
-# Optional features
-buddy-slab-allocator = { version = "0.1.0", features = ["log"] } # Enable logging
-buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # Enable tracking
-```
+The design details are documented in [docs/design.md](docs/design.md).
-### Basic Usage
+## Architecture
-#### Using Global Allocator
+```mermaid
+flowchart TD
+ GA[GlobalAllocator] --> B[SpinMutex]
+ GA --> OS[OsImpl]
+ GA --> PCS[per_cpu_slabs: *mut SpinMutex[]]
-```rust
-use buddy_slab_allocator::GlobalAllocator;
-use core::alloc::Layout;
+ B --> PM[PageMeta[]]
+ B --> FL[free_lists by order]
-// Create global allocator
-let mut global = GlobalAllocator::new();
+ PCS --> SA[SlabAllocator]
+ SA --> SC[SlabCache x 9]
+ SC --> P[partial]
+ SC --> F[full]
+ SC --> E[empty]
+ SC --> H[SlabPageHeader]
+```
-// Initialize the global allocator with memory region
-let heap_start = 0x8000_0000;
-let heap_size = 16 * 1024 * 1024; // 16MB
-global.init(heap_start, heap_size).unwrap();
+### Allocation routing
+
+```mermaid
+flowchart TD
+ A[GlobalAllocator::alloc(layout)] --> B{size <= 2048 and align <= 2048?}
+ B -- Yes --> C[Use current CPU slab allocator]
+ C --> D{Allocated from slab?}
+ D -- Yes --> E[Return object pointer]
+ D -- No --> F[Allocate pages from buddy as a new slab]
+ F --> G[add_slab and retry]
+ G --> E
+ B -- No --> H[Allocate pages directly from buddy]
+ H --> I[Return page-backed pointer]
+```
-// Or add multiple memory pools
-global.add_memory(0x80000000, 0x1000000).unwrap();
-global.add_memory(0x81000000, 0x1000000).unwrap();
+### Cross-CPU free path
-// Small object allocation (automatically uses Slab allocator)
-let small_layout = Layout::from_size_align(64, 8).unwrap();
-let small_ptr = global.alloc(small_layout).unwrap();
+```mermaid
+sequenceDiagram
+ participant CPU1 as current CPU
+ participant H as SlabPageHeader
+ participant CPU0 as owner CPU
+ participant S as owner SlabAllocator
-// Large object allocation (automatically uses page allocator)
-let large_layout = Layout::from_size_align(0x1000, 0x1000).unwrap();
-let large_ptr = global.alloc(large_layout).unwrap();
+ CPU1->>H: remote_free(obj_addr)
+ Note right of H: lock-free CAS push to remote_free_head
+ CPU1-->>CPU1: return immediately
-// Free memory
-global.dealloc(small_ptr, small_layout);
-global.dealloc(large_ptr, large_layout);
+ CPU0->>S: next local alloc/dealloc under slab lock
+ S->>H: drain_remote_frees()
+ H-->>S: slots moved back into local bitmap
```
-#### Using Page Allocator Directly
-
-```rust
-use buddy_slab_allocator::CompositePageAllocator;
+## Features
-const PAGE_SIZE: usize = 0x1000;
-let mut page_alloc = CompositePageAllocator::::new();
+- Buddy page allocation with splitting and merging
+- Dynamic hot-add of managed regions via `add_region`
+- Slab allocation for 9 size classes: `8..=2048`
+- Per-CPU slab caches
+- Lock-free cross-CPU remote frees
+- DMA32 / lowmem page allocation via `alloc_pages_lowmem`
+- `no_std` friendly
+- Built-in `log` integration
+- Standalone `BuddyAllocator` and `SlabAllocator` usage
-// Initialize with memory region
-page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
+## Add dependency
-// Allocate pages
-let addr = page_alloc.alloc_pages(4, PAGE_SIZE).unwrap();
-// Use the allocated memory...
-page_alloc.dealloc_pages(addr, 4);
+```toml
+[dependencies]
+buddy-slab-allocator = "0.2.0"
```
-#### Using Slab Allocator Directly
+## Using `GlobalAllocator`
```rust
-use buddy_slab_allocator::SlabByteAllocator;
-use buddy_slab_allocator::page_allocator::PageAllocatorForSlab;
-use buddy_slab_allocator::CompositePageAllocator;
+use buddy_slab_allocator::{GlobalAllocator, OsImpl};
use core::alloc::Layout;
const PAGE_SIZE: usize = 0x1000;
-let mut page_alloc = CompositePageAllocator::::new();
-page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
-
-let mut slab_alloc = SlabByteAllocator::::new();
-// Small allocations are fast
-let layout = Layout::from_size_align(64, 8).unwrap();
-let ptr = slab_alloc.alloc(&mut page_alloc, layout).unwrap();
+struct DemoOs;
-// Free memory
-slab_alloc.dealloc(&mut page_alloc, ptr, layout);
-```
+impl OsImpl for DemoOs {
+ fn current_cpu_idx(&self) -> usize { 0 }
+ fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr }
+}
-## Features Details
+static OS: DemoOs = DemoOs;
-### Conditional Logging
+let allocator = GlobalAllocator::::new();
+let region_start = 0x8000_0000 as *mut u8;
+let region_size = 16 * 1024 * 1024;
+let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) };
-Enable logging via `log` feature:
+unsafe {
+ allocator.init(region, 1, &OS).unwrap();
+}
-```toml
-buddy-slab-allocator = { version = "0.1.0", features = ["log"] }
+let layout = Layout::from_size_align(64, 8).unwrap();
+let ptr = allocator.alloc(layout).unwrap();
+
+unsafe {
+ allocator.dealloc(ptr, layout);
+}
+
+// More memory can be added later.
+let extra_region_start = 0x9000_0000 as *mut u8;
+let extra_region_size = 8 * 1024 * 1024;
+let extra_region = unsafe {
+ core::slice::from_raw_parts_mut(extra_region_start, extra_region_size)
+};
+
+unsafe {
+ allocator.add_region(extra_region).unwrap();
+}
```
-After enabling, you can use standard `log` crate macros to log allocation events:
+## Using Buddy and Slab separately
+
+For lower-level control, the two building blocks can be used directly.
```rust
-log::info!("Allocated memory at {:x}", addr);
-```
+use buddy_slab_allocator::{
+ BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult,
+};
+use core::alloc::Layout;
-When disabled, log calls are compiled to no-ops with zero runtime overhead.
+const PAGE_SIZE: usize = 0x1000;
+let region_start = 0x8000_0000 as *mut u8;
+let region_size = 16 * 1024 * 1024;
+let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) };
-### Memory Tracking
+let mut buddy = BuddyAllocator::::new();
+unsafe {
+ buddy.init(region, None).unwrap();
+}
-Enable detailed memory usage tracking via `tracking` feature:
+let mut slab = SlabAllocator::::new();
+let layout = Layout::from_size_align(64, 8).unwrap();
-```toml
-buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] }
+let ptr = loop {
+ match slab.alloc(layout).unwrap() {
+ SlabAllocResult::Allocated(ptr) => break ptr,
+ SlabAllocResult::NeedsSlab { size_class, pages } => {
+ let slab_bytes = pages * PAGE_SIZE;
+ let addr = buddy.alloc_pages(pages, slab_bytes).unwrap();
+ slab.add_slab(size_class, addr, slab_bytes, 0);
+ }
+ }
+};
+
+match slab.dealloc(ptr, layout) {
+ SlabDeallocResult::Done => {}
+ SlabDeallocResult::FreeSlab { base, pages } => {
+ buddy.dealloc_pages(base, pages);
+ }
+}
+
+let extra_region_start = 0x9000_0000 as *mut u8;
+let extra_region_size = 8 * 1024 * 1024;
+let extra_region = unsafe {
+ core::slice::from_raw_parts_mut(extra_region_start, extra_region_size)
+};
+
+unsafe {
+ buddy.add_region(extra_region).unwrap();
+}
```
-After enabling, you can:
-- Collect statistics of bytes allocated for each memory usage
-- Record backtrace information for each allocation
-- Track allocation generation changes
-
-## Performance
-
-- **Fast Allocation**: Small object allocation has O(1) time complexity
-- **Memory Efficiency**: Buddy algorithm effectively reduces external fragmentation
-- **Auto Merge**: Freed pages automatically merge to reduce fragmentation
+## Public API summary
+
+- `GlobalAllocator`
+ High-level allocator facade that can also implement `GlobalAlloc`, and supports `add_region`, `managed_section_count`, `managed_section`, `managed_bytes`, and `allocated_bytes`.
+- `BuddyAllocator`
+ Standalone multi-section page allocator, supporting `init`, `add_region`, section queries, `managed_bytes`, and `allocated_bytes`.
+- `ManagedSection`
+ Read-only summary for one managed section.
+- `SlabAllocator`
+ Standalone slab allocator.
+- `SizeClass`
+ Fixed object size classes used by slab.
+- `SlabAllocResult`
+ `Allocated(ptr)` or `NeedsSlab { size_class, pages }`.
+- `SlabDeallocResult`
+ `Done` or `FreeSlab { base, pages }`.
+- `OsImpl`
+ Provides `current_cpu_idx()` and `virt_to_phys()` for per-CPU routing and lowmem selection.
+
+`managed_bytes` counts only allocatable heap bytes and excludes region-prefix metadata.
+`allocated_bytes` is backend page occupancy, not the exact sum of requested `layout.size()`.
## Testing
-Run the test suite:
-
```bash
-# Run all tests
-cargo test --package buddy-slab-allocator
-
-# Run tests with logging enabled
-cargo test --package buddy-slab-allocator --features log
+# Run normal tests
+cargo test
-# Run tests with tracking enabled
-cargo test --package buddy-slab-allocator --features tracking
-```
+# Run tests serially
+cargo test -- --test-threads=1
-## Benchmarking
+# Run ignored stress tests
+cargo test --test stress_test -- --ignored --nocapture
-This project includes comprehensive benchmarks to evaluate performance and stability under various conditions. For detailed instructions, see `benches/README.md`.
+# Check benchmarks compile
+cargo check --benches
-```bash
-# Run all benchmarks
+# Run benchmarks
cargo bench
```
-For detailed usage instructions, refer to `benches/README.md`.
-
-## Documentation
-
-API documentation is available on [docs.rs](https://docs.rs/buddy-slab-allocator).
-
-To build and view documentation locally:
-
-```bash
-cargo doc --no-deps --open
-```
+More test notes are in [tests/README.md](tests/README.md).
## License
-This project is licensed under:
-
-- **GPL-3.0-or-later** OR
-- **Apache-2.0** OR
-- **MIT**
-
-You may choose any of these licenses for your use.
-
-## Contributing
-
-Contributions are welcome! Please feel free to submit a Pull Request.
-
-## Repository
-
-[https://github.com/arceos-hypervisor/buddy-slab-allocator](https://github.com/arceos-hypervisor/buddy-slab-allocator)
-
-## Chinese Documentation
-
-中文文档请查看 [README_CN.md](README_CN.md)
+Licensed under [Apache-2.0](LICENSE).
diff --git a/README_CN.md b/README_CN.md
index bf9e49b..1c46fd2 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -1,191 +1,230 @@
# buddy-slab-allocator 内存分配器
-一个高效的页级和字节级内存分配器，为嵌入式/内核环境设计。
+这是一个面向内核和嵌入式环境的 `no_std` 两级内存分配器，结合了 buddy 页分配器与 per-CPU slab 小对象分配器。
-## 特性
+## 总览
-- **Buddy 页分配器**：页级内存分配
-- **Slab 字节分配器**：小对象分配
-- **复合页分配器**：统一的多区域页分配接口
-- **全局分配器**：协调页分配器和字节分配器，提供统一的分配接口
-- **零 `std` 依赖**：完全 `#![no_std]`，适合嵌入式和内核环境
-- **条件日志**：支持 `log` feature 启用日志，默认无依赖
-- **内存追踪**：支持 `tracking` feature 收集详细统计信息
+当前实现由三层组成：
-## 快速开始
+1. `BuddyAllocator`
+ 管理一个或多个虚拟内存 section，支持按 2 的幂分裂与合并。
+2. `SlabAllocator`
+ 管理 `<= 2048` 字节的小对象，采用固定 size class。
+3. `GlobalAllocator`
+ 将两者组合起来，对小对象走 per-CPU slab，对大对象走 buddy 页分配。
-### 添加依赖
+更完整的设计细节见 [docs/design.md](docs/design.md)。
-在 `Cargo.toml` 中添加：
+## 架构图
-```toml
-[dependencies]
-buddy-slab-allocator = "0.1.0"
+```mermaid
+flowchart TD
+ GA[GlobalAllocator] --> B[SpinMutex]
+ GA --> OS[OsImpl]
+ GA --> PCS[per_cpu_slabs: *mut SpinMutex[]]
-# 可选功能
-buddy-slab-allocator = { version = "0.1.0", features = ["log"] } # 启用日志
-buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] } # 启用追踪
+ B --> PM[PageMeta[]]
+ B --> FL[按 order 的 free_lists]
+
+ PCS --> SA[SlabAllocator]
+ SA --> SC[9 个 SlabCache]
+ SC --> P[partial]
+ SC --> F[full]
+ SC --> E[empty]
+ SC --> H[SlabPageHeader]
```
-### 基本使用
+### 分配路由
+
+```mermaid
+flowchart TD
+ A[GlobalAllocator::alloc(layout)] --> B{size <= 2048 且 align <= 2048?}
+ B -- 是 --> C[走当前 CPU 的 slab allocator]
+ C --> D{slab 直接命中?}
+ D -- 是 --> E[返回对象指针]
+ D -- 否 --> F[从 buddy 申请新 slab 页]
+ F --> G[add_slab 后重试]
+ G --> E
+ B -- 否 --> H[直接走 buddy 页分配]
+ H --> I[返回页级指针]
+```
-#### 使用全局分配器
+### 跨 CPU 释放
-```rust
-use buddy_slab_allocator::GlobalAllocator;
-use core::alloc::Layout;
+```mermaid
+sequenceDiagram
+ participant CPU1 as 当前 CPU
+ participant H as SlabPageHeader
+ participant CPU0 as owner CPU
+ participant S as owner SlabAllocator
-// 创建全局分配器
-let mut global = GlobalAllocator::new();
+ CPU1->>H: remote_free(obj_addr)
+ Note right of H: 无锁 CAS 推入 remote_free_head
+ CPU1-->>CPU1: 立即返回
-// 使用内存区域初始化全局分配器
-let heap_start = 0x8000_0000;
-let heap_size = 16 * 1024 * 1024; // 16MB
-global.init(heap_start, heap_size).unwrap();
+ CPU0->>S: 后续本地 alloc/dealloc
+ S->>H: drain_remote_frees()
+ H-->>S: 远程释放对象回收到本地 bitmap
+```
-// 或添加多个内存池
-global.add_memory(0x80000000, 0x1000000).unwrap();
-global.add_memory(0x81000000, 0x1000000).unwrap();
+## 特性
-// 小于 2048 byte 的小对象分配（自动使用 Slab）
-let small_layout = Layout::from_size_align(64, 8).unwrap();
-let small_ptr = global.alloc(small_layout).unwrap();
+- Buddy 页分配，支持拆分与合并
+- 支持通过 `add_region` 动态追加可管理 region
+- Slab 小对象分配，固定 9 个 size class：`8..=2048`
+- per-CPU slab cache
+- 跨 CPU 释放走 lock-free remote free
+- 支持 `alloc_pages_lowmem` 低地址页分配
+- 适合 `no_std`
+- 内置 `log` 日志支持
+- 可分别独立使用 `BuddyAllocator` 与 `SlabAllocator`
-// 大对象分配（自动使用页分配器）
-let large_layout = Layout::from_size_align(0x1000, 0x1000).unwrap();
-let large_ptr = global.alloc(large_layout).unwrap();
+## 添加依赖
-// 释放内存
-global.dealloc(small_ptr, small_layout);
-global.dealloc(large_ptr, large_layout);
+```toml
+[dependencies]
+buddy-slab-allocator = "0.2.0"
```
-#### 直接使用页分配器
+## 使用 `GlobalAllocator`
```rust
-use buddy_slab_allocator::CompositePageAllocator;
+use buddy_slab_allocator::{GlobalAllocator, OsImpl};
+use core::alloc::Layout;
const PAGE_SIZE: usize = 0x1000;
-let mut page_alloc = CompositePageAllocator::::new();
-// 使用内存区域初始化
-page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
+struct DemoOs;
-// 分配页
-let addr = page_alloc.alloc_pages(4, PAGE_SIZE).unwrap();
-// 使用分配的内存...
-page_alloc.dealloc_pages(addr, 4);
-```
-
-#### 直接使用 Slab 分配器
+impl OsImpl for DemoOs {
+ fn current_cpu_idx(&self) -> usize { 0 }
+ fn virt_to_phys(&self, vaddr: usize) -> usize { vaddr }
+}
-```rust
-use buddy_slab_allocator::SlabByteAllocator;
-use buddy_slab_allocator::page_allocator::PageAllocatorForSlab;
-use buddy_slab_allocator::CompositePageAllocator;
-use core::alloc::Layout;
+static OS: DemoOs = DemoOs;
-const PAGE_SIZE: usize = 0x1000;
-let mut page_alloc = CompositePageAllocator::::new();
-page_alloc.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
+let allocator = GlobalAllocator::::new();
+let region_start = 0x8000_0000 as *mut u8;
+let region_size = 16 * 1024 * 1024;
+let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) };
-let mut slab_alloc = SlabByteAllocator::::new();
+unsafe {
+ allocator.init(region, 1, &OS).unwrap();
+}
-// 小对象分配快速
let layout = Layout::from_size_align(64, 8).unwrap();
-let ptr = slab_alloc.alloc(&mut page_alloc, layout).unwrap();
+let ptr = allocator.alloc(layout).unwrap();
-// 释放内存
-slab_alloc.dealloc(&mut page_alloc, ptr, layout);
-```
+unsafe {
+ allocator.dealloc(ptr, layout);
+}
-## 特性详解
+let extra_region_start = 0x9000_0000 as *mut u8;
+let extra_region_size = 8 * 1024 * 1024;
+let extra_region = unsafe {
+ core::slice::from_raw_parts_mut(extra_region_start, extra_region_size)
+};
-### 条件日志
-
-通过 `log` feature 启用日志功能：
-
-```toml
-buddy-slab-allocator = { version = "0.1.0", features = ["log"] }
+unsafe {
+ allocator.add_region(extra_region).unwrap();
+}
```
-启用后可使用标准 `log` crate 的宏记录分配事件：
+## 分别使用 Buddy 与 Slab
+
+如果需要更底层的控制，也可以分别使用这两个组件。
```rust
-log::info!("分配内存于 {:x}", addr);
-```
+use buddy_slab_allocator::{
+ BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult,
+};
+use core::alloc::Layout;
-未启用时，日志调用会被编译为空操作，零运行时开销。
+const PAGE_SIZE: usize = 0x1000;
+let region_start = 0x8000_0000 as *mut u8;
+let region_size = 16 * 1024 * 1024;
+let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) };
-### 内存追踪
+let mut buddy = BuddyAllocator::::new();
+unsafe {
+ buddy.init(region, None).unwrap();
+}
-通过 `tracking` feature 启用详细的内存使用追踪：
+let mut slab = SlabAllocator::::new();
+let layout = Layout::from_size_align(64, 8).unwrap();
-```toml
-buddy-slab-allocator = { version = "0.1.0", features = ["tracking"] }
+let ptr = loop {
+ match slab.alloc(layout).unwrap() {
+ SlabAllocResult::Allocated(ptr) => break ptr,
+ SlabAllocResult::NeedsSlab { size_class, pages } => {
+ let slab_bytes = pages * PAGE_SIZE;
+ let addr = buddy.alloc_pages(pages, slab_bytes).unwrap();
+ slab.add_slab(size_class, addr, slab_bytes, 0);
+ }
+ }
+};
+
+match slab.dealloc(ptr, layout) {
+ SlabDeallocResult::Done => {}
+ SlabDeallocResult::FreeSlab { base, pages } => {
+ buddy.dealloc_pages(base, pages);
+ }
+}
+
+let extra_region_start = 0x9000_0000 as *mut u8;
+let extra_region_size = 8 * 1024 * 1024;
+let extra_region = unsafe {
+ core::slice::from_raw_parts_mut(extra_region_start, extra_region_size)
+};
+
+unsafe {
+ buddy.add_region(extra_region).unwrap();
+}
```
-启用后可以：
-- 收集每种内存用途的字节数统计
-- 记录每次分配的回溯信息
-- 跟踪分配代际变化
-
-## 性能特性
-
-- **快速分配**：小对象分配 O(1) 时间复杂度
-- **内存效率**：Buddy 算法有效减少外部碎片
-- **自动合并**：释放的页面自动合并，减少碎片
+## 公开 API 摘要
+
+- `GlobalAllocator`
+ 高层门面，也可以作为 `GlobalAlloc` 使用，并支持 `add_region`、`managed_section_count`、`managed_section`、`managed_bytes` 与 `allocated_bytes`。
+- `BuddyAllocator`
+ 独立多 section 页分配器，支持 `init`、`add_region`、section 查询、`managed_bytes` 与 `allocated_bytes`。
+- `ManagedSection`
+ 单个 managed section 的只读摘要。
+- `SlabAllocator`
+ 独立 slab 分配器。
+- `SizeClass`
+ slab 使用的固定对象尺寸类。
+- `SlabAllocResult`
+ `Allocated(ptr)` 或 `NeedsSlab { size_class, pages }`。
+- `SlabDeallocResult`
+ `Done` 或 `FreeSlab { base, pages }`。
+- `OsImpl`
+ 提供 `current_cpu_idx()` 和 `virt_to_phys()`，用于 per-CPU 路由与 lowmem 选择。
+
+`managed_bytes` 只统计可分配 heap，不包含 region 前缀 metadata。
+`allocated_bytes` 表示后端页占用，不是用户请求的 `layout.size()` 精确求和。
## 测试
-运行测试套件：
-
```bash
-# 运行所有测试
-cargo test --package buddy-slab-allocator
-
-# 启用日志运行测试
-cargo test --package buddy-slab-allocator --features log
+# 常规测试
+cargo test
-# 启用追踪运行测试
-cargo test --package buddy-slab-allocator --features tracking
-```
+# 串行执行，便于排查问题
+cargo test -- --test-threads=1
-## 性能测试
+# 运行忽略的压力测试
+cargo test --test stress_test -- --ignored --nocapture
-此项目包含全面的性能测试，用于在各种条件下评估分配器的性能和稳定性。详细的中文说明请查看 `benches/README_CN.md`。
+# 检查 benchmark 是否可编译
+cargo check --benches
-```bash
-# 运行所有 benchmark
+# 运行 benchmark
cargo bench
```
-详细使用方法请参考 `benches/README_CN.md`。
-
-## 文档
-
-API 文档可在 [docs.rs](https://docs.rs/buddy-slab-allocator) 上查看。
-
-在本地构建和查看文档：
-
-```bash
-cargo doc --no-deps --open
-```
+更多测试说明见 [tests/README.md](tests/README.md)。
## 许可证
-本项目采用以下许可证：
-
-- **GPL-3.0-or-later** 或
-- **Apache-2.0** 或
-- **MIT**
-
-你可以选择其中任何一种许可证使用。
-
-## 贡献
-
-欢迎贡献！请随时提交 Pull Request。
-
-## 仓库
-
-[https://github.com/arceos-hypervisor/buddy-slab-allocator](https://github.com/arceos-hypervisor/buddy-slab-allocator)
+采用 [Apache-2.0](LICENSE) 许可证。
diff --git a/benches/README_CN.md b/benches/README_CN.md
index fe6b470..fa76b8a 100644
--- a/benches/README_CN.md
+++ b/benches/README_CN.md
@@ -1,284 +1,45 @@
# Benchmark 使用说明
-本目录包含 buddy-slab-allocator 项目的性能测试和稳定性测试。
+本目录包含基于 `divan` 的 benchmark 套件，围绕当前 crate 的三层结构组织：
-## 运行 Benchmark
+- `buddy_allocator.rs`
+ `BuddyAllocator` 的页分配、对齐分配、碎片恢复、随机页 workload
+- `slab_allocator.rs`
+ `SlabAllocator` 的 size class alloc/free、hot reuse、mixed-size batch、steady-state recycle
+- `global_allocator.rs`
+ `GlobalAllocator` 的小对象、大对象、页接口、混合 workload、cross-CPU free cycle
-```bash
-cargo bench
-```
-
-### 运行特定 benchmark suite
-
-```bash
-# 测试全局分配器
-cargo bench --bench global_allocator
-
-# 测试 Buddy 页分配器
-cargo bench --bench buddy_allocator
-
-# 测试 Slab 字节分配器
-cargo bench --bench slab_allocator
+共享的 host-side harness 放在 `common.rs`，负责：
-# 测试稳定性
-cargo bench --bench stability
-```
+- region / metadata 分配
+- buddy / slab / global 初始化
+- 固定随机种子
+- mock `OsImpl`
-### 高级选项
+## 运行方式
```bash
-# 启用 memory tracking 功能运行 benchmark
-cargo bench --features tracking
-
-# 保存基线用于后续对比
-cargo bench -- --save-baseline main
-
-# 与基线对比
-cargo bench -- --baseline main
-
-# 只运行特定 benchmark
-cargo bench global_alloc_small
-```
-
-## Benchmark Suite 说明
-
-### 1. Global Allocator Benchmarks (`global_allocator.rs`)
-
-测试全局分配器的统一接口，自动在 Slab 和 Buddy 分配器之间路由。
-
-**测试内容**：
-- **小对象分配** (8-1024 字节)：使用 Slab 分配器
-- **大对象分配** (>2048 字节)：使用 Buddy 分配器
-- **分配/释放循环**：模拟真实使用场景
-- **随机分配**：不同大小和模式的测试
-- **混合模式**：交替进行小对象和大对象分配
-- **页分配**：直接进行页级分配
-
-**测试目标**：
-- 验证自动路由机制的效率
-- 评估不同大小分配的性能
-- 测试实际使用场景下的表现
-
-### 2. Buddy Allocator Benchmarks (`buddy_allocator.rs`)
-
-专门测试 Buddy 页分配器的页级内存分配功能。
-
-**测试内容**：
-- **单页/多页分配**：不同分配大小的测试
-- **带合并的释放**：自动块合并的效率
-- **对齐要求**：不同对齐约束的测试
-- **碎片化抗性**：长期碎片化行为测试
-- **随机模式**：随机分配/释放的压力测试
-- **复合分配器**：连续块组合测试
-- **统计查询**：统计信息查询的性能
-
-**测试目标**：
-- 验证 Buddy 算法的性能特征
-- 测试自动合并机制的有效性
-- 评估碎片化程度
-
-### 3. Slab Allocator Benchmarks (`slab_allocator.rs`)
-
-专门测试 Slab 字节分配器的小对象分配优化。
-
-**测试内容**：
-- **大小类别**：所有支持的分配大小（8-2048 字节）
-- **释放性能**：小对象的释放速度
-- **对齐支持**：各种对齐要求的测试
-- **随机分配**：不同大小模式的测试
-- **对象池**：重用效率测试
-- **混合大小**：多个大小类别的交互测试
-- **压力测试**：高容量分配压力测试
-- **内存压力**：接近耗尽时的行为测试
-
-**测试目标**：
-- 验证小对象分配的 O(1) 时间复杂度
-- 测试不同大小类别的性能
-- 评估对象池重用的效率
-
-### 4. Stability Benchmarks (`stability.rs`)
-
-全面的稳定性和压力测试，验证分配器在极端条件下的表现。
-
-**测试内容**：
-- **随机模式稳定性**：长期运行的随机分配/释放
-- **耗尽处理**：内存限制下的优雅失败
-- **碎片化抗性**：碎片化下的持续使用
-- **交替模式**：特定的分配/释放序列
-- **长时间运行**：扩展持续时间测试
-- **快速混合大小**：不同大小的压力测试
-- **页分配压力**：页级分配的压力测试
-- **内存泄漏检测**：验证没有内存泄漏
-- **边界情况**：最小/最大大小、奇数对齐
-
-**测试目标**：
-- 验证长时间运行的稳定性
-- 检测内存泄漏
-- 测试极限条件下的行为
-- 确保正确处理边界情况
-
-## Benchmark 结果解读
-
-### 结果位置
-
-运行 benchmark 后，Criterion 会生成详细的 HTML 格式报告：
-
-- **位置**：`target/criterion/report/index.html`
-- **内容**：性能图表、统计分析、对比基线
-
-### 报告内容
-
-Criterion 生成的报告包含以下信息：
+# 仅检查 benchmark 是否可编译
+cargo check --benches
-- **平均时间**：执行的平均耗时
-- **标准差**：性能波动程度
-- **中位数**：典型性能（排除异常值）
-- **最小值/最大值**：性能范围
-- **样本大小**：迭代次数
-- **性能图表**：随时间变化的可视化图表
-- **对比分析**：与基线或之前运行的对比
-
-### 关键性能指标
-
-**需要关注的性能指标**：
-
-1. **分配速度**：
- - 不同大小类别的分配时间
- - 最佳和最差情况
- - 性能一致性
-
-2. **释放开销**：
- - 释放操作的时间
- - Buddy 系统的合并开销
- - Slab 系统的重用效率
-
-3. **碎片化影响**：
- - 长期运行后的性能变化
- - 内存利用率
- - 大块分配成功率
-
-4. **可扩展性**：
- - 增加负载时的性能变化
- - 并发性能（如果测试）
- - 极限容量测试
-
-**稳定性指标**：
-
-1. **一致性**：
- - 多次运行结果的一致性
- - 标准差大小
- - 无明显性能下降
-
-2. **内存泄漏**：
- - 使用 `tracking` feature 验证
- - 长时间运行后内存占用稳定
- - 释放后内存完全回收
-
-3. **错误处理**：
- - 内存耗尽时的优雅失败
- - 边界情况的正确处理
- - 无崩溃或 panic
-
-## 自定义 Benchmark
-
-### 修改测试参数
-
-可以在 `benches/` 目录下的源文件中修改 benchmark 参数：
-
-- `HEAP_SIZE`：测试堆大小（默认：16MB）
-- `PAGE_SIZE`：页大小（默认：4096 字节）
-- 迭代次数：根据硬件调整
-
-### 添加新测试
-
-1. 在对应的 benchmark 文件中添加新的测试函数
-2. 使用 `criterion` 提供的宏和类型
-3. 在 `criterion_group!` 中注册新测试
-4. 在 `criterion_main!` 中包含新测试组
-
-### 示例：添加新的 benchmark
-
-```rust
-use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId};
-
-fn bench_my_test(c: &mut Criterion) {
- let sizes = [64, 128, 256];
-
- let mut group = c.benchmark_group("my_test");
-
- for size in sizes.iter() {
- group.bench_with_input(
- BenchmarkId::from_parameter(size),
- size,
- |b, _| {
- b.iter(|| {
- // 你的测试代码
- black_box(size)
- })
- }
- );
- }
-
- group.finish();
-}
-
-criterion_group!(benches, bench_my_test);
-criterion_main!(benches);
-```
-
-## 性能优化建议
-
-根据 benchmark 结果，可以：
-
-1. **优化热点路径**：找到最耗时的操作进行优化
-2. **减少碎片化**：调整分配策略或增加内存池
-3. **改进缓存友好性**：优化数据结构和访问模式
-4. **并行化**：如果合适，添加并发支持
-
-## 常见问题
-
-### Q: Benchmark 编译失败怎么办？
-
-A: 最常见的原因是 Rust 版本不兼容。请确保使用 **Rust 1.93.0 或更高版本**.
-
-如果仍然失败：
-1. 清理并重新构建：`cargo clean && cargo build`
-2. 检查网络连接和依赖下载
-3. 更新 rustup：`rustup update`
-
-### Q: Benchmark 运行时间太长怎么办？
-
-A: 可以：
-1. 减少迭代次数
-2. 只运行特定的 benchmark
-3. 调整 `HEAP_SIZE` 减小测试范围
-
-### Q: 如何比较不同版本的性能？
-
-A: 使用 Criterion 的基线功能：
-```bash
-# 第一次运行，保存基线
-cargo bench -- --save-baseline v1
+# 运行全部 benchmark
+cargo bench
-# 修改代码后，对比基线
-cargo bench -- --baseline v1
+# 单独运行某个 suite
+cargo bench --bench buddy_allocator
+cargo bench --bench slab_allocator
+cargo bench --bench global_allocator
```
-### Q: Benchmark 结果不稳定怎么办？
-
-A: 可能原因：
-1. 系统负载过高
-2. 频率调整或电源管理
-3. 后台程序干扰
+## 设计原则
-建议：
-1. 关闭不必要的程序
-2. 多次运行取平均值
-3. 使用固定 CPU 频率
+- 使用 `divan::Bencher` 和 `divan::black_box`
+- 不再保留旧的 `criterion` 代码路径
+- benchmark 只使用当前公开 API，不依赖历史类型名
+- 尽量让每次迭代在闭环内恢复 allocator 状态，减少跨迭代污染
+- workload 使用固定模式或固定随机种子，便于复现
-## 参考文献
+## CI 策略
-- [Criterion.rs 文档](https://bheisler.github.io/criterion.rs/book/)
-- [Rust 性能优化指南](https://nnethercote.github.io/perf-book/)
-- [Buddy 算法原理](https://en.wikipedia.org/wiki/Buddy_memory_allocation)
-- [Slab 分配器设计](https://en.wikipedia.org/wiki/Slab_allocation)
+CI 仅执行 `cargo check --benches`，确保 benchmark 工程持续可编译。
+真实性能测量和结果对比保留在本地执行。
diff --git a/benches/buddy_allocator.rs b/benches/buddy_allocator.rs
index b156a49..82c745b 100644
--- a/benches/buddy_allocator.rs
+++ b/benches/buddy_allocator.rs
@@ -1,288 +1,105 @@
-//! Benchmarks for BuddyPageAllocator - page-level memory allocation
-//!
-//! This benchmark suite tests the performance and stability of the Buddy allocator
-//! which handles page-level allocations with automatic merging.
+//! Benchmarks for the buddy page allocator.
-use buddy_slab_allocator::{
- BaseAllocator, BuddyPageAllocator, CompositePageAllocator, PageAllocator,
-};
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
-use rand::{Rng, SeedableRng};
-use std::alloc::Layout;
-use std::alloc::{alloc, dealloc};
-
-const PAGE_SIZE: usize = 0x1000;
-const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB
+mod common;
-/// Allocate a test heap from the system allocator
-fn alloc_test_heap(size: usize) -> (*mut u8, Layout) {
- let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap();
- let ptr = unsafe { alloc(layout) };
- assert!(!ptr.is_null(), "Failed to allocate test heap");
- (ptr, layout)
-}
+use common::{
+ BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE, seeded_rng,
+};
+use divan::{Bencher, black_box};
+use rand::RngExt;
-/// Deallocate the test heap
-fn dealloc_test_heap(ptr: *mut u8, layout: Layout) {
- unsafe { dealloc(ptr, layout) };
+fn main() {
+ divan::main();
}
-/// Benchmark single page allocation
-fn bench_single_page_alloc(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
+#[divan::bench_group]
+mod buddy {
+ use super::*;
- c.bench_function("buddy_single_page_alloc", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
+ #[divan::bench(args = [1usize, 2, 4, 16, 64])]
+ fn page_alloc_free(bencher: Bencher, pages: usize) {
+ let mut harness = BuddyHarness::new(HEAP_SIZE);
- b.iter(|| {
- let addr = allocator.alloc_pages(black_box(1), PAGE_SIZE);
+ bencher.bench_local(|| {
+ let addr = harness
+ .allocator
+ .alloc_pages(black_box(pages), PAGE_SIZE)
+ .unwrap();
+ harness.allocator.dealloc_pages(addr, pages);
black_box(addr)
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark multiple page allocations
-fn bench_multi_page_alloc(c: &mut Criterion) {
- let page_counts = [1, 2, 4, 8, 16, 32, 64];
-
- let mut group = c.benchmark_group("buddy_multi_page_alloc");
-
- for num_pages in page_counts.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- group.bench_with_input(BenchmarkId::from_parameter(num_pages), num_pages, |b, _| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- b.iter(|| {
- let addr = allocator.alloc_pages(black_box(*num_pages), PAGE_SIZE);
- black_box(addr)
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
-
-/// Benchmark deallocation with automatic merging
-fn bench_dealloc_with_merge(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("buddy_dealloc_with_merge", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- // Pre-allocate multiple buddy blocks that will merge
- let mut addrs = Vec::new();
- for _ in 0..64 {
- let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap();
- addrs.push(addr);
- }
+ #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4, PAGE_SIZE * 16])]
+ fn aligned_page_alloc_free(bencher: Bencher, align: usize) {
+ let mut harness = BuddyHarness::new(HEAP_SIZE);
- b.iter(|| {
- // Deallocate and allocate to test merging efficiency
- let addr = addrs.pop().unwrap();
- allocator.dealloc_pages(black_box(addr), 1);
- // Allocate again to reuse merged blocks
- let new_addr = allocator.alloc_pages(1, PAGE_SIZE);
- addrs.push(new_addr.unwrap());
- });
-
- // Cleanup remaining allocations
- for addr in addrs {
- allocator.dealloc_pages(addr, 1);
- }
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark allocation with different alignments
-fn bench_alloc_with_alignment(c: &mut Criterion) {
- let alignments = [PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4, PAGE_SIZE * 8];
-
- let mut group = c.benchmark_group("buddy_alloc_with_alignment");
-
- for alignment in alignments.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- group.bench_with_input(BenchmarkId::from_parameter(alignment), alignment, |b, _| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- b.iter(|| {
- let addr = allocator.alloc_pages(black_box(4), black_box(*alignment));
- black_box(addr)
- });
+ bencher.bench_local(|| {
+ let addr = harness
+ .allocator
+ .alloc_pages(black_box(4), black_box(align))
+ .unwrap();
+ harness.allocator.dealloc_pages(addr, 4);
+ black_box(addr)
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
-
-/// Benchmark fragmentation resistance
-fn bench_fragmentation(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("buddy_fragmentation", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
+ #[divan::bench]
+ fn fragmentation_recovery_cycle(bencher: Bencher) {
+ let mut harness = BuddyHarness::new(HEAP_SIZE);
- b.iter(|| {
- // Allocate many small blocks, then free half
- let mut addrs = Vec::new();
- for _ in 0..512 {
- let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap();
- addrs.push(addr);
+ bencher.bench_local(|| {
+ let mut addrs = Vec::with_capacity(FRAGMENTATION_PAGES);
+ for _ in 0..FRAGMENTATION_PAGES {
+ addrs.push(harness.allocator.alloc_pages(1, PAGE_SIZE).unwrap());
}
- // Free every other allocation
- for i in (0..512).step_by(2) {
- allocator.dealloc_pages(addrs[i], 1);
+ for idx in (0..addrs.len()).step_by(2) {
+ harness.allocator.dealloc_pages(addrs[idx], 1);
}
- // Try to allocate a large block - should succeed if fragmentation is low
- let large_layout = Layout::from_size_align(PAGE_SIZE * 64, PAGE_SIZE).unwrap();
- let large_addr = allocator.alloc_pages(64, PAGE_SIZE);
+ let large = harness.allocator.alloc_pages(64, PAGE_SIZE).unwrap();
+ harness.allocator.dealloc_pages(large, 64);
- // Cleanup
- if let Ok(addr) = large_addr {
- allocator.dealloc_pages(addr, 64);
+ for idx in (1..addrs.len()).step_by(2) {
+ harness.allocator.dealloc_pages(addrs[idx], 1);
}
- for i in (1..512).step_by(2) {
- allocator.dealloc_pages(addrs[i], 1);
- }
- });
- });
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark random allocation pattern
-fn bench_random_pattern(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("buddy_random_pattern", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- let mut allocated = Vec::new();
+ black_box(large)
+ });
+ }
- // Random alloc/dealloc pattern
- for _ in 0..1000 {
- if allocated.is_empty() || rng.gen_bool(0.7) {
- // Allocate
- let pages = 1 << rng.gen_range(0..5); // 1, 2, 4, 8, 16 pages
- match allocator.alloc_pages(pages, PAGE_SIZE) {
- Ok(addr) => allocated.push((addr, pages)),
- Err(_) => break, // Out of memory
+ #[divan::bench]
+ fn random_page_workload(bencher: Bencher) {
+ let mut harness = BuddyHarness::new(HEAP_SIZE);
+ let mut rng = seeded_rng();
+ let plan: Vec<(bool, usize, usize)> = (0..OPERATIONS_PER_BATCH)
+ .map(|_| {
+ let allocate = rng.random_bool(0.65);
+ let pages = 1usize << rng.random_range(0..=4);
+ let free_hint = rng.random_range(0..OPERATIONS_PER_BATCH.max(1));
+ (allocate, pages, free_hint)
+ })
+ .collect();
+
+ bencher.bench_local(|| {
+ let mut active = Vec::new();
+
+ for &(allocate, pages, free_hint) in &plan {
+ if allocate || active.is_empty() {
+ if let Ok(addr) = harness.allocator.alloc_pages(pages, PAGE_SIZE) {
+ active.push((addr, pages));
}
} else {
- // Deallocate random item
- let idx = rng.gen_range(0..allocated.len());
- let (addr, pages) = allocated.swap_remove(idx);
- allocator.dealloc_pages(addr, pages);
+ let idx = free_hint % active.len();
+ let (addr, count) = active.swap_remove(idx);
+ harness.allocator.dealloc_pages(addr, count);
}
}
- // Cleanup
- for (addr, pages) in allocated {
- allocator.dealloc_pages(addr, pages);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark CompositePageAllocator with contiguous block combination
-fn bench_composite_allocator(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("composite_allocator", |b| {
- let mut allocator = CompositePageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- b.iter(|| {
- // Allocate and deallocate pages
- for _ in 0..100 {
- let addr = allocator.alloc_pages(1, PAGE_SIZE);
- if let Ok(a) = addr {
- allocator.dealloc_pages(a, 1);
- }
+ for (addr, count) in active {
+ harness.allocator.dealloc_pages(addr, count);
}
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark allocate at specific address
-fn bench_alloc_at(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("buddy_alloc_at", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- // Pre-allocate to create space for specific allocation
- let addr1 = allocator.alloc_pages(1, PAGE_SIZE).unwrap();
- allocator.dealloc_pages(addr1, 1);
-
- b.iter(|| {
- // Try to allocate at a specific address
- let target = addr1;
- let addr = allocator.alloc_pages_at(black_box(target), 1, PAGE_SIZE);
- black_box(addr)
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark statistics retrieval
-fn bench_statistics(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("buddy_statistics", |b| {
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE);
-
- // Make some allocations
- for _ in 0..10 {
- let _ = allocator.alloc_pages(1, PAGE_SIZE);
- }
-
- b.iter(|| {
- let total = allocator.total_pages();
- let used = allocator.used_pages();
- let available = allocator.available_pages();
- black_box((total, used, available))
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
+ }
}
-
-criterion_group!(
- benches,
- bench_single_page_alloc,
- bench_multi_page_alloc,
- bench_dealloc_with_merge,
- bench_alloc_with_alignment,
- bench_fragmentation,
- bench_random_pattern,
- bench_composite_allocator,
- bench_alloc_at,
- bench_statistics
-);
-criterion_main!(benches);
diff --git a/benches/common.rs b/benches/common.rs
new file mode 100644
index 0000000..075f203
--- /dev/null
+++ b/benches/common.rs
@@ -0,0 +1,164 @@
+#![allow(dead_code)]
+
+use buddy_slab_allocator::{
+ BuddyAllocator, GlobalAllocator, OsImpl, SlabAllocResult, SlabAllocator, SlabDeallocResult,
+};
+use core::alloc::Layout;
+use rand::{SeedableRng, rngs::StdRng};
+use std::alloc::{alloc, dealloc};
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+pub const PAGE_SIZE: usize = 0x1000;
+pub const HEAP_SIZE: usize = 64 * 1024 * 1024;
+pub const OPERATIONS_PER_BATCH: usize = 256;
+pub const FRAGMENTATION_PAGES: usize = 512;
+
+const REGION_ALIGN: usize = 64 * 1024;
+
+pub struct HostRegion {
+ ptr: *mut u8,
+ layout: Layout,
+}
+
+impl HostRegion {
+ pub fn new(size: usize) -> Self {
+ let layout = Layout::from_size_align(size, REGION_ALIGN).unwrap();
+ let ptr = unsafe { alloc(layout) };
+ assert!(!ptr.is_null(), "failed to allocate host region");
+ Self { ptr, layout }
+ }
+
+ pub fn addr(&self) -> usize {
+ self.ptr as usize
+ }
+
+ pub fn as_mut_slice(&mut self) -> &mut [u8] {
+ unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) }
+ }
+}
+
+impl Drop for HostRegion {
+ fn drop(&mut self) {
+ unsafe { dealloc(self.ptr, self.layout) };
+ }
+}
+
+pub struct MockOs {
+ cpu: AtomicUsize,
+}
+
+impl MockOs {
+ pub const fn new() -> Self {
+ Self {
+ cpu: AtomicUsize::new(0),
+ }
+ }
+
+ pub fn set_cpu(&self, cpu: usize) {
+ self.cpu.store(cpu, Ordering::Relaxed);
+ }
+}
+
+impl OsImpl for MockOs {
+ fn current_cpu_idx(&self) -> usize {
+ self.cpu.load(Ordering::Relaxed)
+ }
+
+ fn virt_to_phys(&self, vaddr: usize) -> usize {
+ vaddr
+ }
+}
+
+pub static MOCK_OS: MockOs = MockOs::new();
+
+pub fn seeded_rng() -> StdRng {
+ StdRng::from_seed([0; 32])
+}
+
+pub struct BuddyHarness {
+ _region: HostRegion,
+ pub allocator: BuddyAllocator,
+}
+
+impl BuddyHarness {
+ pub fn new(heap_size: usize) -> Self {
+ let region_size =
+ heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4;
+ let mut region = HostRegion::new(region_size);
+ let mut allocator = BuddyAllocator::::new();
+ unsafe {
+ allocator.init(region.as_mut_slice(), None).unwrap();
+ }
+ Self {
+ _region: region,
+ allocator,
+ }
+ }
+}
+
+pub struct SlabHarness {
+ _region: HostRegion,
+ buddy: BuddyAllocator,
+ slab: SlabAllocator,
+}
+
+impl SlabHarness {
+ pub fn new(heap_size: usize) -> Self {
+ let region_size =
+ heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4;
+ let mut region = HostRegion::new(region_size);
+ let mut buddy = BuddyAllocator::::new();
+ unsafe {
+ buddy.init(region.as_mut_slice(), None).unwrap();
+ }
+ Self {
+ _region: region,
+ buddy,
+ slab: SlabAllocator::new(),
+ }
+ }
+
+ pub fn alloc(&mut self, layout: Layout) -> core::ptr::NonNull {
+ loop {
+ match self.slab.alloc(layout).unwrap() {
+ SlabAllocResult::Allocated(ptr) => return ptr,
+ SlabAllocResult::NeedsSlab { size_class, pages } => {
+ let slab_bytes = pages * PAGE_SIZE;
+ let base = self.buddy.alloc_pages(pages, slab_bytes).unwrap();
+ self.slab.add_slab(size_class, base, slab_bytes, 0);
+ }
+ }
+ }
+ }
+
+ pub fn dealloc(&mut self, ptr: core::ptr::NonNull, layout: Layout) {
+ match self.slab.dealloc(ptr, layout) {
+ SlabDeallocResult::Done => {}
+ SlabDeallocResult::FreeSlab { base, pages } => {
+ self.buddy.dealloc_pages(base, pages);
+ }
+ }
+ }
+}
+
+pub struct GlobalHarness {
+ _region: HostRegion,
+ pub allocator: GlobalAllocator,
+}
+
+impl GlobalHarness {
+ pub fn new(region_size: usize, cpu_count: usize) -> Self {
+ let mut region = HostRegion::new(region_size);
+ let allocator = GlobalAllocator::::new();
+ MOCK_OS.set_cpu(0);
+ unsafe {
+ allocator
+ .init(region.as_mut_slice(), cpu_count, &MOCK_OS)
+ .unwrap();
+ }
+ Self {
+ _region: region,
+ allocator,
+ }
+ }
+}
diff --git a/benches/global_allocator.rs b/benches/global_allocator.rs
index aa5d4e1..a639ff8 100644
--- a/benches/global_allocator.rs
+++ b/benches/global_allocator.rs
@@ -1,247 +1,111 @@
-//! Benchmarks for GlobalAllocator - the unified allocator interface
-//!
-//! This benchmark suite tests the performance and stability of the GlobalAllocator
-//! which automatically routes small allocations to Slab and large allocations to Buddy.
-
-use buddy_slab_allocator::GlobalAllocator;
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
-use rand::{Rng, SeedableRng};
-use std::alloc::Layout;
-use std::alloc::{alloc, dealloc};
-
-const PAGE_SIZE: usize = 0x1000;
-const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB
-
-/// Allocate a test heap from the system allocator
-fn alloc_test_heap(size: usize) -> (*mut u8, Layout) {
- let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap();
- let ptr = unsafe { alloc(layout) };
- assert!(!ptr.is_null(), "Failed to allocate test heap");
- (ptr, layout)
-}
+//! Benchmarks for the unified global allocator.
-/// Deallocate the test heap
-fn dealloc_test_heap(ptr: *mut u8, layout: Layout) {
- unsafe { dealloc(ptr, layout) };
-}
+mod common;
-/// Benchmark small allocations (≤2048 bytes) - uses Slab allocator
-fn bench_small_alloc(c: &mut Criterion) {
- let sizes: [usize; 6] = [8, 16, 64, 256, 512, 1024];
+use common::{GlobalHarness, HEAP_SIZE, MOCK_OS, OPERATIONS_PER_BATCH, PAGE_SIZE};
+use core::alloc::Layout;
+use divan::{Bencher, black_box};
- let mut group = c.benchmark_group("global_alloc_small");
+fn main() {
+ divan::main();
+}
- for size in sizes.iter() {
- let layout = Layout::from_size_align(*size, 8).unwrap();
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
+#[divan::bench_group]
+mod global {
+ use super::*;
- group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
+ #[divan::bench(args = [8usize, 64, 512, 2048])]
+ fn small_alloc_free(bencher: Bencher, size: usize) {
+ let harness = GlobalHarness::new(HEAP_SIZE, 2);
+ let layout = Layout::from_size_align(size, 8).unwrap();
- b.iter(|| {
- let ptr = allocator.alloc(black_box(layout));
- black_box(ptr)
- });
+ bencher.bench_local(|| {
+ MOCK_OS.set_cpu(0);
+ let ptr = harness.allocator.alloc(black_box(layout)).unwrap();
+ unsafe { harness.allocator.dealloc(ptr, layout) };
+ black_box(ptr)
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
+ #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16])]
+ fn large_alloc_free(bencher: Bencher, size: usize) {
+ let harness = GlobalHarness::new(HEAP_SIZE, 2);
+ let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap();
-/// Benchmark large allocations (>2048 bytes) - uses Buddy allocator
-fn bench_large_alloc(c: &mut Criterion) {
- let sizes: [usize; 5] = [
- PAGE_SIZE,
- PAGE_SIZE * 2,
- PAGE_SIZE * 4,
- PAGE_SIZE * 8,
- PAGE_SIZE * 16,
- ];
-
- let mut group = c.benchmark_group("global_alloc_large");
-
- for size in sizes.iter() {
- let layout = Layout::from_size_align(*size, PAGE_SIZE).unwrap();
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- let ptr = allocator.alloc(black_box(layout));
- black_box(ptr)
- });
+ bencher.bench_local(|| {
+ MOCK_OS.set_cpu(0);
+ let ptr = harness.allocator.alloc(black_box(layout)).unwrap();
+ unsafe { harness.allocator.dealloc(ptr, layout) };
+ black_box(ptr)
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
-
-/// Benchmark deallocation for small allocations
-fn bench_small_dealloc(c: &mut Criterion) {
- let layout = Layout::from_size_align(64, 8).unwrap();
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("global_dealloc_small", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- // Pre-allocate a batch of pointers
- let mut pointers = Vec::new();
- for _ in 0..1000 {
- let ptr = allocator.alloc(layout).unwrap();
- pointers.push(ptr);
- }
-
- b.iter(|| {
- // Release and acquire to keep benchmark going
- let ptr = pointers.pop().unwrap();
- allocator.dealloc(black_box(ptr), layout);
- // Allocate again to maintain pool
- let new_ptr = allocator.alloc(layout).unwrap();
- pointers.push(new_ptr);
- });
-
- // Cleanup
- for ptr in pointers {
- allocator.dealloc(ptr, layout);
- }
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark allocation/deallocation cycle
-fn bench_alloc_dealloc_cycle(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("global_alloc_dealloc_cycle", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- // Simulate realistic usage: allocate and immediately dealloc
- for _ in 0..100 {
- let layout = Layout::from_size_align(64, 8).unwrap();
- let ptr = allocator.alloc(layout).unwrap();
- allocator.dealloc(ptr, layout);
- }
+ #[divan::bench(args = [1usize, 4, 16, 64])]
+ fn page_interface(bencher: Bencher, pages: usize) {
+ let harness = GlobalHarness::new(HEAP_SIZE, 2);
+
+ bencher.bench_local(|| {
+ MOCK_OS.set_cpu(0);
+ let addr = harness
+ .allocator
+ .alloc_pages(black_box(pages), PAGE_SIZE)
+ .unwrap();
+ harness.allocator.dealloc_pages(addr, pages);
+ black_box(addr)
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
+ }
-/// Benchmark random size allocations
-fn bench_random_allocations(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("global_random_allocations", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- // 100 random allocations
- for _ in 0..100 {
- let size: usize = rng.gen_range(8..2048);
- let size = size.next_power_of_two();
- let layout = Layout::from_size_align(size, 8).unwrap();
- let ptr = allocator.alloc(layout).unwrap();
- allocator.dealloc(ptr, layout);
+ #[divan::bench]
+ fn mixed_object_page_workload(bencher: Bencher) {
+ let harness = GlobalHarness::new(HEAP_SIZE, 2);
+ let object_layouts = [
+ Layout::from_size_align(64, 8).unwrap(),
+ Layout::from_size_align(256, 8).unwrap(),
+ Layout::from_size_align(1024, 8).unwrap(),
+ Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(),
+ ];
+ let page_counts = [1usize, 2, 4, 8];
+
+ bencher.bench_local(|| {
+ for idx in 0..OPERATIONS_PER_BATCH {
+ MOCK_OS.set_cpu(idx % 2);
+ let layout = object_layouts[idx % object_layouts.len()];
+ let ptr = harness.allocator.alloc(layout).unwrap();
+ unsafe { harness.allocator.dealloc(ptr, layout) };
+
+ let pages = page_counts[idx % page_counts.len()];
+ let addr = harness.allocator.alloc_pages(pages, PAGE_SIZE).unwrap();
+ harness.allocator.dealloc_pages(addr, pages);
+
+ black_box((ptr, addr));
}
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark mixed allocation pattern - simulates real-world usage
-fn bench_mixed_pattern(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("global_mixed_pattern", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- // Pre-allocate a pool of objects
- let mut small_ptrs = Vec::new();
- let mut large_ptrs = Vec::new();
-
- for _ in 0..100 {
- let small_layout = Layout::from_size_align(64, 8).unwrap();
- small_ptrs.push(allocator.alloc(small_layout).unwrap());
- }
-
- for _ in 0..10 {
- let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap();
- large_ptrs.push(allocator.alloc(large_layout).unwrap());
- }
-
- b.iter(|| {
- // Alternate between small and large allocations
- let small_layout = Layout::from_size_align(64, 8).unwrap();
- let ptr = allocator.alloc(small_layout).unwrap();
- allocator.dealloc(ptr, small_layout);
-
- let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap();
- let ptr2 = allocator.alloc(large_layout).unwrap();
- allocator.dealloc(ptr2, large_layout);
- });
-
- // Cleanup
- let small_layout = Layout::from_size_align(64, 8).unwrap();
- for ptr in small_ptrs {
- allocator.dealloc(ptr, small_layout);
- }
- let large_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap();
- for ptr in large_ptrs {
- allocator.dealloc(ptr, large_layout);
- }
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
+ }
-/// Benchmark page allocation through GlobalAllocator
-fn bench_page_allocation(c: &mut Criterion) {
- let page_counts = [1, 2, 4, 8, 16];
+ #[divan::bench]
+ fn remote_free_cycle(bencher: Bencher) {
+ let harness = GlobalHarness::new(HEAP_SIZE, 2);
+ let layout = Layout::from_size_align(64, 8).unwrap();
- let mut group = c.benchmark_group("global_alloc_pages");
+ bencher.bench_local(|| {
+ let mut ptrs = Vec::with_capacity(64);
- for num_pages in page_counts.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
+ MOCK_OS.set_cpu(0);
+ for _ in 0..64 {
+ ptrs.push(harness.allocator.alloc(layout).unwrap());
+ }
- group.bench_with_input(BenchmarkId::from_parameter(num_pages), num_pages, |b, _| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
+ MOCK_OS.set_cpu(1);
+ for &ptr in &ptrs {
+ unsafe { harness.allocator.dealloc(ptr, layout) };
+ }
- b.iter(|| {
- let addr = allocator.alloc_pages(black_box(*num_pages), PAGE_SIZE);
- black_box(addr)
- });
+ MOCK_OS.set_cpu(0);
+ for _ in 0..64 {
+ let ptr = harness.allocator.alloc(layout).unwrap();
+ unsafe { harness.allocator.dealloc(ptr, layout) };
+ black_box(ptr);
+ }
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
-
- group.finish();
}
-
-criterion_group!(
- benches,
- bench_small_alloc,
- bench_large_alloc,
- bench_small_dealloc,
- bench_alloc_dealloc_cycle,
- bench_random_allocations,
- bench_mixed_pattern,
- bench_page_allocation
-);
-criterion_main!(benches);
diff --git a/benches/slab_allocator.rs b/benches/slab_allocator.rs
index cf1accc..9197103 100644
--- a/benches/slab_allocator.rs
+++ b/benches/slab_allocator.rs
@@ -1,353 +1,101 @@
-//! Benchmarks for SlabByteAllocator - small object allocation
-//!
-//! This benchmark suite tests the performance and stability of the Slab allocator
-//! which is optimized for small object allocations (≤2048 bytes).
+//! Benchmarks for the slab allocator with buddy-backed slab refill.
-use buddy_slab_allocator::{
- BaseAllocator, ByteAllocator, CompositePageAllocator, PageAllocatorForSlab, SlabByteAllocator,
-};
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
-use rand::{Rng, SeedableRng};
-use std::alloc::Layout;
-use std::alloc::{alloc, dealloc};
+mod common;
-const PAGE_SIZE: usize = 0x1000;
-const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB
+use common::{HEAP_SIZE, OPERATIONS_PER_BATCH, SlabHarness};
+use core::alloc::Layout;
+use divan::{Bencher, black_box};
-/// Allocate a test heap from the system allocator
-fn alloc_test_heap(size: usize) -> (*mut u8, Layout) {
- let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap();
- let ptr = unsafe { alloc(layout) };
- assert!(!ptr.is_null(), "Failed to allocate test heap");
- (ptr, layout)
+fn main() {
+ divan::main();
}
-/// Deallocate the test heap
-fn dealloc_test_heap(ptr: *mut u8, layout: Layout) {
- unsafe { dealloc(ptr, layout) };
-}
-
-/// Create initialized slab allocator with page allocator
-fn create_slab_allocator(
- heap_ptr: *mut u8,
-) -> (
- CompositePageAllocator,
- SlabByteAllocator,
-) {
- let mut page_alloc = CompositePageAllocator::::new();
- page_alloc.init(heap_ptr as usize, HEAP_SIZE);
-
- let mut slab_alloc = SlabByteAllocator::::new();
- slab_alloc.init();
-
- (page_alloc, slab_alloc)
-}
+#[divan::bench_group]
+mod slab {
+ use super::*;
-/// Benchmark allocations for different size classes
-fn bench_size_classes(c: &mut Criterion) {
- let size_classes: [usize; 6] = [8, 64, 512, 1024, 1536, 2048];
+ #[divan::bench(args = [8usize, 64, 256, 512, 1024, 2048])]
+ fn size_class_alloc_free(bencher: Bencher, size: usize) {
+ let mut harness = SlabHarness::new(HEAP_SIZE);
+ let layout = Layout::from_size_align(size, size).unwrap();
- let mut group = c.benchmark_group("slab_size_classes");
-
- for size in size_classes.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, _| {
- let layout = Layout::from_size_align(*size, 8).unwrap();
-
- b.iter(|| {
- let ptr = slab_alloc.alloc(black_box(layout)).unwrap();
- slab_alloc.dealloc(ptr, layout);
- black_box(ptr)
- });
+ bencher.bench_local(|| {
+ let ptr = harness.alloc(black_box(layout));
+ harness.dealloc(ptr, layout);
+ black_box(ptr)
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
-
-/// Benchmark deallocation
-fn bench_dealloc(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_dealloc", |b| {
- let layout = Layout::from_size_align(64, 8).unwrap();
-
- // Pre-allocate pointers
- let mut pointers = Vec::new();
- for _ in 0..1000 {
- let ptr = slab_alloc.alloc(layout).unwrap();
- pointers.push(ptr);
- }
-
- b.iter(|| {
- let ptr = pointers.pop().unwrap();
- slab_alloc.dealloc(black_box(ptr), layout);
- // Allocate again to maintain pool
- let new_ptr = slab_alloc.alloc(layout).unwrap();
- pointers.push(new_ptr);
- });
-
- // Cleanup
- for ptr in pointers {
- slab_alloc.dealloc(ptr, layout);
- }
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark allocation/deallocation cycle
-fn bench_alloc_dealloc_cycle(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_alloc_dealloc_cycle", |b| {
- b.iter(|| {
- for _ in 0..100 {
- let layout = Layout::from_size_align(64, 8).unwrap();
- let ptr = slab_alloc.alloc(layout).unwrap();
- slab_alloc.dealloc(ptr, layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark allocations with different alignments
-fn bench_alignment(c: &mut Criterion) {
- let alignments = [8, 1024, 2048];
-
- let mut group = c.benchmark_group("slab_alignment");
-
- for alignment in alignments.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- group.bench_with_input(BenchmarkId::from_parameter(alignment), alignment, |b, _| {
- let layout = Layout::from_size_align(64, *alignment).unwrap();
-
- b.iter(|| {
- let ptr = slab_alloc.alloc(black_box(layout)).unwrap();
- slab_alloc.dealloc(ptr, layout);
- black_box(ptr)
- });
+ #[divan::bench]
+ fn hot_reuse(bencher: Bencher) {
+ let mut harness = SlabHarness::new(HEAP_SIZE);
+ let layout = Layout::from_size_align(128, 128).unwrap();
+ let ptr = harness.alloc(layout);
+ harness.dealloc(ptr, layout);
+
+ bencher.bench_local(|| {
+ let ptr = harness.alloc(layout);
+ harness.dealloc(ptr, layout);
+ black_box(ptr)
});
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
- group.finish();
-}
-
-/// Benchmark random size allocations
-fn bench_random_allocations(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_random_allocations", |b| {
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- for _ in 0..100 {
- let size: usize = rng.gen_range(8..2049);
- let size = if size.is_power_of_two() {
- size
- } else {
- size.next_power_of_two()
- };
- let layout = Layout::from_size_align(size, 8).unwrap();
- let ptr = slab_alloc.alloc(layout).unwrap();
- slab_alloc.dealloc(ptr, layout);
+ #[divan::bench]
+ fn mixed_size_batch(bencher: Bencher) {
+ let mut harness = SlabHarness::new(HEAP_SIZE);
+ let layouts = [
+ Layout::from_size_align(8, 8).unwrap(),
+ Layout::from_size_align(64, 64).unwrap(),
+ Layout::from_size_align(256, 256).unwrap(),
+ Layout::from_size_align(512, 512).unwrap(),
+ Layout::from_size_align(1024, 1024).unwrap(),
+ Layout::from_size_align(2048, 2048).unwrap(),
+ ];
+
+ bencher.bench_local(|| {
+ for layout in layouts {
+ let ptr = harness.alloc(layout);
+ harness.dealloc(ptr, layout);
+ black_box(ptr);
}
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark object pooling - allocate, use, dealloc repeatedly
-fn bench_object_pooling(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_object_pooling", |b| {
- let layout = Layout::from_size_align(128, 8).unwrap();
-
- // Simulate object pool with fixed size
- let mut pool = Vec::new();
- for _ in 0..100 {
- let ptr = slab_alloc.alloc(layout).unwrap();
- pool.push(ptr);
- }
-
- b.iter(|| {
- // Get an object from pool, use it, put it back
- let ptr = pool.pop().unwrap();
- // Simulate usage
- black_box(ptr);
- pool.push(ptr);
- });
+ }
- // Cleanup
- for ptr in pool {
- slab_alloc.dealloc(ptr, layout);
+ #[divan::bench]
+ fn steady_state_recycle(bencher: Bencher) {
+ let mut harness = SlabHarness::new(HEAP_SIZE);
+ let layout = Layout::from_size_align(64, 64).unwrap();
+ let mut active = Vec::with_capacity(256);
+ for _ in 0..256 {
+ active.push(harness.alloc(layout));
}
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark mixed size allocation pattern
-fn bench_mixed_sizes(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_mixed_sizes", |b| {
- let sizes = [8, 64, 256, 512, 1024, 2048];
- let layouts: Vec = sizes
- .iter()
- .map(|&s| Layout::from_size_align(s, 8).unwrap())
- .collect();
-
- b.iter(|| {
- for layout in &layouts {
- let ptr = slab_alloc.alloc(*layout).unwrap();
- slab_alloc.dealloc(ptr, *layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark rapid allocation and deallocation stress test
-fn bench_stress_test(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
- c.bench_function("slab_stress_test", |b| {
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- let mut allocated = Vec::new();
-
- // Stress test: rapid alloc/dealloc with varying sizes
- for _ in 0..1000 {
- if allocated.is_empty() || rng.gen_bool(0.7) {
- // Allocate
- let size = rng.gen_range(8..2049);
- let layout = Layout::from_size_align(size, 8).unwrap();
- if let Ok(ptr) = slab_alloc.alloc(layout) {
- allocated.push((ptr, layout));
- }
- } else {
- // Deallocate random
- let idx = rng.gen_range(0..allocated.len());
- let (ptr, layout) = allocated.swap_remove(idx);
- slab_alloc.dealloc(ptr, layout);
- }
- }
-
- // Cleanup
- allocated.clear();
- let ptrs_to_free: Vec<_> = std::mem::take(&mut allocated);
- for (ptr, layout) in ptrs_to_free {
- slab_alloc.dealloc(ptr, layout);
- }
+ bencher.bench_local(|| {
+ let ptr = active.pop().unwrap();
+ harness.dealloc(ptr, layout);
+ let new_ptr = harness.alloc(layout);
+ active.push(new_ptr);
+ black_box(new_ptr)
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark performance under memory pressure
-fn bench_memory_pressure(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- c.bench_function("slab_memory_pressure", |b| {
- let layout = Layout::from_size_align(64, 8).unwrap();
-
- b.iter(|| {
- // Allocate until near capacity
- let mut allocated = Vec::new();
- loop {
- match slab_alloc.alloc(layout) {
- Ok(ptr) => allocated.push(ptr),
- Err(_) => break,
- }
- }
-
- // Free half and reallocate
- for i in (0..allocated.len()).step_by(2) {
- slab_alloc.dealloc(allocated[i], layout);
- }
-
- // Try to allocate more
- for _ in 0..10 {
- let _ = slab_alloc.alloc(layout);
- }
+ }
- // Cleanup
- for ptr in allocated {
- slab_alloc.dealloc(ptr, layout);
+ #[divan::bench]
+ fn mixed_size_workload(bencher: Bencher) {
+ let mut harness = SlabHarness::new(HEAP_SIZE);
+ let layouts = [
+ Layout::from_size_align(8, 8).unwrap(),
+ Layout::from_size_align(64, 64).unwrap(),
+ Layout::from_size_align(256, 256).unwrap(),
+ Layout::from_size_align(1024, 1024).unwrap(),
+ ];
+
+ bencher.bench_local(|| {
+ for idx in 0..OPERATIONS_PER_BATCH {
+ let layout = layouts[idx % layouts.len()];
+ let ptr = harness.alloc(layout);
+ harness.dealloc(ptr, layout);
+ black_box(ptr);
}
});
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Benchmark statistics retrieval
-fn bench_statistics(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
- let (mut page_alloc, mut slab_alloc) = create_slab_allocator(heap_ptr);
- slab_alloc.set_page_allocator(&mut page_alloc as *mut _ as *mut dyn PageAllocatorForSlab);
-
- // Make some allocations
- for _ in 0..10 {
- let layout = Layout::from_size_align(64, 8).unwrap();
- let _ = slab_alloc.alloc(layout);
}
-
- c.bench_function("slab_statistics", |b| {
- b.iter(|| {
- let total = slab_alloc.total_bytes();
- let used = slab_alloc.used_bytes();
- let available = slab_alloc.available_bytes();
- black_box((total, used, available))
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
}
-
-criterion_group!(
- benches,
- bench_size_classes,
- bench_dealloc,
- bench_alloc_dealloc_cycle,
- bench_alignment,
- bench_random_allocations,
- bench_object_pooling,
- bench_mixed_sizes,
- bench_stress_test,
- bench_memory_pressure,
- bench_statistics
-);
-criterion_main!(benches);
diff --git a/benches/stability.rs b/benches/stability.rs
deleted file mode 100644
index 4139f1f..0000000
--- a/benches/stability.rs
+++ /dev/null
@@ -1,442 +0,0 @@
-//! Stability and stress testing benchmarks
-//!
-//! This benchmark suite focuses on stability testing under various stress conditions:
-//! - Memory exhaustion handling
-//! - Long-running operation stability
-//! - Random allocation/deallocation patterns
-//! - Fragmentation resistance
-//! - Memory leak detection
-
-use buddy_slab_allocator::GlobalAllocator;
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
-use rand::{Rng, SeedableRng};
-use std::alloc::Layout;
-use std::alloc::{alloc, dealloc};
-
-const PAGE_SIZE: usize = 0x1000;
-const HEAP_SIZE: usize = 64 * 1024 * 1024; // 64MB
-
-/// Allocate a test heap from the system allocator
-fn alloc_test_heap(size: usize) -> (*mut u8, Layout) {
- let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap();
- let ptr = unsafe { alloc(layout) };
- assert!(!ptr.is_null(), "Failed to allocate test heap");
- (ptr, layout)
-}
-
-/// Deallocate the test heap
-fn dealloc_test_heap(ptr: *mut u8, layout: Layout) {
- unsafe { dealloc(ptr, layout) };
-}
-
-/// Stability test: Random allocation/deallocation pattern over many iterations
-fn bench_random_pattern_stability(c: &mut Criterion) {
- let iterations = [1000, 5000, 10000];
-
- let mut group = c.benchmark_group("stability_random_pattern");
-
- for &iter in iterations.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- group.bench_with_input(BenchmarkId::from_parameter(iter), &iter, |b, _| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- let mut allocated = Vec::new();
-
- for _ in 0..iter {
- if allocated.is_empty() || rng.gen_bool(0.6) {
- // 60% allocate
- let size = rng.gen_range(8..8193);
- let size = if size <= 2048 {
- size
- } else {
- // Align to page size for large allocations
- ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE
- };
- let layout = Layout::from_size_align(size, 8).unwrap();
- if let Ok(ptr) = allocator.alloc(layout) {
- allocated.push((ptr, layout));
- }
- } else {
- // 40% deallocate
- let idx = rng.gen_range(0..allocated.len());
- let (ptr, layout) = allocated.swap_remove(idx);
- allocator.dealloc(ptr, layout);
- }
- }
-
- // Cleanup - verify no leaks by checking stats
- #[cfg(feature = "tracking")]
- {
- let before = allocator.get_stats();
- }
-
- for (ptr, layout) in allocated {
- allocator.dealloc(ptr, layout);
- }
-
- #[cfg(feature = "tracking")]
- {
- let after = allocator.get_stats();
- // Verify we're back to initial state
- assert_eq!(after.used_pages, 0, "Memory leak detected!");
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-
- group.finish();
-}
-
-/// Stability test: Allocate to exhaustion and handle gracefully
-fn bench_exhaustion_handling(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stability_exhaustion", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- let mut allocated = Vec::new();
- let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap();
-
- // Allocate until exhaustion
- loop {
- match allocator.alloc(layout) {
- Ok(ptr) => allocated.push(ptr),
- Err(_) => break, // Expected - out of memory
- }
- }
-
- // Verify we can still deallocate
- for ptr in allocated.iter().take(10) {
- allocator.dealloc(*ptr, layout);
- }
-
- // Verify we can allocate again after freeing
- let new_ptr = allocator.alloc(layout);
- assert!(new_ptr.is_ok(), "Failed to allocate after freeing memory");
-
- // Cleanup
- if let Ok(ptr) = new_ptr {
- allocator.dealloc(ptr, layout);
- }
- for ptr in allocated {
- allocator.dealloc(ptr, layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Stability test: Fragmentation resistance
-fn bench_fragmentation_resistance(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stability_fragmentation", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- let mut small_ptrs = Vec::new();
- let small_layout = Layout::from_size_align(64, 8).unwrap();
-
- // Allocate many small objects
- for _ in 0..1000 {
- if let Ok(ptr) = allocator.alloc(small_layout) {
- small_ptrs.push(ptr);
- }
- }
-
- // Free every other one to create fragmentation
- for i in (0..small_ptrs.len()).step_by(2) {
- allocator.dealloc(small_ptrs[i], small_layout);
- }
-
- // Try to allocate a large contiguous block
- let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap();
- let large_ptr = allocator.alloc(large_layout);
-
- // Cleanup
- for ptr in small_ptrs {
- allocator.dealloc(ptr, small_layout);
- }
- if let Ok(ptr) = large_ptr {
- allocator.dealloc(ptr, large_layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Stability test: Alternating allocation/deallocation pattern
-fn bench_alternating_pattern(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stability_alternating", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- let mut ptrs = Vec::new();
-
- // Alternating allocate/dealloc pattern
- for i in 0..500 {
- if i % 2 == 0 {
- let layout = Layout::from_size_align(64, 8).unwrap();
- if let Ok(ptr) = allocator.alloc(layout) {
- ptrs.push((ptr, layout));
- }
- } else if !ptrs.is_empty() {
- let (ptr, layout) = ptrs.remove(0);
- allocator.dealloc(ptr, layout);
- }
- }
-
- // Cleanup
- for (ptr, layout) in ptrs {
- allocator.dealloc(ptr, layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Stability test: Long-running operation with periodic checks
-fn bench_long_running(c: &mut Criterion) {
- let durations = [1000, 5000, 10000];
-
- let mut group = c.benchmark_group("stability_long_running");
-
- for &duration in durations.iter() {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- group.bench_with_input(BenchmarkId::from_parameter(duration), &duration, |b, _| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- for _ in 0..duration {
- let size = rng.gen_range(8..2049);
- let layout = Layout::from_size_align(size, 8).unwrap();
- let ptr = allocator.alloc(layout);
-
- if let Ok(p) = ptr {
- allocator.dealloc(p, layout);
- }
- }
-
- #[cfg(feature = "tracking")]
- {
- let stats = allocator.get_stats();
- // Verify stats are consistent
- assert!(stats.used_pages <= stats.total_pages);
- assert!(stats.free_pages <= stats.total_pages);
- assert_eq!(stats.used_pages + stats.free_pages, stats.total_pages);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-
- group.finish();
-}
-
-/// Stress test: Rapid allocation/deallocation of mixed sizes
-fn bench_rapid_mixed_sizes(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stress_rapid_mixed", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- b.iter(|| {
- let mut allocated = Vec::new();
-
- // Rapid mixed-size allocations
- for _ in 0..5000 {
- let size = rng.gen_range(8..16385);
- let aligned_size = if size <= 2048 {
- size
- } else {
- ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE
- };
- let layout = Layout::from_size_align(aligned_size, 8).unwrap();
-
- if let Ok(ptr) = allocator.alloc(layout) {
- if rng.gen_bool(0.5) {
- // Immediately free 50% of allocations
- allocator.dealloc(ptr, layout);
- } else {
- allocated.push((ptr, layout));
- }
- }
- }
-
- // Cleanup
- for (ptr, layout) in allocated {
- allocator.dealloc(ptr, layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Stress test: Page allocation pressure
-fn bench_page_allocation_pressure(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stress_page_pressure", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- let mut page_addrs = Vec::new();
-
- // Allocate pages until near exhaustion
- loop {
- match allocator.alloc_pages(1, PAGE_SIZE) {
- Ok(addr) => page_addrs.push(addr),
- Err(_) => break,
- }
- }
-
- // Free and reallocate in different patterns
- for i in (0..page_addrs.len()).step_by(2) {
- allocator.dealloc_pages(page_addrs[i], 1);
- }
-
- // Try to allocate more
- let mut new_addrs = Vec::new();
- for _ in 0..5 {
- if let Ok(addr) = allocator.alloc_pages(1, PAGE_SIZE) {
- new_addrs.push(addr);
- }
- }
-
- // Cleanup
- for addr in page_addrs {
- allocator.dealloc_pages(addr, 1);
- }
- for addr in new_addrs {
- allocator.dealloc_pages(addr, 1);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Memory leak detection test
-fn bench_memory_leak_detection(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stability_leak_detection", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- #[cfg(feature = "tracking")]
- {
- let initial_stats = allocator.get_stats();
- }
-
- b.iter(|| {
- let mut ptrs = Vec::new();
- let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
-
- // Perform random allocations
- for _ in 0..1000 {
- let size = rng.gen_range(8..2049);
- let layout = Layout::from_size_align(size, 8).unwrap();
- if let Ok(ptr) = allocator.alloc(layout) {
- ptrs.push((ptr, layout));
- }
- }
-
- // Free all allocations
- for (ptr, layout) in &ptrs {
- allocator.dealloc(*ptr, *layout);
- }
-
- #[cfg(feature = "tracking")]
- {
- let stats = allocator.get_stats();
- // Verify no memory leaks
- assert_eq!(stats.used_pages, 0, "Memory leak detected!");
- }
-
- ptrs.clear();
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-/// Stability test: Edge cases
-fn bench_edge_cases(c: &mut Criterion) {
- let (heap_ptr, heap_layout) = alloc_test_heap(HEAP_SIZE);
-
- c.bench_function("stability_edge_cases", |b| {
- let mut allocator = GlobalAllocator::::new();
- allocator.init(heap_ptr as usize, HEAP_SIZE).unwrap();
-
- b.iter(|| {
- // Test minimum size allocation
- let min_layout = Layout::from_size_align(1, 1).unwrap();
- let min_ptr = allocator.alloc(min_layout);
- if let Ok(ptr) = min_ptr {
- allocator.dealloc(ptr, min_layout);
- }
-
- // Test maximum small size allocation
- let max_small_layout = Layout::from_size_align(2048, 2048).unwrap();
- let max_small_ptr = allocator.alloc(max_small_layout);
- if let Ok(ptr) = max_small_ptr {
- allocator.dealloc(ptr, max_small_layout);
- }
-
- // Test page-aligned allocation
- let page_layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap();
- let page_ptr = allocator.alloc(page_layout);
- if let Ok(ptr) = page_ptr {
- allocator.dealloc(ptr, page_layout);
- }
-
- // Test odd alignment - alignment must be power of two
- let odd_align = 8; // Changed from 7 to 8 (power of 2)
- let odd_layout = Layout::from_size_align(64, odd_align).unwrap();
- let odd_ptr = allocator.alloc(odd_layout);
- if let Ok(ptr) = odd_ptr {
- allocator.dealloc(ptr, odd_layout);
- }
- });
- });
-
- dealloc_test_heap(heap_ptr, heap_layout);
-}
-
-criterion_group!(
- benches,
- bench_random_pattern_stability,
- bench_exhaustion_handling,
- bench_fragmentation_resistance,
- bench_alternating_pattern,
- bench_long_running,
- bench_rapid_mixed_sizes,
- bench_page_allocation_pressure,
- bench_memory_leak_detection,
- bench_edge_cases
-);
-criterion_main!(benches);
diff --git a/docs/design.md b/docs/design.md
new file mode 100644
index 0000000..a53caa8
--- /dev/null
+++ b/docs/design.md
@@ -0,0 +1,615 @@
+# buddy-slab-allocator 设计文档
+
+本文档面向阅读和维护当前实现的开发者，描述这个分配器在 `buddy + slab` 两级结构下的核心数据结构、初始化过程、`add_region` 热添加流程、`alloc` / `dealloc` 路径，以及多核并发场景下的协作方式。
+
+文中描述以当前仓库实现为准，重点对应以下模块：
+
+- `src/global.rs`
+- `src/buddy/mod.rs`
+- `src/slab/mod.rs`
+- `src/slab/cache.rs`
+- `src/slab/page.rs`
+
+## 1. 设计目标
+
+这个分配器服务于 `no_std`、内核或嵌入式环境，目标是同时满足两类需求：
+
+- 页级分配：按页返回连续内存，支持 power-of-two 分裂和合并。
+- 小对象分配：对 `<= 2048` 字节的对象提供更低碎片、更低分配开销的快速路径。
+
+因此整体采用两级结构：
+
+- 后端：`BuddyAllocator`
+ 负责管理一个或多个 section 中的页。
+- 前端：`SlabAllocator`
+ 负责管理不同 size class 的小对象。
+- 门面：`GlobalAllocator`
+ 统一对外暴露 `alloc` / `dealloc` / `alloc_pages` / `dealloc_pages` 等接口。
+
+## 2. 总体结构
+
+```mermaid
+flowchart TD
+ GA[GlobalAllocator] --> B[BuddyAllocator]
+ GA --> PCS[per-CPU SlabAllocator[]]
+ GA --> OS[OsImpl]
+
+ PCS --> SC8[SlabCache: 8B]
+ PCS --> SC64[SlabCache: 64B]
+ PCS --> SC2K[SlabCache: 2048B]
+
+ SC8 --> SPH[SlabPageHeader]
+ SC64 --> SPH
+ SC2K --> SPH
+
+ B --> SH[BuddySection intrusive list]
+ SH --> PM[PageMeta[]]
+ SH --> FL[free_lists by order]
+```
+
+### 2.1 地址语义
+
+当前实现的主语义是“虚拟地址”：
+
+- `GlobalAllocator::init()` 接收的是一段可写 `&mut [u8]` 区域。
+- `BuddyAllocator` 管理的是一段连续虚拟地址区间。
+- `alloc_pages()`、`alloc()` 返回的也都是虚拟地址。
+
+物理地址只在 `alloc_pages_lowmem()` 里参与判断：
+
+- 候选块的虚拟地址通过 `OsImpl::virt_to_phys()` 翻译成物理地址。
+- 只有物理地址低于 `4 GiB` 的块才会被当作 DMA32 候选。
+
+### 2.2 容量统计语义
+
+当前公开了两类整体容量统计：
+
+- `managed_bytes`
+ 所有 section 中可分配 heap 的总字节数，不包含 region 前缀 metadata。
+- `allocated_bytes`
+ 后端页占用字节数，按 `managed_bytes - free_pages * PAGE_SIZE` 计算。
+
+因此 `allocated_bytes` 的口径是“页后端已经占住多少字节”，不是用户请求的 `layout.size()` 精确累加。
+它会包含：
+
+- slab 页
+- empty slab cache 保留页
+- 对齐放大
+- buddy / slab 的内部碎片
+
+## 3. 初始化、追加 Region 与内存布局
+
+当前实现支持两种 region 进入 allocator 的方式：
+
+- `GlobalAllocator::init(region, cpu_count, os)`
+ 注册首个 region，并初始化唯一一份 per-CPU slab 槽位。
+- `GlobalAllocator::add_region(region)`
+ 在运行时追加新的 region，只扩展 buddy 后端，不再重复创建 per-CPU slab。
+
+对 buddy 而言，每个 region 都会被拆成：
+
+- 前缀：region 自带元数据
+- 后缀：该 section 的 managed heap
+
+首个 global region 的元数据前缀包含：
+
+- `BuddySection`
+- `PageMeta[]`
+- `per_cpu_slabs: [SpinMutex; cpu_count]`
+
+后续追加 region 的元数据前缀只包含：
+
+- `BuddySection`
+- `PageMeta[]`
+
+```mermaid
+flowchart LR
+ A[region start] --> B[aligned section_start]
+ B --> C[BuddySection]
+ C --> D[PageMeta array]
+ D --> E{first global region?}
+ E -- Yes --> F[per-CPU SlabAllocator slots]
+ E -- No --> G[no slab slots here]
+ F --> H[padding to PAGE_SIZE]
+ G --> H
+ H --> I[section heap start]
+ I --> J[managed heap of this section]
+```
+
+### 3.1 初始化步骤
+
+`GlobalAllocator::init()` 的主要步骤如下：
+
+1. 根据 `region.len()` 和 `cpu_count` 计算首个 section 最多能管理多少页。
+2. 在 region 前缀预留 `BuddySection + PageMeta[] + per_cpu_slabs`。
+3. 选择一个页对齐的 section heap 起点。
+4. 将首个 section 注册进 `BuddyAllocator`。
+5. 原地构造每个 CPU 对应的 `SpinMutex`。
+6. 将 `per_cpu_slabs`、`cpu_count`、`os` 写入 `GlobalAllocator`。
+
+`GlobalAllocator::add_region()` 的步骤更简单：
+
+1. 校验 allocator 已初始化。
+2. 在新 region 前缀预留 `BuddySection + PageMeta[]`。
+3. 计算该 section 的可管理 heap。
+4. 将该 section 追加到 buddy 的 intrusive list 尾部。
+
+### 3.2 为什么 metadata 放在 region 前缀
+
+这样做有几个好处：
+
+- 不需要额外的启动期堆分配。
+- `BuddySection` 与 `PageMeta[]` 都跟随 region 自身生命周期。
+- `per_cpu_slabs` 的内存来源和被管理堆绑在一起，部署简单。
+- 不需要预先声明最大 section 数量。
+
+代价是：
+
+- metadata 会消耗一部分可分配空间。
+- 每个 section 的 heap 起点都可能晚于各自的 region 起点。
+- section 查询与地址归属定位当前采用线性扫描。
+
+## 4. BuddyAllocator 设计
+
+`BuddyAllocator` 管理页级内存，是整个系统的共享后端。它的基本单位不再是单一 heap，而是一个按注册顺序串起来的 section 链表。
+
+### 4.1 核心数据结构
+
+```mermaid
+classDiagram
+ class BuddyAllocator {
+ +sections_head: *mut BuddySection
+ +section_count: usize
+ +os: Option<&dyn OsImpl>
+ }
+
+ class BuddySection {
+ +next: *mut BuddySection
+ +meta: *mut PageMeta
+ +heap_start: usize
+ +heap_size: usize
+ +free_lists: [u32; MAX_ORDER+1]
+ +free_pages: usize
+ +total_pages: usize
+ }
+
+ class PageFlags {
+ <>
+ Free
+ Allocated
+ Slab
+ }
+```
+
+说明：
+
+- `BuddyAllocator` 自身只维护 section 链表头和 section 数量。
+- 每个 `BuddySection` 都拥有自己的一组 `PageMeta[]` 和 `free_lists`。
+- `PageMeta` 仍然是每页一项的外部元数据。
+- 同一 section 内按 buddy 规则拆分和合并，section 之间不跨界合并。
+
+### 4.2 buddy 初始化
+
+初始化某个 section 时，buddy 会从低地址到高地址扫描该 section 的整段 heap：
+
+- 每次尽量切出“当前地址自然对齐、且仍然放得下”的最大 order 块。
+- 将该块挂到对应 order 的 free list。
+
+这一步的结果是：
+
+- 该 section 中的所有页都被覆盖。
+- 该 section 的 free list 从一开始就处于可分裂、可合并的标准 buddy 形态。
+
+### 4.3 页分配流程
+
+```mermaid
+flowchart TD
+ A[alloc_pages(count, align)] --> B[计算 order]
+ B --> C[计算 align_order]
+ C --> D[effective_order = max(order, align_order)]
+ D --> E[按注册顺序扫描 section]
+ E --> F[在 section 内从 effective_order 向上找非空 free list]
+ F --> G{找到块?}
+ G -- 否 --> H[尝试下一个 section]
+ H --> I{还有 section?}
+ I -- 否 --> J[返回 NoMemory]
+ I -- 是 --> F
+ G -- 是 --> K[弹出大块]
+ K --> L[逐级 split 到目标 order]
+ L --> M[标记头页为 Allocated]
+ M --> N[返回虚拟地址]
+```
+
+注意点：
+
+- `count` 会向上取整到 2 的幂。
+- `align` 也会转成对应的 `align_order`。
+- 真正分配的块大小可能大于用户请求。
+
+### 4.4 低地址页分配
+
+`alloc_pages_lowmem()` 的差异在于：
+
+- 不只看 free list 是否有块。
+- 还要检查块的物理地址区间是否完全落在 `DMA32_LIMIT` 下方。
+
+流程是：
+
+1. 逐个扫描候选 free list。
+2. 对每个候选块调用 `os.virt_to_phys(addr)`。
+3. 计算 `phys + block_bytes <= 4GiB` 是否成立。
+4. 找到合格块后执行和普通分配相同的拆分逻辑。
+
+### 4.5 页释放流程
+
+`dealloc_pages(addr, count)` 的关键点是：
+
+- `addr` 必须是原始返回地址。
+- 实际释放的 order 以 `PageMeta.order` 为准。
+- `count` 主要用于 debug 断言，确保调用者没有传得比真实块更大。
+
+```mermaid
+flowchart TD
+ A[dealloc_pages(addr, count)] --> B[根据 addr 算出 pfn]
+ B --> C[读取 PageMeta.order]
+ C --> D[与 buddy pfn 检查是否可合并]
+ D --> E{buddy 同 order 且空闲?}
+ E -- 是 --> F[从 free list 移除 buddy]
+ F --> G[合并并提升 order]
+ G --> D
+ E -- 否 --> H[将最终块挂回 free list]
+```
+
+## 5. SlabAllocator 设计
+
+`SlabAllocator` 不直接申请页，它只管理“已经拿到的 slab page”。
+
+### 5.1 size class
+
+当前固定 9 个 size class：
+
+- 8
+- 16
+- 32
+- 64
+- 128
+- 256
+- 512
+- 1024
+- 2048
+
+布局规则：
+
+- `layout.size()` 与 `layout.align()` 取较大值。
+- 选择能容纳它的最小 size class。
+- 超过 `2048` 字节则不走 slab。
+
+### 5.2 SlabCache 的三条链
+
+每个 size class 对应一个 `SlabCache`，内部维护三条 intrusive list：
+
+- `partial`
+ 还有空位的 slab，优先分配。
+- `full`
+ 本地 bitmap 没空位，但可能积累了 remote free。
+- `empty`
+ 所有对象都空闲的 slab。
+
+实现上还有限制：
+
+- 每个 `SlabCache` 最多缓存 1 个空 slab。
+- 如果又出现新的空 slab，多出来的那个会还给 buddy。
+
+这也是压力测试里“页数不一定精确回到初始值”的原因之一：每个 CPU、每个 size class 允许保留一个空 slab 作为缓存。
+
+### 5.3 SlabPageHeader
+
+每个 slab page 起始位置都嵌着一个 `SlabPageHeader`。
+
+它包含：
+
+- `magic`
+- `size_class`
+- `object_count`
+- `local_free_count`
+- `owner_cpu`
+- `slab_bytes`
+- `list_prev/list_next`
+- `local_bitmap`
+- `remote_free_head`
+- `remote_free_count`
+
+### 5.4 本地分配
+
+owner CPU 在持有对应 slab 锁时执行本地分配：
+
+1. 先看 `partial` 头 slab。
+2. 如果它有 remote free，先 drain 回本地 bitmap。
+3. 从 bitmap 找一个空位。
+4. 如果分完后满了，把 slab 从 `partial` 挪到 `full`。
+
+如果 `partial` 不行，则：
+
+1. 扫 `full`，找是否有 slab 因 remote free 重新出现空位。
+2. 如果有，把它移回 `partial`。
+3. 还不行就尝试复用 `empty` 中缓存的 slab。
+4. 再不行就返回 `NeedsSlab`，要求上层先提供新页。
+
+## 6. GlobalAllocator 的分配路径
+
+`GlobalAllocator` 是真正对外使用的入口。
+
+它把请求分成两类：
+
+- 小对象：走 slab
+- 大对象：走 buddy
+
+判断条件：
+
+- `layout.size() <= 2048`
+- `layout.align() <= 2048`
+
+满足才走 slab。
+
+### 6.1 alloc 总流程
+
+```mermaid
+flowchart TD
+ A[alloc(layout)] --> B{已初始化?}
+ B -- 否 --> X[NotInitialized]
+ B -- 是 --> C{slab eligible?}
+ C -- 是 --> D[slab_alloc]
+ C -- 否 --> E[large_alloc]
+```
+
+### 6.2 小对象分配
+
+```mermaid
+sequenceDiagram
+ participant Caller
+ participant GA as GlobalAllocator
+ participant S as per-CPU SlabAllocator
+ participant B as BuddyAllocator
+
+ Caller->>GA: alloc(layout)
+ GA->>GA: current_cpu_idx()
+ GA->>S: lock + alloc(layout)
+ alt Slab 命中
+ S-->>GA: Allocated(ptr)
+ GA-->>Caller: ptr
+ else 需要新 slab
+ S-->>GA: NeedsSlab(size_class, pages)
+ GA->>S: drop lock
+ GA->>B: alloc_pages(pages, slab_bytes)
+ GA->>B: set_page_flags(addr, Slab)
+ GA->>S: lock + add_slab(...)
+ GA->>S: alloc(layout)
+ S-->>GA: Allocated(ptr)
+ GA-->>Caller: ptr
+ end
+```
+
+这里有一个很重要的锁顺序细节：
+
+- 当 slab 缺页时，`GlobalAllocator` 会先释放 slab 锁，再去拿 buddy 锁。
+
+这样可以避免：
+
+- 持有 slab 锁时阻塞在 buddy 上。
+- 未来演化成更复杂锁图时出现循环等待。
+
+### 6.3 大对象分配
+
+大对象路径很直接：
+
+1. 根据 `layout.size()` 计算页数。
+2. 对齐至少是 `PAGE_SIZE`。
+3. 调用 `BuddyAllocator::alloc_pages()`。
+
+大对象不会进入 slab，也不会设置 `PageFlags::Slab`。
+
+## 7. dealloc 路径
+
+### 7.1 大对象释放
+
+大对象释放和分配对称：
+
+1. 根据 `layout.size()` 推导页数。
+2. 直接进入 `BuddyAllocator::dealloc_pages()`。
+
+### 7.2 小对象释放：本地路径
+
+如果当前 CPU 就是该 slab 的 owner：
+
+1. 通过对象地址反推出 slab base。
+2. 拿 owner CPU 对应的 slab 锁。
+3. 调用 `slab.dealloc(ptr, layout)`。
+4. 如果结果是 `Done`，结束。
+5. 如果结果是 `FreeSlab { base, pages }`，释放 slab 锁后把整页还给 buddy。
+
+为什么空 slab 不总是立刻还给 buddy：
+
+- `SlabCache` 会保留一个 empty slab 作为热缓存。
+- 只有当已有一个 cached empty slab 时，新的空 slab 才会还给 buddy。
+
+### 7.3 小对象释放：远程路径
+
+如果当前 CPU 不是 owner CPU，不会直接拿对方的 slab 锁。
+
+而是走 lock-free remote free：
+
+```mermaid
+sequenceDiagram
+ participant CPU1 as current CPU
+ participant H as SlabPageHeader
+ participant CPU0 as owner CPU
+ participant S as owner SlabCache
+
+ CPU1->>H: remote_free(obj_addr)
+ Note right of H: CAS 推入 remote_free_head\n并增加 remote_free_count
+ CPU1-->>CPU1: 返回，不加锁
+
+ CPU0->>S: 下次 alloc/dealloc 时持锁进入
+ S->>H: drain_remote_frees()
+ Note right of H: 将 remote 栈中的对象\n重新记回 local bitmap
+ S-->>CPU0: 对象重新可分配
+```
+
+这条路径的核心特点：
+
+- 释放方不需要知道 owner CPU 的锁状态。
+- 释放方不抢远端锁。
+- owner CPU 在后续本地操作时，顺手把 remote free 合并回 bitmap。
+
+这也是当前实现中“跨 CPU free 无锁，但重新利用对象仍由 owner CPU 控制”的关键设计。
+
+## 8. 多线程与并发模型
+
+### 8.1 并发边界
+
+当前实现中真正共享的状态主要有两类：
+
+- `BuddyAllocator`
+ 整体包在一个 `SpinMutex` 里，页级操作串行化。
+- 每 CPU 一个 `SlabAllocator`
+ 每个都各自包在一个 `SpinMutex` 里。
+
+因此并发模型可以概括为：
+
+- 小对象本地路径：只竞争本 CPU 的 slab 锁。
+- 小对象远程 free：不走锁，直接原子 CAS。
+- 缺页或空 slab 回收：会短暂进入全局 buddy 锁。
+
+### 8.2 为什么是 per-CPU slab
+
+如果所有小对象都走单个全局 slab 锁，会有两个问题：
+
+- 多核下锁竞争重。
+- 小对象热点路径和页级路径容易互相干扰。
+
+per-CPU slab 的好处是：
+
+- 本地分配/释放命中时只碰本 CPU 锁。
+- 跨 CPU free 通过 remote list 延后处理，不把释放方拖进远端临界区。
+
+### 8.3 远程释放的正确性依赖
+
+远程释放正确性依赖几个事实：
+
+- slab 页头里记录了 `owner_cpu`。
+- 远程释放只把对象地址压到无锁链表里。
+- owner CPU 持锁 drain 时，再把对象重新放回本地 bitmap。
+- remote 栈中的 next 指针写在对象自身内存开头，这要求被释放对象已经不再被用户使用。
+
+### 8.4 锁顺序
+
+当前实现中建议理解为以下顺序：
+
+- 常规 slab 本地路径：只拿 slab 锁。
+- 常规 buddy 页路径：只拿 buddy 锁。
+- slab 缺页：先释放 slab 锁，再拿 buddy 锁，再重新拿 slab 锁。
+- slab 回收空页：先释放 slab 锁，再拿 buddy 锁。
+
+这样可以避免出现“持有 slab 锁后等待 buddy，同时另一路持有 buddy 锁又等待 slab”的循环。
+
+## 9. 关键实现细节
+
+### 9.1 `PageFlags`
+
+页状态分三种：
+
+- `Free`
+- `Allocated`
+- `Slab`
+
+含义是：
+
+- `Free`：在 buddy free list 中。
+- `Allocated`：普通页分配返回的块头页。
+- `Slab`：该块当前作为 slab 页使用。
+
+### 9.2 buddy 只在块头记录 order
+
+`PageMeta.order` 只有块头页有意义。
+
+因此：
+
+- 分配时只标记头页的 `order`。
+- 释放时调用方必须传回原始返回地址。
+
+### 9.3 `dealloc_pages(count)` 为什么不完全信任 `count`
+
+因为分配时请求可能被扩大：
+
+- `count` 会按 buddy order 向上取整。
+- `align` 也可能把块提升到更高阶。
+
+所以释放时真正信任的是 `PageMeta.order`，`count` 只用于做 debug 级别的合理性检查。
+
+### 9.4 empty slab 缓存策略
+
+每个 `SlabCache` 最多保留一个 empty slab。
+
+这是一个很重要的折中：
+
+- 优点：后续同 size class 再分配时更快，减少 buddy 往返。
+- 代价：空闲页不会总是立刻全部回收到 buddy。
+
+所以测试或观测时不能简单假设：
+
+- “所有对象释放后，buddy free pages 一定精确回到初始值”。
+
+更准确的理解应是：
+
+- 除去每 CPU、每 size class 允许缓存的 empty slab 后，应当没有额外页泄漏。
+
+## 10. realloc 行为
+
+`GlobalAllocator` 通过 `GlobalAlloc` 实现了 `realloc`：
+
+1. 构造新的 `Layout`
+2. 新分配一块内存
+3. 拷贝 `min(old_size, new_size)` 字节
+4. 释放旧块
+
+它不是原地扩容，也没有针对 slab/buddy 做特殊优化。
+
+## 11. 当前实现的优点与限制
+
+### 11.1 优点
+
+- 结构清晰，页级与对象级职责分离。
+- 小对象本地路径开销低。
+- 跨 CPU free 不抢远端锁。
+- 初始化不依赖额外堆分配。
+- 支持低地址页分配。
+
+### 11.2 限制
+
+- buddy 后端仍然是全局单锁。
+- `remote_free` 的对象复用必须等 owner CPU 后续 drain。
+- empty slab 缓存会牺牲一部分“空闲页立刻回收”的极致性。
+- `realloc` 为通用 copy 语义，不做原地优化。
+
+## 12. 阅读源码建议
+
+如果想快速建立整体理解，建议按下面顺序读：
+
+1. `src/lib.rs`
+ 先看公开模块和 `OsImpl`。
+2. `src/global.rs`
+ 先理解门面层如何在 buddy/slab 间路由。
+3. `src/buddy/mod.rs`
+ 看页级分配、拆分和合并。
+4. `src/slab/mod.rs`
+ 看 slab 与上层的接口契约。
+5. `src/slab/cache.rs`
+ 看 `partial/full/empty` 三链切换。
+6. `src/slab/page.rs`
+ 看 bitmap、remote free 栈与 owner CPU drain。
+
+## 13. 一句话总结
+
+这个分配器的核心思想可以概括成一句话：
+
+> 用 buddy 统一管理页，用 per-CPU slab 加速小对象，把跨 CPU free 变成“无锁登记、延迟回收”，从而在实现复杂度、局部性和并发开销之间取得平衡。
diff --git a/src/buddy/buddy_allocator.rs b/src/buddy/buddy_allocator.rs
deleted file mode 100644
index 819cd43..0000000
--- a/src/buddy/buddy_allocator.rs
+++ /dev/null
@@ -1,606 +0,0 @@
-//! Multi-zone buddy allocator using global node pool
-//!
-//! Provides buddy allocator with support for multiple memory zones and
-//! a single shared global node pool for all zones and orders.
-
-use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator};
-
-#[cfg(feature = "log")]
-use log::{debug, error, info, warn};
-
-#[cfg(feature = "tracking")]
-use super::ZoneInfo;
-
-use super::{buddy_block::MAX_ZONES, buddy_set::BuddySet, global_node_pool::GlobalNodePool};
-
-const NODE_POOL_PAGES: usize = 10;
-const NODE_POOL_LOW_WATER_NODES: usize = 128;
-const NODE_POOL_EXPAND_PAGES: usize = 5;
-/// Threshold for low-memory (DMA32-like) physical addresses: 4GiB.
-const LOWMEM_PHYS_THRESHOLD: usize = 1usize << 32;
-
-#[cfg(feature = "tracking")]
-use super::stats::{BuddyStats, MemoryStatsReporter};
-
-/// Buddy page allocator with multi-zone support and global node pool
-///
-/// The `global_node_pool` stores linked-list nodes (ListNode),
-/// which are used to construct the free lists in each BuddySet.
-/// Memory pages themselves are tracked by BuddyBlock values, not by this pool.
-pub struct BuddyPageAllocator {
- zones: [BuddySet; MAX_ZONES],
- num_zones: usize,
- global_node_pool: GlobalNodePool,
- #[cfg(feature = "tracking")]
- stats: BuddyStats,
- /// Optional address translator used to reason about physical addresses.
- addr_translator: Option<&'static dyn crate::AddrTranslator>,
-}
-
-impl BuddyPageAllocator {
- pub const fn new() -> Self {
- Self {
- zones: [const { BuddySet::::empty() }; MAX_ZONES],
- num_zones: 0,
- global_node_pool: GlobalNodePool::new(),
- #[cfg(feature = "tracking")]
- stats: BuddyStats::new(),
- addr_translator: None,
- }
- }
-
- /// Set the address translator so that the allocator can reason about
- /// physical address ranges (e.g. low-memory regions below 4GiB).
- pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) {
- self.addr_translator = Some(translator);
- }
-
- /// Initialize the global node pool and bootstrap with initial memory region
- pub fn init(&mut self, base_addr: usize, size: usize) {
- self.bootstrap(base_addr, size);
- }
-
- /// Bootstrap allocator with initial memory region
- pub fn bootstrap(&mut self, base_addr: usize, size: usize) {
- if self.num_zones >= MAX_ZONES {
- panic!("Cannot bootstrap: maximum zones reached");
- }
-
- // Reserve a small region at the beginning for the global node pool
- let node_region_size = NODE_POOL_PAGES * PAGE_SIZE;
- if size <= node_region_size {
- panic!("Cannot bootstrap: region too small for node pool");
- }
- let node_region_start = base_addr;
-
- self.global_node_pool
- .init(node_region_start, node_region_size);
-
- let zone_base = base_addr + node_region_size;
- let zone_size = size - node_region_size;
-
- self.zones[0] = BuddySet::new(zone_base, zone_size, 0);
- self.zones[0].init(&mut self.global_node_pool, zone_base, zone_size);
- // Mark whether this zone contains any low (<4G) physical memory, if translator is set.
- if let Some(translator) = self.addr_translator {
- if let Some(pa_start) = translator.virt_to_phys(zone_base) {
- if pa_start < LOWMEM_PHYS_THRESHOLD {
- self.zones[0].is_lowmem = true;
- }
- }
- }
- self.num_zones = 1;
-
- #[cfg(feature = "tracking")]
- self.update_stats();
- }
-
- #[cfg(feature = "tracking")]
- pub fn get_stats(&self) -> BuddyStats {
- self.stats
- }
-
- /// Get global node pool statistics
- pub fn get_node_pool_stats(&self) -> super::global_node_pool::GlobalPoolStats {
- self.global_node_pool.get_stats()
- }
-
- /// Get number of zones in the allocator
- pub fn get_zone_count(&self) -> usize {
- self.num_zones
- }
-
- /// Allocate contiguous low-memory pages (physical address < 4GiB).
- ///
- /// This API is intended for DMA32-like use cases. It only considers zones
- /// that are marked as `is_lowmem`, and after allocation it performs a
- /// strict physical boundary check to ensure that both the start and end
- /// physical addresses are below the 4GiB threshold.
- pub fn alloc_pages_lowmem(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- if num_pages == 0 {
- return Err(AllocError::InvalidParam);
- }
-
- let translator = self.addr_translator.ok_or(AllocError::InvalidParam)?;
-
- // Try to expand node pool if we are close to exhaustion
- self.maybe_expand_node_pool();
-
- let size_bytes = num_pages * PAGE_SIZE;
-
- for i in 0..self.num_zones {
- if !self.zones[i].is_lowmem {
- continue;
- }
-
- match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) {
- Ok(addr) => {
- let start_va = addr;
- let end_va = addr + size_bytes - 1;
-
- // Ensure both start and end physical addresses are below the
- // low-memory threshold to avoid crossing the 4GiB boundary.
- if let (Some(pa_start), Some(pa_end)) = (
- translator.virt_to_phys(start_va),
- translator.virt_to_phys(end_va),
- ) {
- if pa_start < LOWMEM_PHYS_THRESHOLD && pa_end < LOWMEM_PHYS_THRESHOLD {
- #[cfg(feature = "tracking")]
- self.update_stats();
- return Ok(addr);
- }
- }
-
- // Boundary check failed: roll back this allocation and
- // continue searching for another suitable block.
- self.zones[i].dealloc_pages(&mut self.global_node_pool, addr, num_pages);
- }
- Err(_) => {
- continue;
- }
- }
- }
-
- info!(
- "buddy allocator: Low-memory allocation failure: {} Byte, align {}",
- num_pages * PAGE_SIZE,
- alignment
- );
- Err(AllocError::NoMemory)
- }
-
- /// Get free blocks of a specific order from a zone
- /// Returns None if zone doesn't exist
- pub fn get_free_blocks_by_order(
- &self,
- zone_id: usize,
- order: u32,
- ) -> Option> {
- if zone_id >= self.num_zones {
- return None;
- }
- Some(self.zones[zone_id].get_free_blocks_by_order(&self.global_node_pool, order))
- }
-
- /// Update aggregated statistics from all zones
- #[cfg(feature = "tracking")]
- fn update_stats(&mut self) {
- let mut total_stats = BuddyStats::new();
-
- for i in 0..self.num_zones {
- let zone_stats = self.zones[i].get_stats();
- total_stats.add(&zone_stats);
- }
-
- self.stats = total_stats;
- }
-
- /// Ensure the global node pool has enough free nodes.
- ///
- /// When the number of free nodes falls below a low-water mark,
- /// try to reserve a few pages from any available zone and add them as a new node region.
- fn maybe_expand_node_pool(&mut self) {
- let free_nodes = self.global_node_pool.free_node_count();
- if free_nodes >= NODE_POOL_LOW_WATER_NODES {
- return;
- }
- if self.num_zones == 0 {
- error!("buddy allocator: no zones available to expand node pool");
- return;
- }
-
- let expand_pages = NODE_POOL_EXPAND_PAGES;
- let expand_size = expand_pages * PAGE_SIZE;
-
- // Try all zones to allocate memory for node pool expansion
- for i in 0..self.num_zones {
- match self.zones[i].alloc_pages(&mut self.global_node_pool, expand_pages, PAGE_SIZE) {
- Ok(addr) => {
- debug!(
- "buddy allocator: expanding node pool from zone {}: free_nodes={} region=[{:#x}, {:#x})",
- i,
- free_nodes,
- addr,
- addr + expand_size
- );
- self.global_node_pool.add_region(addr, expand_size);
- return;
- }
- Err(_) => {
- continue;
- }
- }
- }
-
- warn!(
- "buddy allocator: failed to expand node pool at low water: free_nodes={} tried {} zones",
- free_nodes,
- self.num_zones
- );
- }
-
- /// Add a new memory region as a new zone
- pub fn add_memory_region(&mut self, start: usize, size: usize) -> AllocResult<()> {
- if self.num_zones >= MAX_ZONES {
- error!("buddy allocator: Cannot add region: maximum zones ({MAX_ZONES}) reached");
- return Err(AllocError::NoMemory);
- }
-
- // Align to page boundaries
- let aligned_start = start & !(PAGE_SIZE - 1);
- let end = start + size;
- let aligned_end = (end + PAGE_SIZE - 1) & !(PAGE_SIZE - 1);
- let aligned_size = aligned_end - aligned_start;
-
- if aligned_size == 0 || aligned_size < PAGE_SIZE {
- warn!("buddy allocator: Aligned size is too small: {aligned_size:#x}, skipping region");
- return Err(AllocError::InvalidParam);
- }
-
- // Check for overlap with existing zones
- for i in 0..self.num_zones {
- let zone = &self.zones[i];
- if !(aligned_end <= zone.base_addr || aligned_start >= zone.end_addr) {
- error!(
- "buddy allocator: Region [{:#x}, {:#x}) overlaps with zone {} [{:#x}, {:#x})",
- aligned_start, aligned_end, i, zone.base_addr, zone.end_addr
- );
- return Err(AllocError::MemoryOverlap);
- }
- }
-
- let zone_id = self.num_zones;
- self.zones[zone_id] = BuddySet::new(aligned_start, aligned_size, zone_id);
- self.zones[zone_id].init(&mut self.global_node_pool, aligned_start, aligned_size);
- // Mark whether this zone contains any low (<4G) physical memory, if translator is set.
- if let Some(translator) = self.addr_translator {
- if let Some(pa_start) = translator.virt_to_phys(aligned_start) {
- if pa_start < LOWMEM_PHYS_THRESHOLD {
- self.zones[zone_id].is_lowmem = true;
- }
- }
- }
- self.num_zones += 1;
-
- // Print all zone information after successfully adding a new memory region
- #[cfg(feature = "tracking")]
- self.print_zone_info();
-
- Ok(())
- }
-
- /// Find the zone that contains the given address
- pub fn find_zone_for_addr(&self, addr: usize) -> Option {
- (0..self.num_zones).find(|&i| self.zones[i].addr_in_zone(addr))
- }
-
- /// Print detailed allocation failure statistics
- ///
- /// This method is public to allow CompositePageAllocator to call it
- /// when allocation fails, providing detailed per-zone failure information.
- #[cfg(feature = "tracking")]
- pub fn print_alloc_failure_stats(&self, num_pages: usize, alignment: usize) {
- let mut zone_infos: [Option; MAX_ZONES] = [None; MAX_ZONES];
- let mut zone_stats: [Option; MAX_ZONES] = [None; MAX_ZONES];
-
- for i in 0..self.num_zones {
- zone_infos[i] = Some(self.zones[i].zone_info());
- zone_stats[i] = Some(self.zones[i].get_stats());
- }
-
- // Create slices from the initialized elements
- let zone_infos_slice: &[ZoneInfo] = unsafe {
- core::slice::from_raw_parts(zone_infos.as_ptr() as *const ZoneInfo, self.num_zones)
- };
- let zone_stats_slice: &[BuddyStats] = unsafe {
- core::slice::from_raw_parts(zone_stats.as_ptr() as *const BuddyStats, self.num_zones)
- };
-
- MemoryStatsReporter::print_alloc_failure_stats(
- PAGE_SIZE,
- self.num_zones,
- &self.stats,
- zone_infos_slice,
- zone_stats_slice,
- num_pages,
- alignment,
- );
- }
-
- /// Print all zone information and block distribution
- pub fn print_zone_info(&self) {
- info!("========== Buddy Allocator Zones Info ==========");
- info!("Total zones: {}", self.num_zones);
- info!("Page size: {PAGE_SIZE:#x} ({PAGE_SIZE})");
- info!("");
-
- for i in 0..self.num_zones {
- let zone = &self.zones[i];
- let _zone_info = zone.zone_info();
- info!("Zone {i}:");
- info!(
- " Address range: [{:#x}, {:#x})",
- _zone_info.start_addr, _zone_info.end_addr
- );
- info!(" Total pages: {}", _zone_info.total_pages);
- info!(
- " Total size: {:#x} ({} MB)",
- _zone_info.total_pages * PAGE_SIZE,
- (_zone_info.total_pages * PAGE_SIZE) / (1024 * 1024)
- );
- info!(" Free blocks distribution:");
-
- // Print block distribution for each order
- for order in 0..=zone.max_order() {
- let block_count = zone.get_order_block_count(order);
- if block_count > 0 {
- let _block_size = (1 << order) * PAGE_SIZE;
- info!(
- " Order {}: {} blocks (size {} bytes each, total {:#x})",
- order,
- block_count,
- _block_size,
- block_count * _block_size
- );
- }
- }
- info!("");
- }
-
- info!("Global node pool stats:");
- let _pool_stats = self.global_node_pool.get_stats();
- info!(" Total allocations: {}", _pool_stats.total_allocations);
- info!(" Free nodes: {}", _pool_stats.free_nodes);
- info!("==============================================");
- }
-
- #[cfg(not(feature = "tracking"))]
- pub fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) {
- // No-op when tracking is disabled
- }
-}
-
-impl crate::slab::PageAllocatorForSlab for BuddyPageAllocator {
- fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- ::alloc_pages(self, num_pages, alignment)
- }
-
- fn dealloc_pages(&mut self, pos: usize, num_pages: usize) {
- ::dealloc_pages(self, pos, num_pages);
- }
-}
-
-impl Default for BuddyPageAllocator {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl BaseAllocator for BuddyPageAllocator {
- fn init(&mut self, start: usize, size: usize) {
- self.bootstrap(start, size);
- }
-
- fn add_memory(&mut self, start: usize, size: usize) -> AllocResult<()> {
- self.add_memory_region(start, size)?;
- #[cfg(feature = "tracking")]
- self.update_stats();
- Ok(())
- }
-}
-
-impl PageAllocator for BuddyPageAllocator {
- const PAGE_SIZE: usize = PAGE_SIZE;
-
- fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- // Try to expand node pool if we are close to exhaustion
- self.maybe_expand_node_pool();
-
- // First, prefer zones that are NOT marked as low-memory so that
- // low-memory regions can be reserved for special use (e.g. DMA32).
- for i in 0..self.num_zones {
- if self.zones[i].is_lowmem {
- continue;
- }
- match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) {
- Ok(addr) => {
- #[cfg(feature = "tracking")]
- self.update_stats();
- return Ok(addr);
- }
- Err(_) => {
- continue;
- }
- }
- }
-
- // Then, fall back to zones that are marked as low-memory.
- for i in 0..self.num_zones {
- if !self.zones[i].is_lowmem {
- continue;
- }
- match self.zones[i].alloc_pages(&mut self.global_node_pool, num_pages, alignment) {
- Ok(addr) => {
- #[cfg(feature = "tracking")]
- self.update_stats();
- return Ok(addr);
- }
- Err(_) => {
- continue;
- }
- }
- }
-
- debug!(
- "buddy allocator: Allocation failure: {} Byte, align {}",
- num_pages * PAGE_SIZE,
- alignment
- );
- Err(AllocError::NoMemory)
- }
-
- fn dealloc_pages(&mut self, pos: usize, num_pages: usize) {
- self.maybe_expand_node_pool();
- if let Some(zone_idx) = self.find_zone_for_addr(pos) {
- self.zones[zone_idx].dealloc_pages(&mut self.global_node_pool, pos, num_pages);
- #[cfg(feature = "tracking")]
- self.update_stats();
- } else {
- warn!("buddy allocator: Dealloc pages at {pos:#x}: address not in any zone");
- }
- }
-
- fn alloc_pages_at(
- &mut self,
- base: usize,
- num_pages: usize,
- alignment: usize,
- ) -> AllocResult {
- // Try to expand node pool if we are close to exhaustion
- self.maybe_expand_node_pool();
-
- if let Some(zone_idx) = self.find_zone_for_addr(base) {
- match self.zones[zone_idx].alloc_pages_at(
- &mut self.global_node_pool,
- base,
- num_pages,
- alignment,
- ) {
- Ok(addr) => {
- #[cfg(feature = "tracking")]
- self.update_stats();
- Ok(addr)
- }
- Err(e) => Err(e),
- }
- } else {
- warn!("buddy allocator: alloc_pages_at: address {base:#x} not in any zone");
- Err(AllocError::InvalidParam)
- }
- }
-
- fn total_pages(&self) -> usize {
- #[cfg(feature = "tracking")]
- return self.stats.total_pages;
- #[cfg(not(feature = "tracking"))]
- return 0;
- }
-
- fn used_pages(&self) -> usize {
- #[cfg(feature = "tracking")]
- return self.stats.used_pages;
- #[cfg(not(feature = "tracking"))]
- return 0;
- }
-
- fn available_pages(&self) -> usize {
- #[cfg(feature = "tracking")]
- return self.stats.free_pages;
- #[cfg(not(feature = "tracking"))]
- return 0;
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use alloc::alloc::{alloc, dealloc};
- use core::alloc::Layout;
-
- const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024;
- const TEST_PAGE_SIZE: usize = 0x1000;
-
- fn alloc_test_heap(size: usize) -> (*mut u8, Layout) {
- let layout = Layout::from_size_align(size, TEST_PAGE_SIZE).unwrap();
- let ptr = unsafe { alloc(layout) };
- assert!(!ptr.is_null());
- (ptr, layout)
- }
-
- fn dealloc_test_heap(ptr: *mut u8, layout: Layout) {
- unsafe { dealloc(ptr, layout) };
- }
-
- #[test]
- fn test_buddy_allocator_init() {
- let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE);
- let heap_addr = heap_ptr as usize;
-
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_addr, TEST_HEAP_SIZE);
-
- let addr = allocator.alloc_pages(1, TEST_PAGE_SIZE);
- assert!(addr.is_ok());
- if let Ok(a) = addr {
- allocator.dealloc_pages(a, 1);
- }
-
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-
- #[test]
- fn test_buddy_allocator_multi_pages() {
- let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE);
- let heap_addr = heap_ptr as usize;
-
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_addr, TEST_HEAP_SIZE);
-
- let addr1 = allocator.alloc_pages(4, TEST_PAGE_SIZE).unwrap();
- let addr2 = allocator.alloc_pages(8, TEST_PAGE_SIZE).unwrap();
- assert_ne!(addr1, addr2);
-
- allocator.dealloc_pages(addr1, 4);
- allocator.dealloc_pages(addr2, 8);
-
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-
- #[test]
- fn test_buddy_allocator_alignment() {
- let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE);
- let heap_addr = heap_ptr as usize;
-
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_addr, TEST_HEAP_SIZE);
-
- let addr = allocator.alloc_pages(1, TEST_PAGE_SIZE * 4).unwrap();
- assert_eq!(addr & (TEST_PAGE_SIZE * 4 - 1), 0);
-
- allocator.dealloc_pages(addr, 1);
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-
- #[test]
- fn test_buddy_allocator_zone_count() {
- let (heap_ptr, heap_layout) = alloc_test_heap(TEST_HEAP_SIZE);
- let heap_addr = heap_ptr as usize;
-
- let mut allocator = BuddyPageAllocator::::new();
- allocator.init(heap_addr, TEST_HEAP_SIZE);
-
- assert_eq!(allocator.get_zone_count(), 1);
-
- dealloc_test_heap(heap_ptr, heap_layout);
- }
-}
diff --git a/src/buddy/buddy_block.rs b/src/buddy/buddy_block.rs
deleted file mode 100644
index 297475d..0000000
--- a/src/buddy/buddy_block.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-//! Buddy block metadata
-//!
-//! Represents a block of memory in the buddy system with order and address information.
-
-use core::cmp::PartialOrd;
-
-/// Maximum number of memory zones supported
-pub const MAX_ZONES: usize = 32;
-
-/// Maximum order supported
-pub const DEFAULT_MAX_ORDER: usize = 28; // Support up to 256GB allocations (2^28 * 4KB)
-
-/// Buddy block metadata
-#[derive(Debug, Clone, Copy)]
-pub struct BuddyBlock {
- pub order: usize,
- pub addr: usize,
-}
-
-impl BuddyBlock {
- /// Create a new buddy block
- pub const fn new(order: usize, addr: usize) -> Self {
- Self { order, addr }
- }
-
- /// Calculate the buddy address for this block
- /// The buddy is the other half of the parent block at the next higher order
- /// For a block at order k with address A, its buddy is at A ^ (2^k * PAGE_SIZE)
- #[allow(dead_code)]
- pub fn buddy_addr(&self) -> usize {
- self.addr ^ ((1 << self.order) * PAGE_SIZE)
- }
-}
-
-impl PartialOrd for BuddyBlock {
- fn partial_cmp(&self, other: &Self) -> Option {
- self.addr.partial_cmp(&other.addr)
- }
-}
-
-impl PartialEq for BuddyBlock {
- fn eq(&self, other: &Self) -> bool {
- self.addr == other.addr && self.order == other.order
- }
-}
-
-impl Eq for BuddyBlock {}
-
-/// A memory zone descriptor (similar to Linux kernel's zone)
-#[derive(Debug, Clone, Copy)]
-pub struct ZoneInfo {
- pub start_addr: usize,
- pub end_addr: usize,
- pub total_pages: usize,
- pub zone_id: usize,
-}
diff --git a/src/buddy/buddy_set.rs b/src/buddy/buddy_set.rs
deleted file mode 100644
index 61bfab1..0000000
--- a/src/buddy/buddy_set.rs
+++ /dev/null
@@ -1,612 +0,0 @@
-//! Single-zone buddy allocator using global node pool
-//!
-//! Implements the core buddy system for a single memory zone using
-//! pooled linked lists that draw nodes from a shared global pool.
-
-use crate::{AllocError, AllocResult};
-
-#[cfg(feature = "log")]
-use log::{error, warn};
-
-use super::{
- buddy_block::{BuddyBlock, ZoneInfo, DEFAULT_MAX_ORDER},
- global_node_pool::GlobalNodePool,
- pooled_list::PooledLinkedList,
-};
-
-/// A buddy set implementation - represents a single zone
-///
-/// Uses pooled linked lists with global node pool for efficient memory usage.
-/// All zones share the same global node pool.
-pub struct BuddySet {
- pub(crate) base_addr: usize,
- pub(crate) end_addr: usize,
- total_pages: usize,
- zone_id: usize,
- /// Whether this zone contains any memory that can map to low (<4G) physical addresses.
- pub(crate) is_lowmem: bool,
- /// Free lists for each order
- free_lists: [PooledLinkedList; DEFAULT_MAX_ORDER + 1],
-}
-
-impl BuddySet {
- /// Create a new buddy set for a zone (uninitialized, must call init())
- pub const fn new(base_addr: usize, size: usize, zone_id: usize) -> Self {
- Self {
- base_addr,
- end_addr: base_addr + size,
- total_pages: size / PAGE_SIZE,
- zone_id,
- is_lowmem: false,
- free_lists: [const { PooledLinkedList::new() }; DEFAULT_MAX_ORDER + 1],
- }
- }
-
- /// Create an empty buddy set
- pub const fn empty() -> Self {
- Self::new(0, 0, 0)
- }
-
- pub const fn max_order(&self) -> usize {
- DEFAULT_MAX_ORDER
- }
-
- /// Add a block to the appropriate list for its order
- fn add_block_to_order(
- &mut self,
- pool: &mut GlobalNodePool,
- order: usize,
- block: BuddyBlock,
- ) -> bool {
- if order > DEFAULT_MAX_ORDER {
- error!(
- "zone {}: Order {} exceeds maximum order {}",
- self.zone_id, order, DEFAULT_MAX_ORDER
- );
- return false;
- }
-
- self.free_lists[order].insert_sorted(pool, block)
- }
-
- /// Find a block with the given address in the free list for its order
- fn find_block_in_order(
- &self,
- pool: &GlobalNodePool,
- order: usize,
- addr: usize,
- ) -> Option<(usize, Option)> {
- if order > DEFAULT_MAX_ORDER {
- return None;
- }
- self.free_lists[order].find_by_addr(pool, addr)
- }
-
- /// Remove a block from its list
- #[allow(dead_code)]
- fn remove_block_from_order(
- &mut self,
- pool: &mut GlobalNodePool,
- order: usize,
- node_idx: usize,
- ) -> bool {
- if order > DEFAULT_MAX_ORDER {
- error!(
- "zone {}: Order {} exceeds maximum order {}",
- self.zone_id, order, DEFAULT_MAX_ORDER
- );
- return false;
- }
- self.free_lists[order].remove(pool, node_idx)
- }
-
- /// Check if an address belongs to this zone
- pub fn addr_in_zone(&self, addr: usize) -> bool {
- addr >= self.base_addr && addr < self.end_addr
- }
-
- /// Get zone information
- pub fn zone_info(&self) -> ZoneInfo {
- ZoneInfo {
- start_addr: self.base_addr,
- end_addr: self.end_addr,
- total_pages: self.total_pages,
- zone_id: self.zone_id,
- }
- }
-
- /// Initialize the buddy set with a memory region
- pub fn init(&mut self, pool: &mut GlobalNodePool, base_addr: usize, size: usize) {
- let aligned_base = base_addr & !(PAGE_SIZE - 1);
- let end = base_addr + size;
- let aligned_end = (end + PAGE_SIZE - 1) & !(PAGE_SIZE - 1);
- let aligned_size = aligned_end - aligned_base;
-
- if aligned_size == 0 || aligned_size < PAGE_SIZE {
- panic!("Aligned size is too small: {:#x}", aligned_size);
- }
-
- self.base_addr = aligned_base;
- self.end_addr = aligned_end;
- self.total_pages = aligned_size / PAGE_SIZE;
-
- // Reset free lists
- for list in &mut self.free_lists {
- list.clear(pool);
- }
-
- self.init_free_blocks(pool);
- }
-
- fn init_free_blocks(&mut self, pool: &mut GlobalNodePool) {
- let mut remaining_pages = self.total_pages;
- let mut current_addr = self.base_addr;
-
- while remaining_pages > 0 {
- // Find the maximum order constrained by:
- // 1. Remaining pages (can't allocate more than what's left)
- // 2. Address alignment (block address must be aligned to block_size)
- // 3. Maximum supported order
-
- // Maximum order based on remaining pages
- let max_order_by_pages = if remaining_pages.is_power_of_two() {
- remaining_pages.trailing_zeros() as usize
- } else {
- // For non-power-of-2, use the highest bit position
- (remaining_pages.next_power_of_two() >> 1).trailing_zeros() as usize
- };
-
- // Maximum order based on address alignment
- // For each order, check if current_addr is aligned to (2^order * PAGE_SIZE)
- let mut max_order_by_alignment = 0;
- for test_order in 0..=self.max_order() {
- let block_size = (1 << test_order) * PAGE_SIZE;
- if current_addr.is_multiple_of(block_size) {
- max_order_by_alignment = test_order;
- } else {
- break;
- }
- }
-
- // Take the minimum of all constraints
- let order = max_order_by_pages
- .min(max_order_by_alignment)
- .min(self.max_order());
-
- let block_pages = 1 << order;
- let block_size = block_pages * PAGE_SIZE;
-
- // Verify alignment: address must be exact multiple of block_size
- assert!(
- current_addr & (block_size - 1) == 0,
- "Block address {current_addr:#x} not aligned to block size {block_size:#x} (order {order})"
- );
-
- // Construct and add the block
- let block = BuddyBlock {
- order,
- addr: current_addr,
- };
-
- if !self.add_block_to_order(pool, order, block) {
- error!(
- "zone {}: Failed to add block during fast init: addr={:#x}, order={}, remaining_pages={}",
- self.zone_id, current_addr, order, remaining_pages
- );
- // Critical failure during initialization
- panic!("Failed to initialize buddy system");
- }
-
- current_addr += block_size;
- remaining_pages -= block_pages;
- }
- }
-
- /// Allocate pages using buddy system
- pub fn alloc_pages(
- &mut self,
- pool: &mut GlobalNodePool,
- num_pages: usize,
- alignment: usize,
- ) -> AllocResult {
- if num_pages == 0 {
- return Err(AllocError::InvalidParam);
- }
-
- // Find the required order (round up to next power of 2)
- let required_order = if num_pages.is_power_of_two() {
- num_pages.trailing_zeros() as usize
- } else {
- num_pages.next_power_of_two().trailing_zeros() as usize
- };
-
- if required_order > self.max_order() {
- error!(
- "required order: {}, max order: {}",
- required_order,
- self.max_order()
- );
- return Err(AllocError::NoMemory);
- }
-
- let align_pages = alignment.div_ceil(PAGE_SIZE);
- let align_order = align_pages.trailing_zeros() as usize;
- let order_needed = required_order.max(align_order);
-
- // Try to find a block of the required order or higher
- for order in order_needed..=self.max_order() {
- if !self.free_lists[order].is_empty() {
- let mut block = self.free_lists[order].pop_front(pool).unwrap();
-
- // Split down to required order
- while block.order > order_needed {
- block.order -= 1;
- let split_size = (1 << block.order) * PAGE_SIZE;
- let buddy_addr = block.addr + split_size;
-
- // Push the second half back to free list (sorted!)
- let success = self.add_block_to_order(
- pool,
- block.order,
- BuddyBlock {
- order: block.order,
- addr: buddy_addr,
- },
- );
- if !success {
- warn!(
- "Failed to push buddy block to free list during split at order {}",
- block.order
- );
- // Put the original block back
- self.add_block_to_order(pool, block.order + 1, block);
- return Err(AllocError::NoMemory);
- }
- }
-
- // Verify alignment requirement
- assert!(
- block.addr.is_multiple_of(alignment),
- "Allocated address {:#x} is not aligned to {:#x} bytes ",
- block.addr,
- alignment
- );
-
- return Ok(block.addr);
- }
- }
-
- Err(AllocError::NoMemory)
- }
-
- /// Deallocate pages back to buddy system with automatic merging
- pub fn dealloc_pages(&mut self, pool: &mut GlobalNodePool, addr: usize, num_pages: usize) {
- if num_pages == 0 {
- warn!("zone {}: Trying to deallocate 0 pages", self.zone_id);
- return;
- }
-
- // Validate address belongs to this zone
- if !self.addr_in_zone(addr) {
- error!(
- "zone {}: Address {:#x} not in zone [{:#x}, {:#x})",
- self.zone_id, addr, self.base_addr, self.end_addr
- );
- return;
- }
-
- // Calculate order for this deallocation
- // Handle non-power-of-2 allocations by rounding up (same as alloc_pages)
- let mut order = if num_pages.is_power_of_two() {
- num_pages.trailing_zeros() as usize
- } else {
- num_pages.next_power_of_two().trailing_zeros() as usize
- };
-
- if order > DEFAULT_MAX_ORDER {
- error!(
- "zone {}: Order {} exceeds maximum supported order {}",
- self.zone_id, order, DEFAULT_MAX_ORDER
- );
- return;
- }
-
- // Convert address and pages to PFN (Page Frame Number)
- let pfn = addr / PAGE_SIZE;
-
- // Check alignment using PFN
- if pfn & ((1 << order) - 1) != 0 {
- error!(
- "zone {}: Page PFN {} is not properly aligned for order {} (needs alignment to {} pages)",
- self.zone_id, pfn, order, 1 << order
- );
- return;
- }
-
- // Check page alignment
- if addr & (PAGE_SIZE - 1) != 0 {
- error!(
- "zone {}: Attempt to free page at non-page-aligned address {:#x}",
- self.zone_id, addr
- );
- return;
- }
-
- // Integrated double-free detection and buddy merging
- let initial_order = order;
- let size = (1 << initial_order) * PAGE_SIZE;
-
- // 1. Descendant check: Check if any part of the block being freed is already free
- // This handles cases where a large block is freed but contains already free sub-blocks
- for i in 0..initial_order {
- if self.free_lists[i].has_block_in_range(pool, addr, addr + size) {
- warn!(
- "zone {}: Double free (descendant) detected at order {} in range [{:#x}, {:#x})",
- self.zone_id, i, addr, addr + size
- );
- return;
- }
- }
-
- // 2. Ancestor check and Merging loop
- // We check from initial_order up to max_order.
- // For each i, we check if the current aligned base is in free_lists[i].
- let mut current_addr = addr;
- let mut merging = true;
-
- for i in initial_order..=self.max_order() {
- let block_size = (1 << i) * PAGE_SIZE;
- let current_base = current_addr & !(block_size - 1);
-
- // Double-free detection (Ancestor check): Check if this block or its parent is already free
- if self.find_block_in_order(pool, i, current_base).is_some() {
- warn!(
- "zone {}: Double free detected at addr {:#x} (found at order {})",
- self.zone_id, addr, i
- );
- return;
- }
-
- // Buddy merging
- if merging && i < self.max_order() {
- let buddy_addr = current_base ^ block_size;
- if self.addr_in_zone(buddy_addr) {
- if let Some((node_idx, prev_idx)) =
- self.find_block_in_order(pool, i, buddy_addr)
- {
- // Buddy found, remove it and continue merging at next order
- self.free_lists[i].remove_with_prev(pool, node_idx, prev_idx);
- current_addr = current_base & buddy_addr;
- order = i + 1;
- continue;
- }
- }
- // No buddy found or out of zone, stop merging but continue checking for double frees
- merging = false;
- }
- }
-
- // Add the final merged block to the appropriate free list (sorted!)
- let final_addr = current_addr;
- let block = BuddyBlock {
- order,
- addr: final_addr,
- };
-
- if !self.add_block_to_order(pool, order, block) {
- error!(
- "zone {}: Failed to push block to free list: addr={:#x}, order={}",
- self.zone_id, final_addr, order
- );
- }
- }
-
- /// Get statistics for this zone
- #[cfg(feature = "tracking")]
- pub fn get_stats(&self) -> super::stats::BuddyStats {
- let mut stats = super::stats::BuddyStats::new();
- stats.total_pages = self.total_pages;
-
- for order in 0..=DEFAULT_MAX_ORDER {
- let block_count = self.free_lists[order].len();
- stats.free_pages_by_order[order] = block_count;
- stats.free_pages += block_count * (1 << order);
- }
-
- stats.used_pages = stats.total_pages.saturating_sub(stats.free_pages);
- stats
- }
-
- /// Get free blocks of a specific order as an iterator
- pub fn get_free_blocks_by_order<'a>(
- &'a self,
- pool: &'a GlobalNodePool,
- order: u32,
- ) -> super::pooled_list::PooledListIter<'a> {
- self.free_lists[order as usize].iter(pool)
- }
-
- /// Get the number of blocks in a specific order
- pub fn get_order_block_count(&self, order: usize) -> usize {
- if order <= DEFAULT_MAX_ORDER {
- self.free_lists[order].len()
- } else {
- 0
- }
- }
-
- /// Allocate pages at a specific address
- ///
- /// This method allocates memory at a specific address. If the address range
- /// is completely free, it will be allocated. If part of a larger free block,
- /// the block will be split appropriately.
- pub fn alloc_pages_at(
- &mut self,
- pool: &mut GlobalNodePool,
- base: usize,
- num_pages: usize,
- alignment: usize,
- ) -> AllocResult {
- if num_pages == 0 {
- return Err(AllocError::InvalidParam);
- }
-
- // Check if address belongs to this zone
- if !self.addr_in_zone(base) {
- error!(
- "zone {}: Address {:#x} not in zone [{:#x}, {:#x})",
- self.zone_id, base, self.base_addr, self.end_addr
- );
- return Err(AllocError::InvalidParam);
- }
-
- // Check page alignment
- if base & (PAGE_SIZE - 1) != 0 {
- error!(
- "zone {}: Address {:#x} is not page-aligned",
- self.zone_id, base
- );
- return Err(AllocError::InvalidParam);
- }
-
- // Check alignment requirement
- if !base.is_multiple_of(alignment) {
- error!(
- "zone {}: Address {:#x} is not aligned to {:#x}",
- self.zone_id, base, alignment
- );
- return Err(AllocError::InvalidParam);
- }
-
- // Check if range fits in zone
- let size = num_pages * PAGE_SIZE;
- if base + size > self.end_addr {
- error!(
- "zone {}: Allocation range [{:#x}, {:#x}) exceeds zone end {:#x}",
- self.zone_id,
- base,
- base + size,
- self.end_addr
- );
- return Err(AllocError::InvalidParam);
- }
-
- // Calculate required order (round up to next power of 2 if needed)
- let required_order = if num_pages.is_power_of_two() {
- num_pages.trailing_zeros() as usize
- } else {
- num_pages.next_power_of_two().trailing_zeros() as usize
- };
-
- // Calculate the order for the block that contains this address
- // The block must be aligned to its size
- let pfn = base / PAGE_SIZE;
- let aligned_pfn = pfn & !((1 << required_order) - 1);
-
- // Check if the requested base is properly aligned for its size
- if aligned_pfn != pfn {
- error!(
- "zone {}: Base address {:#x} (PFN {}) is not aligned for {} pages",
- self.zone_id,
- base,
- pfn,
- 1 << required_order
- );
- return Err(AllocError::InvalidParam);
- }
-
- // Try to find a free block that contains this address
- // Start from the required order and go up to larger blocks
- for order in required_order..=self.max_order() {
- let block_pfn = pfn & !((1 << order) - 1);
- let block_addr = block_pfn * PAGE_SIZE;
-
- // Check if this order can contain the request
- if let Some((node_idx, prev_idx)) = self.find_block_in_order(pool, order, block_addr) {
- // Verify the block is indeed in the free list and capture its data
- let node_data = {
- let node = pool.get_node(node_idx).unwrap();
- if node.data.order != order || node.data.addr != block_addr {
- continue;
- }
- node.data
- };
-
- // Remove this block from free list using prev_idx for O(1) deletion
- self.free_lists[order].remove_with_prev(pool, node_idx, prev_idx);
-
- // Now we have a larger block, need to split it
- // to keep only the part that covers [base, base + size)
- let mut current_block = BuddyBlock {
- order,
- addr: block_addr,
- };
-
- // Split down to required order, keeping the requested region
- while current_block.order > required_order {
- current_block.order -= 1;
- let split_size = (1 << current_block.order) * PAGE_SIZE;
-
- // Calculate buddy address
- let buddy_addr = current_block.addr + split_size;
-
- // Check if buddy is part of the requested region
- let request_end = base + size;
-
- if buddy_addr < base || buddy_addr >= request_end {
- // Buddy is outside the requested region, add it back to free list
- let success = self.add_block_to_order(
- pool,
- current_block.order,
- BuddyBlock {
- order: current_block.order,
- addr: buddy_addr,
- },
- );
- if !success {
- error!(
- "zone {}: Failed to return buddy block during split",
- self.zone_id
- );
- // Put the original block back and fail
- self.add_block_to_order(pool, order, node_data);
- return Err(AllocError::NoMemory);
- }
- }
- // If buddy is inside the requested region, keep it (don't add back)
- }
-
- // Verify we ended up with the correct block
- assert!(
- current_block.addr == base,
- "zone {}: Final block address {:#x} doesn't match requested {:#x}",
- self.zone_id,
- current_block.addr,
- base
- );
- assert!(
- current_block.order == required_order,
- "zone {}: Final block order {} doesn't match required {}",
- self.zone_id,
- current_block.order,
- required_order
- );
-
- return Ok(base);
- }
- }
-
- // No free block found that contains the requested address
- error!(
- "zone {}: No free block found for address {:#x} ({} pages)",
- self.zone_id, base, num_pages
- );
- Err(AllocError::NoMemory)
- }
-}
-
-impl Default for BuddySet {
- fn default() -> Self {
- Self::empty()
- }
-}
diff --git a/src/buddy/global_node_pool.rs b/src/buddy/global_node_pool.rs
deleted file mode 100644
index b921bd6..0000000
--- a/src/buddy/global_node_pool.rs
+++ /dev/null
@@ -1,316 +0,0 @@
-//! Global node pool for buddy allocator
-//!
-//! Provides a single pool of list nodes shared across all zones and orders.
-//! This eliminates the need for per-zone list pools and improves memory efficiency.
-
-use super::buddy_block::BuddyBlock;
-
-/// Simple linked list node used by the global node pool
-#[derive(Debug, Clone, Copy)]
-pub struct ListNode {
- pub data: T,
- pub next: Option,
-}
-
-fn align_up(addr: usize, align: usize) -> usize {
- debug_assert!(align.is_power_of_two());
- (addr + align - 1) & !(align - 1)
-}
-
-/// Global node pool - all zones and orders share nodes from this pool
-///
-/// Nodes are carved from one or more contiguous memory regions provided
-/// by the page allocator. The pool manages a single free list of nodes
-/// using raw pointers (stored as usize indices).
-pub struct GlobalNodePool {
- /// Free list head - points to first available node (node address)
- free_head: Option,
- /// Total number of nodes ever added to the pool
- total_nodes: usize,
- /// Current number of free nodes in the pool
- free_nodes: usize,
- /// Allocation statistics
- total_allocations: usize,
- total_deallocations: usize,
-}
-
-impl GlobalNodePool {
- /// Create a new global node pool (uninitialized, must call init())
- pub const fn new() -> Self {
- Self {
- free_head: None,
- total_nodes: 0,
- free_nodes: 0,
- total_allocations: 0,
- total_deallocations: 0,
- }
- }
-
- /// Initialize the global node pool with a backing memory region
- ///
- /// The region is treated as raw memory and partitioned into
- /// `ListNode` objects, which are all added to the
- /// pool's free list.
- pub fn init(&mut self, region_start: usize, region_size: usize) {
- self.free_head = None;
- self.total_nodes = 0;
- self.free_nodes = 0;
- self.total_allocations = 0;
- self.total_deallocations = 0;
- self.add_region(region_start, region_size);
- }
-
- /// Add an additional memory region to the node pool
- ///
- /// This can be used to grow the pool at runtime by carving
- /// more memory from the page allocator.
- pub fn add_region(&mut self, region_start: usize, region_size: usize) {
- let node_size = core::mem::size_of::>();
- let align = core::mem::align_of::>();
- if region_size < node_size {
- return;
- }
-
- let mut current = align_up(region_start, align);
- let end = region_start + region_size;
-
- while current + node_size <= end {
- unsafe {
- let node_ptr = current as *mut ListNode;
- core::ptr::write(
- node_ptr,
- ListNode {
- data: core::mem::zeroed(),
- next: self.free_head,
- },
- );
- }
-
- self.free_head = Some(current);
- self.total_nodes += 1;
- self.free_nodes += 1;
- current += node_size;
- }
- }
-
- /// Allocate a node from the pool
- ///
- /// Returns the index of the allocated node, or None if pool is exhausted
- pub fn alloc_node(&mut self) -> Option {
- let node_addr = self.free_head?;
-
- unsafe {
- let node = &mut *(node_addr as *mut ListNode);
- self.free_head = node.next;
- node.next = None;
- }
-
- self.total_allocations += 1;
- if self.free_nodes > 0 {
- self.free_nodes -= 1;
- } else {
- panic!("free_nodes: {}", self.free_nodes);
- }
-
- Some(node_addr)
- }
-
- /// Deallocate a node back to the pool
- ///
- /// The node should not be part of any active list when freed
- pub fn dealloc_node(&mut self, node_idx: usize) {
- unsafe {
- let node_ptr = node_idx as *mut ListNode;
- core::ptr::write(
- node_ptr,
- ListNode {
- data: core::mem::zeroed(),
- next: self.free_head,
- },
- );
- }
-
- self.free_head = Some(node_idx);
- self.total_deallocations += 1;
- self.free_nodes += 1;
- }
-
- /// Get a reference to a node by index
- pub fn get_node(&self, node_idx: usize) -> Option<&ListNode> {
- Some(unsafe { &*(node_idx as *const ListNode) })
- }
-
- /// Get a mutable reference to a node by index
- pub fn get_node_mut(&mut self, node_idx: usize) -> Option<&mut ListNode> {
- Some(unsafe { &mut *(node_idx as *mut ListNode) })
- }
-
- /// Get the number of free nodes in the pool
- pub fn free_node_count(&self) -> usize {
- self.free_nodes
- }
-
- /// Get the number of allocated nodes
- pub fn allocated_node_count(&self) -> usize {
- self.total_nodes.saturating_sub(self.free_nodes)
- }
-
- /// Get pool statistics
- pub fn get_stats(&self) -> GlobalPoolStats {
- GlobalPoolStats {
- total_nodes: self.total_nodes,
- free_nodes: self.free_nodes,
- allocated_nodes: self.allocated_node_count(),
- total_allocations: self.total_allocations,
- total_deallocations: self.total_deallocations,
- }
- }
-}
-
-impl Default for GlobalNodePool {
- fn default() -> Self {
- Self::new()
- }
-}
-
-/// Global pool statistics
-#[derive(Debug, Default, Clone)]
-pub struct GlobalPoolStats {
- pub total_nodes: usize,
- pub free_nodes: usize,
- pub allocated_nodes: usize,
- pub total_allocations: usize,
- pub total_deallocations: usize,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- const TEST_NODE_COUNT: usize = 512;
- #[test]
- fn test_pool_init() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT);
- assert_eq!(pool.allocated_node_count(), 0);
- }
-
- #[test]
- fn test_alloc_dealloc() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let idx1 = pool.alloc_node().unwrap();
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1);
- assert_eq!(pool.allocated_node_count(), 1);
-
- let idx2 = pool.alloc_node().unwrap();
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 2);
-
- pool.dealloc_node(idx1);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1);
-
- pool.dealloc_node(idx2);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT);
- }
-
- #[test]
- fn test_pool_exhaustion() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- // Allocate all nodes
- let mut indices = alloc::vec::Vec::new();
- for _ in 0..TEST_NODE_COUNT {
- indices.push(pool.alloc_node().unwrap());
- }
-
- assert_eq!(pool.free_node_count(), 0);
- assert!(pool.alloc_node().is_none());
-
- // Free one and allocate again
- pool.dealloc_node(indices[0]);
- assert_eq!(pool.free_node_count(), 1);
- assert!(pool.alloc_node().is_some());
- }
-
- #[test]
- fn test_node_access() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let idx = pool.alloc_node().unwrap();
-
- // Get mutable reference and set data
- if let Some(node) = pool.get_node_mut(idx) {
- node.data = BuddyBlock {
- order: 0,
- addr: 0x1000,
- };
- }
-
- // Get reference and read data
- if let Some(node) = pool.get_node(idx) {
- assert_eq!(node.data.order, 0);
- assert_eq!(node.data.addr, 0x1000);
- }
- }
-
- #[test]
- fn test_stats() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let idx1 = pool.alloc_node().unwrap();
- let _idx2 = pool.alloc_node().unwrap();
-
- let stats = pool.get_stats();
- assert_eq!(stats.total_nodes, TEST_NODE_COUNT);
- assert_eq!(stats.free_nodes, TEST_NODE_COUNT - 2);
- assert_eq!(stats.allocated_nodes, 2);
- assert_eq!(stats.total_allocations, 2);
- assert_eq!(stats.total_deallocations, 0);
-
- pool.dealloc_node(idx1);
- let stats2 = pool.get_stats();
- assert_eq!(stats2.free_nodes, TEST_NODE_COUNT - 1);
- assert_eq!(stats2.total_deallocations, 1);
- }
-}
diff --git a/src/buddy/mod.rs b/src/buddy/mod.rs
index 67be2c4..e760969 100644
--- a/src/buddy/mod.rs
+++ b/src/buddy/mod.rs
@@ -1,23 +1,729 @@
-//! Buddy page allocator module
+//! Buddy page allocator — page-metadata-based with intrusive free lists.
//!
-//! This module provides a complete buddy system implementation with:
-//! - Sorted linked lists for efficient contiguity checking
-//! - Multi-zone support
-//! - Detailed statistics and debugging
-
-pub mod buddy_allocator;
-pub mod buddy_block;
-pub mod buddy_set;
-pub mod global_node_pool;
-pub mod pooled_list;
-pub mod stats;
-
-pub use buddy_allocator::BuddyPageAllocator;
-pub use buddy_block::{BuddyBlock, ZoneInfo, MAX_ZONES};
-pub use buddy_set::BuddySet;
-pub use global_node_pool::{GlobalNodePool, ListNode};
-pub use pooled_list::PooledLinkedList;
-#[cfg(not(feature = "tracking"))]
-pub use stats::DEFAULT_MAX_ORDER;
-#[cfg(feature = "tracking")]
-pub use stats::{BuddyStats, DEFAULT_MAX_ORDER};
+//! The allocator manages one or more contiguous virtual address ranges ("sections").
+//! Each section stores its own [`BuddySection`] descriptor and [`PageMeta`] array
+//! in the caller-provided region prefix, enabling O(1) free-list operations
+//! without any dynamic allocation.
+
+pub mod page_meta;
+
+pub use page_meta::{PFN_NONE, PageFlags, PageMeta};
+
+use core::ptr;
+
+use crate::error::{AllocError, AllocResult};
+use crate::{OsImpl, align_up, is_aligned};
+use page_meta::{free_list_pop, free_list_push, free_list_remove};
+
+/// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks.
+pub const MAX_ORDER: usize = 20;
+
+/// DMA32 zone upper bound (4 GiB physical).
+const DMA32_LIMIT: usize = 0x1_0000_0000;
+
+struct RegionLayout {
+ section_start: usize,
+ meta_start: usize,
+ managed_heap_start: usize,
+ managed_heap_size: usize,
+}
+
+pub(crate) struct SectionInitSpec {
+ pub(crate) region_start: usize,
+ pub(crate) region_size: usize,
+ pub(crate) section_ptr: *mut BuddySection,
+ pub(crate) meta_ptr: *mut u8,
+ pub(crate) meta_size: usize,
+ pub(crate) heap_start: usize,
+ pub(crate) heap_size: usize,
+}
+
+/// Public read-only summary of a managed section.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ManagedSection {
+ pub start: usize,
+ pub size: usize,
+ pub free_pages: usize,
+ pub total_pages: usize,
+}
+
+/// Per-region buddy state stored in the region prefix.
+#[repr(C)]
+pub(crate) struct BuddySection {
+ pub(crate) next: *mut BuddySection,
+ pub(crate) region_start: usize,
+ pub(crate) region_size: usize,
+ pub(crate) meta: *mut PageMeta,
+ pub(crate) max_pages: usize,
+ pub(crate) heap_start: usize,
+ pub(crate) heap_size: usize,
+ pub(crate) free_lists: [u32; MAX_ORDER + 1],
+ pub(crate) free_pages: usize,
+ pub(crate) total_pages: usize,
+}
+
+impl BuddySection {
+ const fn metadata_align() -> usize {
+ let section_align = core::mem::align_of::();
+ let meta_align = core::mem::align_of::();
+ if section_align > meta_align {
+ section_align
+ } else {
+ meta_align
+ }
+ }
+
+ fn metadata_layout_for_pages(pages: usize) -> Option<(usize, usize)> {
+ let meta_offset = align_up(
+ core::mem::size_of::(),
+ core::mem::align_of::(),
+ );
+ let page_meta_size = pages.checked_mul(core::mem::size_of::())?;
+ let meta_size = meta_offset.checked_add(page_meta_size)?;
+ Some((meta_offset, meta_size))
+ }
+
+ fn available_heap_pages(
+ region_end: usize,
+ section_start: usize,
+ meta_size: usize,
+ ) -> Option {
+ let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE);
+ if managed_heap_start > region_end {
+ return Some(0);
+ }
+ Some((region_end - managed_heap_start) / PAGE_SIZE)
+ }
+
+ fn can_manage_pages(
+ region_end: usize,
+ section_start: usize,
+ pages: usize,
+ ) -> bool {
+ let Some((_, meta_size)) = Self::metadata_layout_for_pages(pages) else {
+ return false;
+ };
+ let Some(available_pages) =
+ Self::available_heap_pages::(region_end, section_start, meta_size)
+ else {
+ return false;
+ };
+ available_pages >= pages
+ }
+
+ fn compute_region_layout(
+ region_start: usize,
+ region_size: usize,
+ ) -> Option {
+ if region_size == 0 || !PAGE_SIZE.is_power_of_two() {
+ return None;
+ }
+
+ let region_end = region_start.checked_add(region_size)?;
+ let section_start = align_up(region_start, Self::metadata_align());
+ if section_start >= region_end {
+ return None;
+ }
+
+ let heap_search_start = align_up(
+ section_start.checked_add(core::mem::size_of::())?,
+ PAGE_SIZE,
+ );
+ let max_pages = if heap_search_start >= region_end {
+ 0
+ } else {
+ (region_end - heap_search_start) / PAGE_SIZE
+ };
+
+ let mut low = 0usize;
+ let mut high = max_pages;
+ while low < high {
+ let mid = low + (high - low).div_ceil(2);
+ if Self::can_manage_pages::(region_end, section_start, mid) {
+ low = mid;
+ } else {
+ high = mid - 1;
+ }
+ }
+
+ if low == 0 {
+ return None;
+ }
+
+ let (meta_offset, meta_size) = Self::metadata_layout_for_pages(low)?;
+ let meta_start = section_start.checked_add(meta_offset)?;
+ let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE);
+ let managed_heap_size = low.checked_mul(PAGE_SIZE)?;
+
+ Some(RegionLayout {
+ section_start,
+ meta_start,
+ managed_heap_start,
+ managed_heap_size,
+ })
+ }
+
+ unsafe fn init_at(
+ section_ptr: *mut BuddySection,
+ region_start: usize,
+ region_size: usize,
+ meta_ptr: *mut u8,
+ meta_size: usize,
+ heap_start: usize,
+ heap_size: usize,
+ ) -> AllocResult {
+ unsafe {
+ if !PAGE_SIZE.is_power_of_two() {
+ return Err(AllocError::InvalidParam);
+ }
+ if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let total_pages = heap_size / PAGE_SIZE;
+ let required = BuddyAllocator::::required_meta_size(heap_size);
+ if meta_size < required {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let meta = meta_ptr as *mut PageMeta;
+ for i in 0..total_pages {
+ meta.add(i).write(PageMeta::new());
+ }
+
+ section_ptr.write(BuddySection {
+ next: ptr::null_mut(),
+ region_start,
+ region_size,
+ meta,
+ max_pages: total_pages,
+ heap_start,
+ heap_size,
+ free_lists: [PFN_NONE; MAX_ORDER + 1],
+ free_pages: 0,
+ total_pages,
+ });
+
+ let section = &mut *section_ptr;
+ let mut pfn: usize = 0;
+ while pfn < total_pages {
+ let mut order = MAX_ORDER;
+ loop {
+ let block_pages = 1usize << order;
+ if block_pages <= total_pages - pfn && (pfn & (block_pages - 1)) == 0 {
+ break;
+ }
+ if order == 0 {
+ break;
+ }
+ order -= 1;
+ }
+ let block_pages = 1usize << order;
+ let m = &mut *section.meta.add(pfn);
+ m.flags = PageFlags::Free;
+ m.order = order as u8;
+ free_list_push(section.meta, &mut section.free_lists, pfn as u32, order);
+ section.free_pages += block_pages;
+ pfn += block_pages;
+ }
+
+ Ok(())
+ }
+ }
+
+ #[inline]
+ fn contains_heap_addr(&self, addr: usize) -> bool {
+ addr >= self.heap_start && addr < self.heap_start + self.heap_size
+ }
+
+ #[inline]
+ fn summary(&self) -> ManagedSection {
+ ManagedSection {
+ start: self.heap_start,
+ size: self.heap_size,
+ free_pages: self.free_pages,
+ total_pages: self.total_pages,
+ }
+ }
+}
+
+/// Page-metadata-based buddy allocator.
+///
+/// `PAGE_SIZE` must be a power of two (commonly 0x1000 = 4 KiB).
+pub struct BuddyAllocator {
+ sections_head: *mut BuddySection,
+ sections_tail: *mut BuddySection,
+ section_count: usize,
+ os: Option<&'static dyn OsImpl>,
+}
+
+// SAFETY: The allocator is designed to be wrapped in a SpinMutex.
+// All section pointers point into caller-provided regions whose lifetime is managed externally.
+unsafe impl Send for BuddyAllocator {}
+
+impl BuddyAllocator {
+ /// Calculate the metadata-region size (in bytes) required for `heap_size` bytes.
+ pub const fn required_meta_size(heap_size: usize) -> usize {
+ let pages = heap_size / PAGE_SIZE;
+ pages * core::mem::size_of::()
+ }
+
+ /// Create an uninitialised allocator. Call [`init`](Self::init) before use.
+ pub const fn new() -> Self {
+ Self {
+ sections_head: ptr::null_mut(),
+ sections_tail: ptr::null_mut(),
+ section_count: 0,
+ os: None,
+ }
+ }
+}
+
+impl Default for BuddyAllocator {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl BuddyAllocator {
+ pub(crate) fn reset(&mut self, os: Option<&'static dyn OsImpl>) {
+ self.sections_head = ptr::null_mut();
+ self.sections_tail = ptr::null_mut();
+ self.section_count = 0;
+ self.os = os;
+ }
+
+ /// Initialise the allocator over the first section.
+ ///
+ /// # Safety
+ /// - `region` must be writable and remain valid for the lifetime of this allocator.
+ /// - Bytes consumed by metadata become unavailable for allocation.
+ pub unsafe fn init(
+ &mut self,
+ region: &mut [u8],
+ os: Option<&'static dyn OsImpl>,
+ ) -> AllocResult {
+ unsafe {
+ self.reset(os);
+ self.add_region(region)
+ }
+ }
+
+ /// Add a new managed region after initialisation.
+ ///
+ /// # Safety
+ /// - `region` must be writable and remain valid for the lifetime of this allocator.
+ /// - The region must not overlap any existing managed region.
+ pub unsafe fn add_region(&mut self, region: &mut [u8]) -> AllocResult {
+ unsafe {
+ let region_start = region.as_mut_ptr() as usize;
+ let region_size = region.len();
+ let layout =
+ BuddySection::compute_region_layout::(region_start, region_size)
+ .ok_or(AllocError::InvalidParam)?;
+ self.add_region_raw(SectionInitSpec {
+ region_start,
+ region_size,
+ section_ptr: layout.section_start as *mut BuddySection,
+ meta_ptr: layout.meta_start as *mut u8,
+ meta_size: Self::required_meta_size(layout.managed_heap_size),
+ heap_start: layout.managed_heap_start,
+ heap_size: layout.managed_heap_size,
+ })
+ }
+ }
+
+ pub(crate) unsafe fn add_region_raw(&mut self, spec: SectionInitSpec) -> AllocResult {
+ unsafe {
+ let region_size = spec.region_size;
+ let region_end = spec
+ .region_start
+ .checked_add(region_size)
+ .ok_or(AllocError::InvalidParam)?;
+ let heap_end = spec
+ .heap_start
+ .checked_add(spec.heap_size)
+ .ok_or(AllocError::InvalidParam)?;
+ if heap_end > region_end {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let mut section = self.sections_head;
+ while !section.is_null() {
+ let existing = &*section;
+ let existing_end = existing
+ .region_start
+ .checked_add(existing.region_size)
+ .ok_or(AllocError::InvalidParam)?;
+ if spec.region_start < existing_end && existing.region_start < region_end {
+ return Err(AllocError::MemoryOverlap);
+ }
+ section = existing.next;
+ }
+
+ BuddySection::init_at::(
+ spec.section_ptr,
+ spec.region_start,
+ spec.region_size,
+ spec.meta_ptr,
+ spec.meta_size,
+ spec.heap_start,
+ spec.heap_size,
+ )?;
+
+ if self.sections_head.is_null() {
+ self.sections_head = spec.section_ptr;
+ } else {
+ (*self.sections_tail).next = spec.section_ptr;
+ }
+ self.sections_tail = spec.section_ptr;
+ self.section_count += 1;
+
+ log::debug!(
+ "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages",
+ spec.region_start,
+ spec.region_size,
+ spec.heap_start,
+ heap_end,
+ spec.heap_size / PAGE_SIZE,
+ );
+
+ Ok(())
+ }
+ }
+
+ /// Number of managed sections.
+ pub fn section_count(&self) -> usize {
+ self.section_count
+ }
+
+ /// Read-only summary for a managed section by registration order.
+ pub fn section(&self, index: usize) -> Option {
+ let mut current = self.sections_head;
+ let mut i = 0usize;
+ while !current.is_null() {
+ if i == index {
+ return Some(unsafe { (&*current).summary() });
+ }
+ current = unsafe { (*current).next };
+ i += 1;
+ }
+ None
+ }
+
+ /// Total number of pages managed across all sections.
+ pub fn total_pages(&self) -> usize {
+ let mut total = 0usize;
+ let mut current = self.sections_head;
+ while !current.is_null() {
+ total += unsafe { (*current).total_pages };
+ current = unsafe { (*current).next };
+ }
+ total
+ }
+
+ /// Total managed heap bytes across all sections.
+ ///
+ /// This counts only bytes in allocatable heaps, excluding region-prefix metadata.
+ pub fn managed_bytes(&self) -> usize {
+ let mut total = 0usize;
+ let mut current = self.sections_head;
+ while !current.is_null() {
+ total += unsafe { (*current).heap_size };
+ current = unsafe { (*current).next };
+ }
+ total
+ }
+
+ /// Number of currently free pages across all sections.
+ pub fn free_pages(&self) -> usize {
+ let mut total = 0usize;
+ let mut current = self.sections_head;
+ while !current.is_null() {
+ total += unsafe { (*current).free_pages };
+ current = unsafe { (*current).next };
+ }
+ total
+ }
+
+ /// Allocated backend bytes across all sections.
+ ///
+ /// This is computed as managed heap bytes minus currently free page bytes.
+ /// It reflects page-level occupancy, so it includes slab pages, alignment
+ /// amplification, and internal fragmentation.
+ pub fn allocated_bytes(&self) -> usize {
+ self.managed_bytes()
+ .saturating_sub(self.free_pages().saturating_mul(PAGE_SIZE))
+ }
+
+ /// Allocate `count` contiguous pages, returning the virtual address.
+ pub fn alloc_pages(&mut self, count: usize, align: usize) -> AllocResult {
+ if count == 0 {
+ return Err(AllocError::InvalidParam);
+ }
+ let align = if align == 0 { PAGE_SIZE } else { align };
+ if !align.is_power_of_two() || align < PAGE_SIZE {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let order = count.next_power_of_two().trailing_zeros() as usize;
+ if order > MAX_ORDER {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let align_order = (align / PAGE_SIZE).trailing_zeros() as usize;
+ let effective_order = order.max(align_order);
+
+ let mut section = self.sections_head;
+ while !section.is_null() {
+ if let Ok(addr) = unsafe { Self::alloc_from_section(&mut *section, effective_order) } {
+ return Ok(addr);
+ }
+ section = unsafe { (*section).next };
+ }
+
+ Err(AllocError::NoMemory)
+ }
+
+ fn alloc_from_section(section: &mut BuddySection, order: usize) -> AllocResult {
+ let mut found_order = order;
+ while found_order <= MAX_ORDER {
+ if section.free_lists[found_order] != PFN_NONE {
+ break;
+ }
+ found_order += 1;
+ }
+ if found_order > MAX_ORDER {
+ return Err(AllocError::NoMemory);
+ }
+
+ let pfn = unsafe { free_list_pop(section.meta, &mut section.free_lists, found_order) };
+ debug_assert_ne!(pfn, PFN_NONE);
+
+ let mut current_order = found_order;
+ while current_order > order {
+ current_order -= 1;
+ let buddy_pfn = pfn as usize + (1 << current_order);
+ unsafe {
+ let bm = &mut *section.meta.add(buddy_pfn);
+ bm.flags = PageFlags::Free;
+ bm.order = current_order as u8;
+ free_list_push(
+ section.meta,
+ &mut section.free_lists,
+ buddy_pfn as u32,
+ current_order,
+ );
+ }
+ }
+
+ unsafe {
+ let m = &mut *section.meta.add(pfn as usize);
+ m.flags = PageFlags::Allocated;
+ m.order = order as u8;
+ }
+
+ section.free_pages -= 1 << order;
+ Ok(section.heap_start + (pfn as usize) * PAGE_SIZE)
+ }
+
+ /// Allocate pages whose *physical* address is below 4 GiB (DMA32 zone).
+ pub fn alloc_pages_lowmem(&mut self, count: usize, align: usize) -> AllocResult {
+ let os = self.os.ok_or(AllocError::InvalidParam)?;
+
+ if count == 0 {
+ return Err(AllocError::InvalidParam);
+ }
+ let align = if align == 0 { PAGE_SIZE } else { align };
+ if !align.is_power_of_two() || align < PAGE_SIZE {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let order = count.next_power_of_two().trailing_zeros() as usize;
+ let align_order = (align / PAGE_SIZE).trailing_zeros() as usize;
+ let effective_order = order.max(align_order);
+ if effective_order > MAX_ORDER {
+ return Err(AllocError::InvalidParam);
+ }
+
+ let mut section = self.sections_head;
+ while !section.is_null() {
+ if let Ok(addr) =
+ unsafe { Self::alloc_lowmem_from_section(&mut *section, effective_order, os) }
+ {
+ return Ok(addr);
+ }
+ section = unsafe { (*section).next };
+ }
+
+ Err(AllocError::NoMemory)
+ }
+
+ fn alloc_lowmem_from_section(
+ section: &mut BuddySection,
+ effective_order: usize,
+ os: &'static dyn OsImpl,
+ ) -> AllocResult {
+ for search_order in effective_order..=MAX_ORDER {
+ let mut pfn_u32 = section.free_lists[search_order];
+ while pfn_u32 != PFN_NONE {
+ let addr = section.heap_start + (pfn_u32 as usize) * PAGE_SIZE;
+ let phys = os.virt_to_phys(addr);
+ let block_bytes = (1usize << search_order) * PAGE_SIZE;
+ if phys + block_bytes <= DMA32_LIMIT {
+ unsafe {
+ free_list_remove(
+ section.meta,
+ &mut section.free_lists,
+ pfn_u32,
+ search_order,
+ );
+ }
+
+ let mut current_order = search_order;
+ while current_order > effective_order {
+ current_order -= 1;
+ let buddy_pfn = pfn_u32 as usize + (1 << current_order);
+ unsafe {
+ let bm = &mut *section.meta.add(buddy_pfn);
+ bm.flags = PageFlags::Free;
+ bm.order = current_order as u8;
+ free_list_push(
+ section.meta,
+ &mut section.free_lists,
+ buddy_pfn as u32,
+ current_order,
+ );
+ }
+ }
+
+ unsafe {
+ let m = &mut *section.meta.add(pfn_u32 as usize);
+ m.flags = PageFlags::Allocated;
+ m.order = effective_order as u8;
+ }
+ section.free_pages -= 1 << effective_order;
+ return Ok(addr);
+ }
+ pfn_u32 = unsafe { (*section.meta.add(pfn_u32 as usize)).next };
+ }
+ }
+
+ Err(AllocError::NoMemory)
+ }
+
+ /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages).
+ ///
+ /// `addr` must be the exact address returned by alloc. The allocator frees
+ /// the full block size recorded in page metadata, which may be larger than
+ /// `count` if the original allocation was rounded up for buddy order or alignment.
+ pub fn dealloc_pages(&mut self, addr: usize, count: usize) {
+ let Some(section) = self.find_section_by_addr_mut(addr) else {
+ debug_assert!(
+ false,
+ "dealloc_pages called with address outside all sections"
+ );
+ return;
+ };
+
+ debug_assert!(is_aligned(addr, PAGE_SIZE));
+ debug_assert!(count > 0);
+
+ let pfn = (addr - section.heap_start) / PAGE_SIZE;
+ debug_assert!(pfn < section.max_pages);
+ let stored = unsafe { &*section.meta.add(pfn) };
+ debug_assert!(
+ stored.flags == PageFlags::Allocated || stored.flags == PageFlags::Slab,
+ "dealloc_pages called on non-allocated block"
+ );
+
+ let expected_order = count.next_power_of_two().trailing_zeros() as usize;
+ let order = stored.order as usize;
+ debug_assert!(
+ expected_order <= order,
+ "dealloc_pages count implies larger order than the allocated block"
+ );
+ Self::dealloc_in_section(section, pfn, order);
+ }
+
+ /// Mark the page at `addr` with the given flags (used by slab to tag pages).
+ ///
+ /// # Safety
+ /// The caller must ensure `addr` is valid and properly allocated.
+ pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) -> AllocResult {
+ unsafe {
+ let section = self
+ .find_section_by_addr_mut(addr)
+ .ok_or(AllocError::NotFound)?;
+ let pfn = (addr - section.heap_start) / PAGE_SIZE;
+ (*section.meta.add(pfn)).flags = flags;
+ Ok(())
+ }
+ }
+
+ /// Read the flags of the page containing `addr`.
+ pub fn page_flags(&self, addr: usize) -> AllocResult {
+ let section = self
+ .find_section_by_addr(addr)
+ .ok_or(AllocError::NotFound)?;
+ let pfn = (addr - section.heap_start) / PAGE_SIZE;
+ Ok(unsafe { (*section.meta.add(pfn)).flags })
+ }
+
+ fn dealloc_in_section(section: &mut BuddySection, mut pfn: usize, mut order: usize) {
+ let freed_pages = 1usize << order;
+
+ while order < MAX_ORDER {
+ let buddy_pfn = pfn ^ (1 << order);
+ if buddy_pfn >= section.max_pages {
+ break;
+ }
+ let buddy = unsafe { &*section.meta.add(buddy_pfn) };
+ if buddy.flags != PageFlags::Free || buddy.order as usize != order {
+ break;
+ }
+ unsafe {
+ free_list_remove(
+ section.meta,
+ &mut section.free_lists,
+ buddy_pfn as u32,
+ order,
+ );
+ }
+ pfn = pfn.min(buddy_pfn);
+ order += 1;
+ }
+
+ unsafe {
+ let m = &mut *section.meta.add(pfn);
+ m.flags = PageFlags::Free;
+ m.order = order as u8;
+ free_list_push(section.meta, &mut section.free_lists, pfn as u32, order);
+ }
+ section.free_pages += freed_pages;
+ }
+
+ fn find_section_by_addr(&self, addr: usize) -> Option<&BuddySection> {
+ let mut section = self.sections_head;
+ while !section.is_null() {
+ let current = unsafe { &*section };
+ if current.contains_heap_addr(addr) {
+ return Some(current);
+ }
+ section = current.next;
+ }
+ None
+ }
+
+ fn find_section_by_addr_mut(&mut self, addr: usize) -> Option<&mut BuddySection> {
+ let mut section = self.sections_head;
+ while !section.is_null() {
+ let current = unsafe { &mut *section };
+ if current.contains_heap_addr(addr) {
+ return Some(current);
+ }
+ section = current.next;
+ }
+ None
+ }
+}
diff --git a/src/buddy/page_meta.rs b/src/buddy/page_meta.rs
new file mode 100644
index 0000000..8f49110
--- /dev/null
+++ b/src/buddy/page_meta.rs
@@ -0,0 +1,137 @@
+//! Per-page metadata stored in the external metadata region.
+//!
+//! Each page frame in the heap has a corresponding [`PageMeta`] entry.
+//! Free pages are linked together via intrusive doubly-linked lists using PFN indices.
+
+/// Sentinel value indicating "no page" in free-list links.
+pub const PFN_NONE: u32 = u32::MAX;
+
+/// Page state flags.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum PageFlags {
+ /// Page is free and sits in a buddy free list.
+ Free = 0,
+ /// Page is allocated (head page of a buddy block).
+ Allocated = 1,
+ /// Page is used as a slab page.
+ Slab = 2,
+}
+
+/// Metadata for a single page frame (12 bytes).
+///
+/// Head pages carry the `order` of the entire block.
+/// Tail pages within a buddy block are marked `Allocated` (or `Slab`) with order 0.
+#[derive(Debug, Clone, Copy)]
+#[repr(C)]
+pub struct PageMeta {
+ /// Current state of this page.
+ pub flags: PageFlags,
+ /// Order of the block (only meaningful on head pages).
+ pub order: u8,
+ /// Reserved padding.
+ pub _pad: u16,
+ /// Previous PFN in the same-order free list (`PFN_NONE` if head or not free).
+ pub prev: u32,
+ /// Next PFN in the same-order free list (`PFN_NONE` if tail or not free).
+ pub next: u32,
+}
+
+const _: () = assert!(core::mem::size_of::() == 12);
+
+impl Default for PageMeta {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl PageMeta {
+ /// Create a zeroed (free, order-0) page meta.
+ pub const fn new() -> Self {
+ Self {
+ flags: PageFlags::Free,
+ order: 0,
+ _pad: 0,
+ prev: PFN_NONE,
+ next: PFN_NONE,
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Free-list helpers operating on a `*mut PageMeta` array + head array
+// ---------------------------------------------------------------------------
+
+/// Push `pfn` onto the front of `free_lists[order]`.
+///
+/// # Safety
+/// `meta` must point to an array with at least `pfn + 1` entries.
+/// `pfn` must not already be in any free list.
+#[inline]
+pub unsafe fn free_list_push(meta: *mut PageMeta, free_lists: &mut [u32], pfn: u32, order: usize) {
+ unsafe {
+ let old_head = free_lists[order];
+ let m = &mut *meta.add(pfn as usize);
+ m.prev = PFN_NONE;
+ m.next = old_head;
+ if old_head != PFN_NONE {
+ (*meta.add(old_head as usize)).prev = pfn;
+ }
+ free_lists[order] = pfn;
+ }
+}
+
+/// Pop the first PFN from `free_lists[order]`, returning `PFN_NONE` if empty.
+///
+/// # Safety
+/// `meta` must be a valid metadata array.
+#[inline]
+pub unsafe fn free_list_pop(meta: *mut PageMeta, free_lists: &mut [u32], order: usize) -> u32 {
+ unsafe {
+ let head = free_lists[order];
+ if head == PFN_NONE {
+ return PFN_NONE;
+ }
+ let m = &mut *meta.add(head as usize);
+ let next = m.next;
+ m.prev = PFN_NONE;
+ m.next = PFN_NONE;
+ if next != PFN_NONE {
+ (*meta.add(next as usize)).prev = PFN_NONE;
+ }
+ free_lists[order] = next;
+ head
+ }
+}
+
+/// Remove `pfn` from the free list at `order`.
+///
+/// # Safety
+/// `pfn` must currently be in `free_lists[order]`.
+#[inline]
+pub unsafe fn free_list_remove(
+ meta: *mut PageMeta,
+ free_lists: &mut [u32],
+ pfn: u32,
+ order: usize,
+) {
+ unsafe {
+ let m = &*meta.add(pfn as usize);
+ let prev = m.prev;
+ let next = m.next;
+
+ if prev != PFN_NONE {
+ (*meta.add(prev as usize)).next = next;
+ } else {
+ // pfn was the head
+ free_lists[order] = next;
+ }
+ if next != PFN_NONE {
+ (*meta.add(next as usize)).prev = prev;
+ }
+
+ let m = &mut *meta.add(pfn as usize);
+ m.prev = PFN_NONE;
+ m.next = PFN_NONE;
+ }
+}
diff --git a/src/buddy/pooled_list.rs b/src/buddy/pooled_list.rs
deleted file mode 100644
index 546f4ad..0000000
--- a/src/buddy/pooled_list.rs
+++ /dev/null
@@ -1,523 +0,0 @@
-//! Pooled linked list implementation using global node pool
-//!
-//! Provides linked lists that draw nodes from a shared global pool,
-//! allowing efficient use of memory across all zones and orders.
-
-#[cfg(feature = "log")]
-use log::{error, warn};
-
-use super::{buddy_block::BuddyBlock, global_node_pool::GlobalNodePool};
-
-/// Pooled linked list - uses nodes from global pool
-///
-/// This maintains only the list structure (head/tail/len), while
-/// all nodes are allocated from the global pool.
-pub struct PooledLinkedList {
- head: Option,
- tail: Option,
- len: usize,
-}
-
-impl PooledLinkedList {
- /// Create a new empty pooled linked list
- pub const fn new() -> Self {
- Self {
- head: None,
- tail: None,
- len: 0,
- }
- }
-
- /// Insert element in sorted order (ascending by address)
- /// This is used for buddy free lists to enable efficient contiguity checking
- pub fn insert_sorted(&mut self, pool: &mut GlobalNodePool, data: BuddyBlock) -> bool {
- let new_node_idx = match pool.alloc_node() {
- Some(idx) => idx,
- None => {
- error!("Global node pool exhausted");
- return false;
- }
- };
-
- // Find insertion position
- let mut prev_idx = None;
- let mut current_idx = self.head;
- let mut visited = 0;
-
- while let Some(idx) = current_idx {
- if visited > self.len {
- error!("Potential cycle detected during insert");
- // Deallocate the node
- pool.dealloc_node(new_node_idx);
- return false;
- }
-
- if let Some(node) = pool.get_node(idx) {
- if node.data.addr == data.addr {
- // Block already in free list - this is a success (no-op)
- // The caller tried to free something that's already free
- pool.dealloc_node(new_node_idx);
- return true;
- }
- if node.data.addr > data.addr {
- break; // Found position
- }
- prev_idx = current_idx;
- current_idx = node.next;
- } else {
- error!("Invalid node reference in list");
- pool.dealloc_node(new_node_idx);
- return false;
- }
- visited += 1;
- }
-
- // Initialize the new node
- if let Some(node) = pool.get_node_mut(new_node_idx) {
- node.data = data;
- node.next = current_idx;
- } else {
- error!("Failed to get mutable reference to newly allocated node");
- pool.dealloc_node(new_node_idx);
- return false;
- }
-
- // Update links
- if let Some(prev) = prev_idx {
- if let Some(prev_node) = pool.get_node_mut(prev) {
- prev_node.next = Some(new_node_idx);
- }
- } else {
- self.head = Some(new_node_idx);
- }
-
- // Update tail if needed
- if current_idx.is_none() {
- self.tail = Some(new_node_idx);
- }
-
- self.len += 1;
- true
- }
-
- /// Check if any block in the list falls within the given address range [start, end)
- pub fn has_block_in_range(&self, pool: &GlobalNodePool, start: usize, end: usize) -> bool {
- let mut current_idx = self.head;
- let mut visited = 0;
-
- while let Some(idx) = current_idx {
- if visited > self.len {
- break;
- }
- if let Some(node) = pool.get_node(idx) {
- // Early termination: list is sorted by address
- if node.data.addr >= end {
- break;
- }
- // Check if block starts within the range
- // For i < initial_order, any block in the range is a conflict.
- // Since blocks are aligned to their size, a block starting before 'start'
- // cannot overlap with [start, end) if its size is smaller than start's alignment.
- if node.data.addr >= start {
- return true;
- }
- current_idx = node.next;
- } else {
- break;
- }
- visited += 1;
- }
- false
- }
-
- /// Pop an element from the front of the list
- pub fn pop_front(&mut self, pool: &mut GlobalNodePool) -> Option {
- self.head?;
-
- let head_idx = self.head?;
-
- if let Some(head_node) = pool.get_node_mut(head_idx) {
- self.head = head_node.next;
- if self.head.is_none() {
- self.tail = None;
- }
-
- let data = head_node.data;
-
- // Return node to pool
- pool.dealloc_node(head_idx);
- self.len -= 1;
-
- Some(data)
- } else {
- error!("Head node {head_idx} is corrupted");
- None
- }
- }
-
- /// Check if the list is empty
- pub fn is_empty(&self) -> bool {
- self.len == 0
- }
-
- /// Get the length of the list
- pub fn len(&self) -> usize {
- self.len
- }
-
- /// Find a node by address (for buddy system)
- ///
- /// Returns (node_idx, prev_idx) where prev_idx is the node before it (or None if head)
- pub fn find_by_addr(
- &self,
- pool: &GlobalNodePool,
- addr: usize,
- ) -> Option<(usize, Option)> {
- let mut prev_idx = None;
- let mut current_idx = self.head;
- let mut visited = 0;
-
- while let Some(idx) = current_idx {
- if visited > self.len {
- error!("Potential cycle detected during search");
- return None;
- }
-
- if let Some(node) = pool.get_node(idx) {
- // Early termination: list is sorted by address
- if node.data.addr > addr {
- break;
- }
- if node.data.addr == addr {
- return Some((idx, prev_idx));
- }
- prev_idx = current_idx;
- current_idx = node.next;
- } else {
- break;
- }
- visited += 1;
- }
-
- None
- }
-
- /// Remove a node at the given index
- pub fn remove(&mut self, pool: &mut GlobalNodePool, node_idx: usize) -> bool {
- // Find the node
- let mut prev_idx = None;
- let mut current_idx = self.head;
- let mut visited = 0;
-
- while let Some(idx) = current_idx {
- if visited > self.len {
- warn!("Potential cycle detected during remove");
- return false;
- }
-
- if idx == node_idx {
- break;
- }
- prev_idx = current_idx;
- if let Some(node) = pool.get_node(idx) {
- current_idx = node.next;
- } else {
- break;
- }
- visited += 1;
- }
-
- if current_idx != Some(node_idx) {
- return false;
- }
-
- self.remove_with_prev_impl(pool, node_idx, prev_idx)
- }
-
- /// Remove a node using known prev_idx (O(1) operation)
- ///
- /// This is used when we already know the previous node index from find_by_addr(),
- /// avoiding a second traversal of the list.
- pub fn remove_with_prev(
- &mut self,
- pool: &mut GlobalNodePool,
- node_idx: usize,
- prev_idx: Option,
- ) -> bool {
- // Verify the node exists
- if pool.get_node(node_idx).is_none() {
- warn!("Invalid node index {node_idx} for remove_with_prev");
- return false;
- }
-
- // Verify prev_idx leads to node_idx if provided
- if let Some(prev) = prev_idx {
- if let Some(prev_node) = pool.get_node(prev) {
- if prev_node.next != Some(node_idx) {
- warn!("prev_idx {prev} does not point to node_idx {node_idx}");
- return false;
- }
- } else {
- warn!("Invalid prev_idx {prev}");
- return false;
- }
- } else if self.head != Some(node_idx) {
- // prev_idx is None means node should be head
- warn!("prev_idx is None but node_idx {node_idx} is not head");
- return false;
- }
-
- self.remove_with_prev_impl(pool, node_idx, prev_idx)
- }
-
- /// Internal implementation of remove with known prev_idx
- fn remove_with_prev_impl(
- &mut self,
- pool: &mut GlobalNodePool,
- node_idx: usize,
- prev_idx: Option,
- ) -> bool {
- // Get the node's next pointer before deallocating
- let next_idx = pool.get_node(node_idx).and_then(|n| n.next);
-
- // Update links
- if let Some(prev) = prev_idx {
- if let Some(prev_node) = pool.get_node_mut(prev) {
- prev_node.next = next_idx;
- }
- } else {
- self.head = next_idx;
- }
-
- // Update tail if needed
- if self.tail == Some(node_idx) {
- if next_idx.is_none() {
- self.tail = prev_idx;
- }
- } else if self.head.is_none() {
- self.tail = None;
- }
-
- // Return node to pool
- pool.dealloc_node(node_idx);
- self.len -= 1;
- true
- }
-
- /// Get iterator over elements
- pub fn iter<'a>(&'a self, pool: &'a GlobalNodePool) -> PooledListIter<'a> {
- PooledListIter {
- pool,
- current: self.head,
- }
- }
-
- /// Clear all nodes from the list
- ///
- /// Returns all nodes to the global pool
- pub fn clear(&mut self, pool: &mut GlobalNodePool) {
- while self.pop_front(pool).is_some() {
- // Just pop all elements
- }
- }
-}
-
-impl Default for PooledLinkedList {
- fn default() -> Self {
- Self::new()
- }
-}
-
-/// Iterator for PooledLinkedList
-pub struct PooledListIter<'a> {
- pool: &'a GlobalNodePool,
- current: Option,
-}
-
-impl<'a> Iterator for PooledListIter<'a> {
- type Item = &'a BuddyBlock;
-
- fn next(&mut self) -> Option {
- self.current.and_then(|idx| {
- if let Some(node) = self.pool.get_node(idx) {
- self.current = node.next;
- Some(&node.data)
- } else {
- self.current = None;
- None
- }
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::buddy::ListNode;
-
- const TEST_NODE_COUNT: usize = 512;
-
- #[test]
- fn test_pooled_list_basic() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let mut list: PooledLinkedList = PooledLinkedList::new();
-
- assert!(list.is_empty());
- assert_eq!(list.len(), 0);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT);
-
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x1000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x2000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x3000,
- },
- );
-
- assert_eq!(list.len(), 3);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 3);
-
- assert_eq!(
- list.pop_front(&mut pool),
- Some(BuddyBlock {
- order: 0,
- addr: 0x1000
- })
- );
- assert_eq!(
- list.pop_front(&mut pool),
- Some(BuddyBlock {
- order: 0,
- addr: 0x2000
- })
- );
- assert_eq!(list.len(), 1);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 1);
-
- // Clear remaining
- list.clear(&mut pool);
- assert!(list.is_empty());
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT);
- }
-
- #[test]
- fn test_insert_sorted() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let mut list: PooledLinkedList = PooledLinkedList::new();
-
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x5000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x3000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x7000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x1000,
- },
- );
-
- let items: alloc::vec::Vec<_> = list.iter(&pool).collect();
- assert_eq!(items.len(), 4);
- assert_eq!(items[0].addr, 0x1000);
- assert_eq!(items[1].addr, 0x3000);
- assert_eq!(items[2].addr, 0x5000);
- assert_eq!(items[3].addr, 0x7000);
- }
-
- #[test]
- fn test_find_and_remove() {
- let mut pool: GlobalNodePool = GlobalNodePool::new();
- let mut backing = [ListNode {
- data: BuddyBlock { order: 0, addr: 0 },
- next: None,
- }; TEST_NODE_COUNT];
-
- let region_start = backing.as_mut_ptr() as usize;
- let region_size = core::mem::size_of_val(&backing);
- pool.init(region_start, region_size);
-
- let mut list: PooledLinkedList = PooledLinkedList::new();
-
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x1000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x2000,
- },
- );
- list.insert_sorted(
- &mut pool,
- BuddyBlock {
- order: 0,
- addr: 0x3000,
- },
- );
-
- let (node_idx, _) = list.find_by_addr(&pool, 0x2000).unwrap();
- assert!(list.remove(&mut pool, node_idx));
-
- assert_eq!(list.len(), 2);
- assert_eq!(pool.free_node_count(), TEST_NODE_COUNT - 2);
-
- let items: alloc::vec::Vec<_> = list.iter(&pool).collect();
- assert_eq!(items[0].addr, 0x1000);
- assert_eq!(items[1].addr, 0x3000);
- }
-}
diff --git a/src/buddy/stats.rs b/src/buddy/stats.rs
deleted file mode 100644
index 1ffe982..0000000
--- a/src/buddy/stats.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-//! Statistics and debugging for buddy allocator
-//!
-//! Provides detailed statistics tracking and failure reporting.
-
-use super::buddy_block::ZoneInfo;
-
-/// Maximum order supported
-pub const DEFAULT_MAX_ORDER: usize = 28;
-
-/// Buddy system statistics
-#[derive(Debug, Clone, Copy)]
-pub struct BuddyStats {
- pub total_pages: usize,
- pub free_pages: usize,
- pub used_pages: usize,
- pub free_pages_by_order: [usize; DEFAULT_MAX_ORDER + 1],
-}
-
-impl Default for BuddyStats {
- fn default() -> Self {
- Self {
- total_pages: 0,
- free_pages: 0,
- used_pages: 0,
- free_pages_by_order: [0; DEFAULT_MAX_ORDER + 1],
- }
- }
-}
-
-impl BuddyStats {
- pub const fn new() -> Self {
- Self {
- total_pages: 0,
- free_pages: 0,
- used_pages: 0,
- free_pages_by_order: [0; DEFAULT_MAX_ORDER + 1],
- }
- }
-
- /// Add statistics from another BuddyStats
- pub fn add(&mut self, other: &BuddyStats) {
- self.total_pages += other.total_pages;
- self.free_pages += other.free_pages;
- self.used_pages += other.used_pages;
- for (i, &count) in other.free_pages_by_order.iter().enumerate() {
- self.free_pages_by_order[i] += count;
- }
- }
-}
-
-/// Detailed memory statistics reporter
-pub struct MemoryStatsReporter;
-
-impl MemoryStatsReporter {
- /// Print detailed allocation failure statistics
- /// This is a standalone function to keep allocation logic clean
- #[allow(unused_variables)]
- pub fn print_alloc_failure_stats(
- page_size: usize,
- num_zones: usize,
- total_stats: &BuddyStats,
- zone_infos: &[ZoneInfo],
- zone_stats: &[BuddyStats],
- request_pages: usize,
- request_align: usize,
- ) {
- {
- #[cfg(feature = "log")]
- use log::error;
- error!("========================================");
- error!(
- "Request: {} pages ({} KB, alignment:{})",
- request_pages,
- (request_pages * page_size) / (1024),
- request_align
- );
-
- error!("Overall Memory State:");
- error!(" Total zones: {num_zones}");
- error!(
- " Total pages: {} ({} KB)",
- total_stats.total_pages,
- (total_stats.total_pages * page_size) / 1024
- );
- error!(
- " Free pages: {} ({} KB)",
- total_stats.free_pages,
- (total_stats.free_pages * page_size) / (1024)
- );
- error!(
- " Used pages: {} ({} KB)",
- total_stats.used_pages,
- (total_stats.used_pages * page_size) / (1024)
- );
- error!("========================================");
-
- for (i, zone_info) in zone_infos.iter().take(num_zones).enumerate() {
- error!("Zone {i}:");
- error!(
- " Range: [{:#x}, {:#x})",
- zone_info.start_addr, zone_info.end_addr
- );
- error!(" Total pages: {}", zone_info.total_pages);
- error!(
- " Free pages: {} / {}",
- zone_stats[i].free_pages, zone_info.total_pages
- );
- error!(" Free blocks by order:");
-
- for order in (0..=DEFAULT_MAX_ORDER).rev() {
- let count = zone_stats[i].free_pages_by_order[order];
- if count > 0 {
- let block_size = (1 << order) * page_size;
- let total_kb = (count * block_size) / (1024);
- error!(
- " Order {}: {} blocks ({} KB each, {} KB total)",
- order,
- count,
- block_size / (1024),
- total_kb
- );
- }
- }
- error!("----------------------------------------");
- }
-
- error!("========================================");
- }
- }
-}
diff --git a/src/error.rs b/src/error.rs
new file mode 100644
index 0000000..e48693d
--- /dev/null
+++ b/src/error.rs
@@ -0,0 +1,34 @@
+use core::fmt;
+
+/// The error type used for allocation operations.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AllocError {
+ /// Invalid size, alignment, or other input parameter.
+ InvalidParam,
+ /// A region overlaps with an existing managed region.
+ MemoryOverlap,
+ /// Not enough memory is available to satisfy the request.
+ NoMemory,
+ /// Attempted to deallocate memory that was not allocated.
+ NotAllocated,
+ /// The allocator has not been initialized.
+ NotInitialized,
+ /// The requested address or entity was not found in any managed region.
+ NotFound,
+}
+
+impl fmt::Display for AllocError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Self::InvalidParam => write!(f, "invalid parameter"),
+ Self::MemoryOverlap => write!(f, "memory regions overlap"),
+ Self::NoMemory => write!(f, "out of memory"),
+ Self::NotAllocated => write!(f, "memory not allocated"),
+ Self::NotInitialized => write!(f, "allocator not initialized"),
+ Self::NotFound => write!(f, "not found"),
+ }
+ }
+}
+
+/// A [`Result`] alias with [`AllocError`] as the error type.
+pub type AllocResult = Result;
diff --git a/src/global.rs b/src/global.rs
new file mode 100644
index 0000000..ead8131
--- /dev/null
+++ b/src/global.rs
@@ -0,0 +1,424 @@
+/// Global allocator composing buddy (pages) + per-CPU slab (objects).
+///
+/// Implements [`core::alloc::GlobalAlloc`] so it can serve as `#[global_allocator]`.
+/// Cross-CPU frees are lock-free via [`SlabPageHeader::remote_free`].
+use core::alloc::{GlobalAlloc, Layout};
+use core::ptr::{self, NonNull};
+use core::sync::atomic::{AtomicBool, Ordering};
+
+use spin::Mutex as SpinMutex;
+
+use crate::buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags, SectionInitSpec};
+use crate::error::{AllocError, AllocResult};
+use crate::slab::page::{SLAB_MAGIC, SlabPageHeader};
+use crate::slab::size_class::{SLAB_MAX_SIZE, SizeClass};
+use crate::slab::{SlabAllocResult, SlabAllocator, SlabDeallocResult};
+use crate::{OsImpl, align_up};
+
+struct InitialRegionLayout {
+ section_start: usize,
+ meta_start: usize,
+ meta_size: usize,
+ buddy_meta_size: usize,
+ slab_offset: usize,
+ managed_heap_start: usize,
+ managed_heap_size: usize,
+}
+
+/// Unified allocator: buddy page allocator + per-CPU slab caches.
+pub struct GlobalAllocator {
+ buddy: SpinMutex>,
+ per_cpu_slabs: *mut SpinMutex>,
+ cpu_count: usize,
+ os: Option<&'static dyn OsImpl>,
+ initialized: AtomicBool,
+}
+
+// SAFETY: All mutable state is behind SpinMutex or AtomicBool.
+// `per_cpu_slabs` is a raw pointer into the reserved metadata prefix of the
+// caller-provided region.
+unsafe impl Sync for GlobalAllocator {}
+unsafe impl Send for GlobalAllocator {}
+
+impl GlobalAllocator {
+ const fn metadata_align() -> usize {
+ let a1 = core::mem::align_of::();
+ let a2 = core::mem::align_of::();
+ let a3 = core::mem::align_of::>>();
+ let m = if a1 > a2 { a1 } else { a2 };
+ if m > a3 { m } else { a3 }
+ }
+
+ fn metadata_layout_for_pages(pages: usize, cpu_count: usize) -> Option<(usize, usize, usize)> {
+ let meta_offset = align_up(
+ core::mem::size_of::(),
+ core::mem::align_of::(),
+ );
+ let buddy_meta_size = pages.checked_mul(core::mem::size_of::())?;
+ let slab_align = core::mem::align_of::>>();
+ let slab_offset = align_up(meta_offset.checked_add(buddy_meta_size)?, slab_align);
+ let slab_size =
+ core::mem::size_of::>>().checked_mul(cpu_count)?;
+ let meta_size = slab_offset.checked_add(slab_size)?;
+ Some((meta_offset, buddy_meta_size, meta_size))
+ }
+
+ fn available_heap_pages(
+ region_end: usize,
+ section_start: usize,
+ meta_size: usize,
+ ) -> Option {
+ let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE);
+ if managed_heap_start > region_end {
+ return Some(0);
+ }
+ Some((region_end - managed_heap_start) / PAGE_SIZE)
+ }
+
+ fn can_manage_pages(
+ region_end: usize,
+ section_start: usize,
+ cpu_count: usize,
+ pages: usize,
+ ) -> bool {
+ let Some((_, _, meta_size)) = Self::metadata_layout_for_pages(pages, cpu_count) else {
+ return false;
+ };
+ let Some(available_pages) =
+ Self::available_heap_pages(region_end, section_start, meta_size)
+ else {
+ return false;
+ };
+ available_pages >= pages
+ }
+
+ fn compute_initial_region_layout(
+ region_start: usize,
+ region_size: usize,
+ cpu_count: usize,
+ ) -> Option {
+ if cpu_count == 0 || region_size == 0 || !PAGE_SIZE.is_power_of_two() {
+ return None;
+ }
+
+ let region_end = region_start.checked_add(region_size)?;
+ let section_start = align_up(region_start, Self::metadata_align());
+ if section_start >= region_end {
+ return None;
+ }
+
+ let heap_search_start = align_up(
+ section_start.checked_add(core::mem::size_of::())?,
+ PAGE_SIZE,
+ );
+ let max_pages = if heap_search_start >= region_end {
+ 0
+ } else {
+ (region_end - heap_search_start) / PAGE_SIZE
+ };
+
+ let mut low = 0usize;
+ let mut high = max_pages;
+ while low < high {
+ let mid = low + (high - low).div_ceil(2);
+ if Self::can_manage_pages(region_end, section_start, cpu_count, mid) {
+ low = mid;
+ } else {
+ high = mid - 1;
+ }
+ }
+
+ if low == 0 {
+ return None;
+ }
+
+ let (meta_offset, buddy_meta_size, meta_size) =
+ Self::metadata_layout_for_pages(low, cpu_count)?;
+ let meta_start = section_start.checked_add(meta_offset)?;
+ let slab_offset = align_up(
+ meta_offset.checked_add(buddy_meta_size)?,
+ core::mem::align_of::>>(),
+ );
+ let managed_heap_start = align_up(section_start.checked_add(meta_size)?, PAGE_SIZE);
+ let managed_heap_size = low.checked_mul(PAGE_SIZE)?;
+
+ Some(InitialRegionLayout {
+ section_start,
+ meta_start,
+ meta_size,
+ buddy_meta_size,
+ slab_offset,
+ managed_heap_start,
+ managed_heap_size,
+ })
+ }
+
+ /// Create an uninitialised global allocator.
+ pub const fn new() -> Self {
+ Self {
+ buddy: SpinMutex::new(BuddyAllocator::new()),
+ per_cpu_slabs: ptr::null_mut(),
+ cpu_count: 0,
+ os: None,
+ initialized: AtomicBool::new(false),
+ }
+ }
+}
+
+impl Default for GlobalAllocator {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl GlobalAllocator {
+ /// Initialise the allocator over the first region.
+ ///
+ /// # Safety
+ /// - `region` must be writable and remain valid for the lifetime of this allocator.
+ /// - Any bytes consumed by metadata or alignment padding become unavailable for allocation.
+ pub unsafe fn init(
+ &self,
+ region: &mut [u8],
+ cpu_count: usize,
+ os: &'static dyn OsImpl,
+ ) -> AllocResult {
+ unsafe {
+ let region_start = region.as_mut_ptr() as usize;
+ let region_size = region.len();
+ let layout = Self::compute_initial_region_layout(region_start, region_size, cpu_count)
+ .ok_or(AllocError::InvalidParam)?;
+ let section_ptr = layout.section_start as *mut BuddySection;
+ let meta_ptr = layout.meta_start as *mut u8;
+ let slab_ptr = (layout.section_start + layout.slab_offset)
+ as *mut SpinMutex>;
+
+ let mut buddy = self.buddy.lock();
+ buddy.reset(Some(os));
+ buddy.add_region_raw(SectionInitSpec {
+ region_start,
+ region_size,
+ section_ptr,
+ meta_ptr,
+ meta_size: layout.buddy_meta_size,
+ heap_start: layout.managed_heap_start,
+ heap_size: layout.managed_heap_size,
+ })?;
+ drop(buddy);
+
+ for i in 0..cpu_count {
+ let slot = slab_ptr.add(i);
+ slot.write(SpinMutex::new(SlabAllocator::new()));
+ }
+
+ let self_mut = self as *const Self as *mut Self;
+ (*self_mut).per_cpu_slabs = slab_ptr;
+ (*self_mut).cpu_count = cpu_count;
+ (*self_mut).os = Some(os);
+ self.initialized.store(true, Ordering::Release);
+
+ log::debug!(
+ "GlobalAllocator: {} CPUs, region {:#x}+{:#x}, meta {:#x}+{:#x}, first heap {:#x}+{:#x}",
+ cpu_count,
+ region_start,
+ region_size,
+ layout.section_start,
+ layout.meta_size,
+ layout.managed_heap_start,
+ layout.managed_heap_size,
+ );
+
+ Ok(())
+ }
+ }
+
+ /// Add a new managed region after [`init`](Self::init).
+ ///
+ /// # Safety
+ /// - `region` must be writable and remain valid for the lifetime of this allocator.
+ /// - The region must not overlap any already managed region.
+ pub unsafe fn add_region(&self, region: &mut [u8]) -> AllocResult {
+ unsafe {
+ if !self.initialized.load(Ordering::Acquire) {
+ return Err(AllocError::NotInitialized);
+ }
+ self.buddy.lock().add_region(region)
+ }
+ }
+
+ /// Number of managed sections.
+ pub fn managed_section_count(&self) -> usize {
+ self.buddy.lock().section_count()
+ }
+
+ /// Read-only summary for a managed section.
+ pub fn managed_section(&self, index: usize) -> Option {
+ self.buddy.lock().section(index)
+ }
+
+ /// Total managed heap bytes across all sections.
+ ///
+ /// This excludes region-prefix metadata such as `BuddySection`, `PageMeta[]`,
+ /// and per-CPU slab slots.
+ pub fn managed_bytes(&self) -> usize {
+ self.buddy.lock().managed_bytes()
+ }
+
+ /// Allocated backend bytes across all sections.
+ ///
+ /// This is page-level occupancy, not the exact sum of requested layout sizes.
+ pub fn allocated_bytes(&self) -> usize {
+ self.buddy.lock().allocated_bytes()
+ }
+
+ /// Allocate contiguous pages. Returns the virtual start address.
+ pub fn alloc_pages(&self, count: usize, align: usize) -> AllocResult {
+ self.buddy.lock().alloc_pages(count, align)
+ }
+
+ /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages).
+ pub fn dealloc_pages(&self, addr: usize, count: usize) {
+ self.buddy.lock().dealloc_pages(addr, count);
+ }
+
+ /// Allocate pages with physical address below 4 GiB.
+ pub fn alloc_pages_lowmem(&self, count: usize, align: usize) -> AllocResult {
+ self.buddy.lock().alloc_pages_lowmem(count, align)
+ }
+
+ /// Allocate memory for `layout`. Returns a pointer on success.
+ pub fn alloc(&self, layout: Layout) -> AllocResult> {
+ if !self.initialized.load(Ordering::Acquire) {
+ return Err(AllocError::NotInitialized);
+ }
+
+ if self.is_slab_eligible(&layout) {
+ self.slab_alloc(layout)
+ } else {
+ self.large_alloc(layout)
+ }
+ }
+
+ /// Deallocate memory previously returned by [`alloc`](Self::alloc).
+ ///
+ /// # Safety
+ /// `ptr` must have been returned by a prior `alloc` with the same `layout`.
+ pub unsafe fn dealloc(&self, ptr: NonNull, layout: Layout) {
+ unsafe {
+ if self.is_slab_eligible(&layout) {
+ self.slab_dealloc(ptr, layout);
+ } else {
+ self.large_dealloc(ptr, layout);
+ }
+ }
+ }
+
+ #[inline]
+ fn is_slab_eligible(&self, layout: &Layout) -> bool {
+ layout.size() <= SLAB_MAX_SIZE && layout.align() <= SLAB_MAX_SIZE
+ }
+
+ fn slab_alloc(&self, layout: Layout) -> AllocResult> {
+ let os = self.os.ok_or(AllocError::NotInitialized)?;
+ let cpu = os.current_cpu_idx();
+ debug_assert!(cpu < self.cpu_count);
+
+ let slab_lock = unsafe { &*self.per_cpu_slabs.add(cpu) };
+ let mut slab = slab_lock.lock();
+
+ match slab.alloc(layout)? {
+ SlabAllocResult::Allocated(ptr) => Ok(ptr),
+ SlabAllocResult::NeedsSlab { size_class, pages } => {
+ drop(slab);
+ let bytes = pages * PAGE_SIZE;
+ let addr = self.buddy.lock().alloc_pages(pages, bytes)?;
+ unsafe {
+ self.buddy.lock().set_page_flags(addr, PageFlags::Slab)?;
+ }
+ let mut slab = slab_lock.lock();
+ slab.add_slab(size_class, addr, bytes, cpu as u16);
+ match slab.alloc(layout)? {
+ SlabAllocResult::Allocated(ptr) => Ok(ptr),
+ SlabAllocResult::NeedsSlab { .. } => Err(AllocError::NoMemory),
+ }
+ }
+ }
+ }
+
+ unsafe fn slab_dealloc(&self, ptr: NonNull, layout: Layout) {
+ unsafe {
+ let os = self.os.expect("not initialized");
+ let sc = SizeClass::from_layout(layout).expect("layout exceeds slab");
+ let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE;
+ let base =
+ SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes);
+ let hdr = &*(base as *const SlabPageHeader);
+ debug_assert_eq!(hdr.magic, SLAB_MAGIC);
+
+ let owner_cpu = hdr.owner_cpu as usize;
+ let current_cpu = os.current_cpu_idx();
+
+ if owner_cpu == current_cpu {
+ let slab_lock = &*self.per_cpu_slabs.add(current_cpu);
+ let mut slab = slab_lock.lock();
+ match slab.dealloc(ptr, layout) {
+ SlabDeallocResult::Done => {}
+ SlabDeallocResult::FreeSlab { base, pages } => {
+ drop(slab);
+ self.buddy.lock().dealloc_pages(base, pages);
+ }
+ }
+ } else {
+ hdr.remote_free(ptr.as_ptr() as usize);
+ }
+ }
+ }
+
+ fn large_alloc(&self, layout: Layout) -> AllocResult> {
+ let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE;
+ let align = layout.align().max(PAGE_SIZE);
+ let addr = self.buddy.lock().alloc_pages(pages, align)?;
+ Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) })
+ }
+
+ unsafe fn large_dealloc(&self, ptr: NonNull, layout: Layout) {
+ let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE;
+ self.buddy
+ .lock()
+ .dealloc_pages(ptr.as_ptr() as usize, pages);
+ }
+}
+
+unsafe impl GlobalAlloc for GlobalAllocator {
+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+ match self.alloc(layout) {
+ Ok(ptr) => ptr.as_ptr(),
+ Err(_) => ptr::null_mut(),
+ }
+ }
+
+ unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+ unsafe {
+ if let Some(nn) = NonNull::new(ptr) {
+ self.dealloc(nn, layout);
+ }
+ }
+ }
+
+ unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+ unsafe {
+ let new_layout = match Layout::from_size_align(new_size, layout.align()) {
+ Ok(l) => l,
+ Err(_) => return ptr::null_mut(),
+ };
+
+ let new_ptr = ::alloc(self, new_layout);
+ if !new_ptr.is_null() {
+ let copy_size = layout.size().min(new_size);
+ ptr::copy_nonoverlapping(ptr, new_ptr, copy_size);
+ ::dealloc(self, ptr, layout);
+ }
+ new_ptr
+ }
+ }
+}
diff --git a/src/global_allocator.rs b/src/global_allocator.rs
deleted file mode 100644
index d5393ee..0000000
--- a/src/global_allocator.rs
+++ /dev/null
@@ -1,524 +0,0 @@
-//! Global allocator implementation.
-//!
-//! This module implements a global allocator that coordinates between
-//! buddy page allocator and slab byte allocator for optimal performance.
-
-extern crate alloc;
-
-use crate::{AllocError, AllocResult, BaseAllocator, ByteAllocator, PageAllocator};
-use core::alloc::Layout;
-use core::ptr::NonNull;
-#[cfg(feature = "tracking")]
-use core::sync::atomic::AtomicUsize;
-use core::sync::atomic::{AtomicBool, Ordering};
-
-#[cfg(feature = "tracking")]
-use super::buddy::BuddyStats;
-use super::page_allocator::CompositePageAllocator;
-use super::slab::{PageAllocatorForSlab, SlabByteAllocator};
-
-#[cfg(feature = "log")]
-use log::error;
-
-const MIN_HEAP_SIZE: usize = 0x8000; // 32KB minimum heap
-
-/// Memory usage statistics
-#[cfg(feature = "tracking")]
-#[derive(Debug, Clone, Copy, Default)]
-pub struct UsageStats {
- pub total_pages: usize,
- pub used_pages: usize,
- pub free_pages: usize,
- pub slab_bytes: usize,
- pub heap_bytes: usize,
-}
-
-/// Internal atomic representation of usage statistics
-#[cfg(feature = "tracking")]
-struct UsageStatsAtomic {
- total_pages: AtomicUsize,
- used_pages: AtomicUsize,
- free_pages: AtomicUsize,
- slab_bytes: AtomicUsize,
- heap_bytes: AtomicUsize,
-}
-
-#[cfg(feature = "tracking")]
-impl UsageStatsAtomic {
- const fn new() -> Self {
- Self {
- total_pages: AtomicUsize::new(0),
- used_pages: AtomicUsize::new(0),
- free_pages: AtomicUsize::new(0),
- slab_bytes: AtomicUsize::new(0),
- heap_bytes: AtomicUsize::new(0),
- }
- }
-
- fn snapshot(&self) -> UsageStats {
- UsageStats {
- total_pages: self.total_pages.load(Ordering::Relaxed),
- used_pages: self.used_pages.load(Ordering::Relaxed),
- free_pages: self.free_pages.load(Ordering::Relaxed),
- slab_bytes: self.slab_bytes.load(Ordering::Relaxed),
- heap_bytes: self.heap_bytes.load(Ordering::Relaxed),
- }
- }
-}
-
-#[cfg(feature = "tracking")]
-#[inline]
-fn saturating_sub_atomic(counter: &AtomicUsize, value: usize) {
- let mut prev = counter.load(Ordering::Relaxed);
- loop {
- let new = prev.saturating_sub(value);
- match counter.compare_exchange(prev, new, Ordering::AcqRel, Ordering::Relaxed) {
- Ok(_) => break,
- Err(actual) => prev = actual,
- }
- }
-}
-
-/// Global allocator that coordinates composite and slab allocators
-pub struct GlobalAllocator {
- page_allocator: CompositePageAllocator,
- slab_allocator: SlabByteAllocator,
- #[cfg(feature = "tracking")]
- stats: UsageStatsAtomic,
- initialized: AtomicBool,
-}
-
-impl GlobalAllocator {
- pub const fn new() -> Self {
- Self {
- page_allocator: CompositePageAllocator::::new(),
- slab_allocator: SlabByteAllocator::::new(),
- #[cfg(feature = "tracking")]
- stats: UsageStatsAtomic::new(),
- initialized: AtomicBool::new(false),
- }
- }
-
- /// Set the address translator so that the underlying page allocator can
- /// reason about physical address ranges (e.g. low-memory regions below 4GiB).
- pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) {
- self.page_allocator.set_addr_translator(translator);
- }
-
- /// Allocate low-memory pages (physical address < 4GiB).
- /// This is a thin wrapper over the composite allocator's lowmem API.
- pub fn alloc_dma32_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- if !self.initialized.load(Ordering::SeqCst) {
- error!("global allocator: Allocator not initialized");
- return Err(AllocError::NoMemory);
- }
-
- let addr = self
- .page_allocator
- .alloc_pages_lowmem(num_pages, alignment)?;
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .used_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- self.stats
- .free_pages
- .fetch_sub(num_pages, Ordering::Relaxed);
- }
-
- Ok(addr)
- }
-
- /// Initialize allocator with given memory region
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use buddy_slab_allocator::GlobalAllocator;
- ///
- /// const PAGE_SIZE: usize = 0x1000;
- /// let mut allocator = GlobalAllocator::::new();
- /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
- /// ```
- pub fn init(&mut self, start_vaddr: usize, size: usize) -> AllocResult<()> {
- if size <= MIN_HEAP_SIZE {
- return Err(AllocError::InvalidParam);
- }
-
- self.page_allocator.init(start_vaddr, size);
-
- self.slab_allocator.init();
-
- {
- let page_alloc_ptr = &mut self.page_allocator as *mut CompositePageAllocator;
- self.slab_allocator
- .set_page_allocator(page_alloc_ptr as *mut dyn PageAllocatorForSlab);
- }
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .total_pages
- .store(self.page_allocator.total_pages(), Ordering::Relaxed);
- self.stats
- .used_pages
- .store(self.page_allocator.used_pages(), Ordering::Relaxed);
- self.stats
- .free_pages
- .store(self.page_allocator.available_pages(), Ordering::Relaxed);
- }
-
- self.initialized.store(true, Ordering::SeqCst);
- Ok(())
- }
-
- /// Dynamically add memory region to allocator
- pub fn add_memory(&mut self, start_vaddr: usize, size: usize) -> AllocResult<()> {
- self.page_allocator.add_memory(start_vaddr, size)?;
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .total_pages
- .store(self.page_allocator.total_pages(), Ordering::Relaxed);
- self.stats
- .free_pages
- .store(self.page_allocator.available_pages(), Ordering::Relaxed);
- }
-
- Ok(())
- }
-
- /// Smart allocation based on size
- ///
- /// Small allocations (≤2048 bytes) use slab allocator,
- /// larger allocations use page allocator.
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use buddy_slab_allocator::GlobalAllocator;
- /// use core::alloc::Layout;
- ///
- /// const PAGE_SIZE: usize = 0x1000;
- /// let mut allocator = GlobalAllocator::::new();
- /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
- ///
- /// let layout = Layout::from_size_align(64, 8).unwrap();
- /// let ptr = allocator.alloc(layout).unwrap();
- /// allocator.dealloc(ptr, layout);
- /// ```
- pub fn alloc(&mut self, layout: Layout) -> AllocResult> {
- if !self.initialized.load(Ordering::SeqCst) {
- error!("global allocator: Allocator not initialized");
- return Err(AllocError::NoMemory);
- }
-
- if layout.size() <= 2048 && layout.align() <= 2048 {
- // Try slab allocator first
- match self.slab_allocator.alloc(layout) {
- Ok(ptr) => {
- #[cfg(feature = "tracking")]
- {
- self.stats
- .slab_bytes
- .fetch_add(layout.size(), Ordering::Relaxed);
- }
- return Ok(ptr);
- }
- Err(e) => {
- // Slab allocator should handle all requests that satisfy constraints
- // If it fails, it's a real error (e.g., out of memory)
- // Log for debugging
- error!(
- "global allocator: Slab allocator failed for layout {layout:?}, error: {e:?}, falling back to page allocator"
- );
- return Err(e);
- }
- }
- }
-
- let pages_needed = layout.size().div_ceil(PAGE_SIZE);
-
- let addr =
- PageAllocator::alloc_pages(&mut self.page_allocator, pages_needed, layout.align())?;
- let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) };
-
- #[cfg(feature = "tracking")]
- {
- self.stats
- .used_pages
- .fetch_add(pages_needed, Ordering::Relaxed);
- self.stats
- .free_pages
- .fetch_sub(pages_needed, Ordering::Relaxed);
- self.stats
- .heap_bytes
- .fetch_add(layout.size(), Ordering::Relaxed);
- }
-
- Ok(ptr)
- }
-
- /// Allocate pages
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use buddy_slab_allocator::{GlobalAllocator, PageAllocator};
- ///
- /// const PAGE_SIZE: usize = 0x1000;
- /// let mut allocator = GlobalAllocator::::new();
- /// allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
- ///
- /// let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap();
- /// allocator.dealloc_pages(addr, 4);
- /// ```
- pub fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- if !self.initialized.load(Ordering::SeqCst) {
- return Err(AllocError::NoMemory);
- }
-
- let addr = PageAllocator::alloc_pages(&mut self.page_allocator, num_pages, alignment)?;
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .used_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- self.stats
- .free_pages
- .fetch_sub(num_pages, Ordering::Relaxed);
- }
-
- Ok(addr)
- }
-
- /// Deallocate memory
- pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) {
- if !self.initialized.load(Ordering::SeqCst) {
- error!("global allocator: Deallocating memory before initializing");
- return;
- }
-
- if layout.size() <= 2048 && layout.align() <= 2048 {
- // This memory must have been allocated by slab allocator
- // If dealloc fails (not found in slab), it's a critical error
- self.slab_allocator.dealloc(ptr, layout);
- #[cfg(feature = "tracking")]
- {
- saturating_sub_atomic(&self.stats.slab_bytes, layout.size());
- }
- return;
- }
-
- // This memory was allocated by page allocator
- let pages_needed = layout.size().div_ceil(PAGE_SIZE);
- PageAllocator::dealloc_pages(
- &mut self.page_allocator,
- ptr.as_ptr() as usize,
- pages_needed,
- );
- #[cfg(feature = "tracking")]
- {
- saturating_sub_atomic(&self.stats.used_pages, pages_needed);
- self.stats
- .free_pages
- .fetch_add(pages_needed, Ordering::Relaxed);
- saturating_sub_atomic(&self.stats.heap_bytes, layout.size());
- }
- }
-
- /// Deallocate pages
- pub fn dealloc_pages(&mut self, pos: usize, num_pages: usize) {
- if !self.initialized.load(Ordering::SeqCst) {
- return;
- }
-
- PageAllocator::dealloc_pages(&mut self.page_allocator, pos, num_pages);
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- saturating_sub_atomic(&self.stats.used_pages, num_pages);
- self.stats
- .free_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- }
- }
-
- /// Reallocate memory
- #[allow(clippy::not_unsafe_ptr_arg_deref)]
- pub fn realloc(&mut self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- if new_size == 0 {
- if let Some(ptr) = NonNull::new(ptr) {
- self.dealloc(ptr, layout);
- }
- return core::ptr::null_mut();
- }
-
- if ptr.is_null() {
- let new_layout = Layout::from_size_align(new_size, layout.align())
- .unwrap_or_else(|_| Layout::new::());
- return match self.alloc(new_layout) {
- Ok(ptr) => ptr.as_ptr(),
- Err(_) => core::ptr::null_mut(),
- };
- }
-
- let new_layout = Layout::from_size_align(new_size, layout.align())
- .unwrap_or_else(|_| Layout::new::());
-
- // If new size fits in old allocation, return old pointer
- if new_size <= layout.size() {
- return ptr;
- }
-
- // Allocate new memory and copy
- match self.alloc(new_layout) {
- Ok(new_ptr) => {
- let new_ptr = new_ptr.as_ptr();
- unsafe {
- core::ptr::copy_nonoverlapping(
- ptr,
- new_ptr,
- core::cmp::min(layout.size(), new_size),
- );
- }
- if let Some(ptr) = NonNull::new(ptr) {
- self.dealloc(ptr, layout);
- }
- new_ptr
- }
- Err(_) => core::ptr::null_mut(),
- }
- }
-}
-
-impl GlobalAllocator {
- /// Get memory statistics
- #[cfg(feature = "tracking")]
- pub fn get_stats(&self) -> UsageStats {
- self.stats.snapshot()
- }
-
- /// Get buddy allocator statistics
- #[cfg(feature = "tracking")]
- pub fn get_buddy_stats(&self) -> BuddyStats {
- self.page_allocator.get_buddy_stats()
- }
-}
-
-impl Default for GlobalAllocator {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl BaseAllocator for GlobalAllocator {
- fn init(&mut self, start: usize, size: usize) {
- self.page_allocator.init(start, size);
- }
-
- fn add_memory(&mut self, start: usize, size: usize) -> AllocResult {
- self.page_allocator.add_memory(start, size)
- }
-}
-
-impl PageAllocator for GlobalAllocator {
- const PAGE_SIZE: usize = PAGE_SIZE;
-
- fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- if !self.initialized.load(Ordering::SeqCst) {
- return Err(AllocError::NoMemory);
- }
-
- let addr = as PageAllocator>::alloc_pages(
- &mut self.page_allocator,
- num_pages,
- alignment,
- )?;
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .used_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- self.stats
- .free_pages
- .fetch_sub(num_pages, Ordering::Relaxed);
- }
-
- Ok(addr)
- }
-
- fn dealloc_pages(&mut self, pos: usize, num_pages: usize) {
- if !self.initialized.load(Ordering::SeqCst) {
- return;
- }
-
- as PageAllocator>::dealloc_pages(
- &mut self.page_allocator,
- pos,
- num_pages,
- );
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- saturating_sub_atomic(&self.stats.used_pages, num_pages);
- self.stats
- .free_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- }
- }
-
- fn alloc_pages_at(
- &mut self,
- base: usize,
- num_pages: usize,
- alignment: usize,
- ) -> AllocResult {
- if !self.initialized.load(Ordering::SeqCst) {
- return Err(AllocError::NoMemory);
- }
-
- let addr = as PageAllocator>::alloc_pages_at(
- &mut self.page_allocator,
- base,
- num_pages,
- alignment,
- )?;
-
- // Update statistics
- #[cfg(feature = "tracking")]
- {
- self.stats
- .used_pages
- .fetch_add(num_pages, Ordering::Relaxed);
- self.stats
- .free_pages
- .fetch_sub(num_pages, Ordering::Relaxed);
- }
-
- Ok(addr)
- }
-
- fn total_pages(&self) -> usize {
- self.page_allocator.total_pages()
- }
-
- fn used_pages(&self) -> usize {
- self.page_allocator.used_pages()
- }
-
- fn available_pages(&self) -> usize {
- self.page_allocator.available_pages()
- }
-}
diff --git a/src/lib.rs b/src/lib.rs
index 157b6a0..5bb26b7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,174 +1,52 @@
-//! buddy-slab-allocator Memory Allocator
+//! # buddy-slab-allocator
//!
-//! This crate implements a high-performance memory allocator designed for embedded
-//! and kernel environments, featuring:
-//! - Buddy page allocator for page-level allocation
-//! - Slab allocator for small object allocation
-//! - Global allocator coordination
-//! - Zero `std` dependency (fully `#![no_std]`)
+//! A `#![no_std]` memory allocator featuring:
//!
-//! # Features
+//! - **Buddy page allocator** — page-metadata-based with intrusive free lists
+//! - **Slab allocator** — bitmap-based with lock-free cross-CPU freeing (Linux SLUB inspired)
+//! - **Global allocator** — composes buddy + per-CPU slab, implements [`core::alloc::GlobalAlloc`]
//!
-//! - **Buddy Page Allocator**: Efficient page-level memory allocation with automatic merging
-//! - **Slab Byte Allocator**: Fast small object allocation (≤2048 bytes)
-//! - **Global Allocator**: Automatic selection between page and slab allocation based on size
-//! - **No_std Compatible**: Fully `#![no_std]` for embedded/kernel use
-//! - **Optional Logging**: Conditional compilation with `log` feature
-//! - **Memory Tracking**: Detailed statistics with `tracking` feature
-//!
-//! # Quick Start
-//!
-//! ```no_run
-//! use buddy_slab_allocator::{GlobalAllocator, PageAllocator};
-//! use core::alloc::Layout;
-//!
-//! const PAGE_SIZE: usize = 0x1000;
-//! let mut allocator = GlobalAllocator::::new();
-//!
-//! // Initialize with memory region
-//! let heap_start = 0x8000_0000;
-//! let heap_size = 16 * 1024 * 1024; // 16MB
-//! allocator.init(heap_start, heap_size).unwrap();
-//!
-//! // Allocate pages
-//! let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap();
-//! // Use the allocated memory...
-//! allocator.dealloc_pages(addr, 4);
-//! ```
-//!
-//! # Small Object Allocation
-//!
-//! ```no_run
-//! use buddy_slab_allocator::GlobalAllocator;
-//! use core::alloc::Layout;
-//!
-//! const PAGE_SIZE: usize = 0x1000;
-//! let mut allocator = GlobalAllocator::::new();
-//! allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
-//!
-//! // Small allocations go through slab allocator
-//! let layout = Layout::from_size_align(64, 8).unwrap();
-//! let ptr = allocator.alloc(layout).unwrap();
-//! // Use the allocated memory...
-//! allocator.dealloc(ptr, layout);
-//! ```
-//!
-//! # Statistics Tracking
-//!
-//! ```no_run
-//! # #[cfg(feature = "tracking")]
-//! # {
-//! use buddy_slab_allocator::GlobalAllocator;
-//!
-//! const PAGE_SIZE: usize = 0x1000;
-//! let mut allocator = GlobalAllocator::::new();
-//! allocator.init(0x8000_0000, 16 * 1024 * 1024).unwrap();
-//!
-//! let stats = allocator.get_stats();
-//! println!("Total pages: {}", stats.total_pages);
-//! println!("Used pages: {}", stats.used_pages);
-//! println!("Free pages: {}", stats.free_pages);
-//! # }
-//! ```
+//! Both buddy and slab allocators can be used standalone.
#![no_std]
-extern crate alloc;
+mod error;
+pub use error::{AllocError, AllocResult};
-// Logging support - conditionally import log crate
-#[cfg(feature = "log")]
-extern crate log;
+pub mod buddy;
+pub use buddy::{BuddyAllocator, ManagedSection};
-// Stub macros when log is disabled - these become no-ops
-#[cfg(not(feature = "log"))]
-macro_rules! error {
- ($($arg:tt)*) => {};
-}
-#[cfg(not(feature = "log"))]
-macro_rules! warn {
- ($($arg:tt)*) => {};
-}
-#[cfg(not(feature = "log"))]
-macro_rules! info {
- ($($arg:tt)*) => {};
-}
-#[cfg(not(feature = "log"))]
-macro_rules! debug {
- ($($arg:tt)*) => {};
-}
-#[cfg(not(feature = "log"))]
-#[allow(unused_macros)]
-macro_rules! trace {
- ($($arg:tt)*) => {};
-}
+pub mod slab;
+pub use slab::{SizeClass, SlabAllocResult, SlabAllocator, SlabDeallocResult};
-pub use axallocator::{
- AllocError, AllocResult, BaseAllocator, ByteAllocator, IdAllocator, PageAllocator,
-};
+pub mod global;
+pub use global::GlobalAllocator;
-/// Default page size for backward compatibility (4KB)
-pub const DEFAULT_PAGE_SIZE: usize = 0x1000;
+// ---------------------------------------------------------------------------
+// OsImpl trait — the only interface the allocator needs from the OS / platform
+// ---------------------------------------------------------------------------
-/// Address translator used by allocators to reason about physical addresses.
-///
-/// Implementations should provide a stable virtual-to-physical mapping
-/// for the allocator-managed address range.
-///
-/// # Examples
+/// Platform abstraction required by [`GlobalAllocator`].
///
-/// ```
-/// use buddy_slab_allocator::AddrTranslator;
-///
-/// struct SimpleMapper;
-///
-/// impl AddrTranslator for SimpleMapper {
-/// fn virt_to_phys(&self, va: usize) -> Option {
-/// // Identity mapping for this example
-/// Some(va)
-/// }
-/// }
-/// ```
-pub trait AddrTranslator: Sync {
+/// Implementations must be safe to call from any CPU at any time.
+pub trait OsImpl: Sync + Send {
+ /// Return the index of the current CPU (0-based).
+ fn current_cpu_idx(&self) -> usize;
+
/// Translate a virtual address to a physical address.
- ///
- /// Returns `None` if the address is not valid or not mapped.
- fn virt_to_phys(&self, va: usize) -> Option;
+ fn virt_to_phys(&self, vaddr: usize) -> usize;
}
-#[inline]
-#[allow(dead_code)]
-const fn align_down(pos: usize, align: usize) -> usize {
- pos & !(align - 1)
-}
+// ---------------------------------------------------------------------------
+// Utility helpers (crate-internal)
+// ---------------------------------------------------------------------------
#[inline]
-#[allow(dead_code)]
-const fn align_up(pos: usize, align: usize) -> usize {
+pub(crate) const fn align_up(pos: usize, align: usize) -> usize {
(pos + align - 1) & !(align - 1)
}
-/// Checks whether the address has the demanded alignment.
-///
-/// Equivalent to `addr % align == 0`, but the alignment must be a power of two.
#[inline]
-#[allow(dead_code)]
-const fn is_aligned(base_addr: usize, align: usize) -> bool {
- base_addr & (align - 1) == 0
+pub(crate) const fn is_aligned(addr: usize, align: usize) -> bool {
+ addr & (align - 1) == 0
}
-
-// Export our allocator implementations
-pub mod buddy;
-#[cfg(feature = "tracking")]
-pub use buddy::BuddyStats;
-pub use buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER, MAX_ZONES};
-
-pub mod page_allocator;
-pub use page_allocator::CompositePageAllocator;
-
-pub mod slab;
-pub use slab::slab_byte_allocator::{PageAllocatorForSlab, SizeClass, SlabByteAllocator};
-
-pub mod global_allocator;
-pub use global_allocator::GlobalAllocator;
-#[cfg(feature = "tracking")]
-pub use global_allocator::UsageStats;
diff --git a/src/page_allocator.rs b/src/page_allocator.rs
deleted file mode 100644
index 6abdff9..0000000
--- a/src/page_allocator.rs
+++ /dev/null
@@ -1,560 +0,0 @@
-//! Page allocator with contiguous block combination support.
-
-use crate::buddy::{BuddyPageAllocator, DEFAULT_MAX_ORDER};
-use crate::{AllocError, AllocResult, BaseAllocator, PageAllocator};
-
-#[cfg(feature = "log")]
-use log::{debug, warn};
-
-/// Maximum number of buddy blocks in a single contiguous allocation
-const MAX_PARTS_PER_ALLOC: usize = 8;
-
-/// Maximum number of concurrent composite allocations tracked
-const MAX_COMPOSITE_ALLOCS: usize = 16;
-
-/// Metadata for a composite allocation (multiple buddy blocks combined)
-#[derive(Clone, Copy, Debug)]
-struct CompositeBlockInfo {
- /// Base address of the composite allocation
- base_addr: usize,
- /// Number of parts in this composite allocation
- part_count: u8,
- /// Individual parts: (address, order)
- parts: [(usize, u32); MAX_PARTS_PER_ALLOC],
-}
-
-/// Tracker for composite allocations using fixed-size array
-struct CompositeBlockTracker {
- /// Array to store composite block metadata
- blocks: [Option; MAX_COMPOSITE_ALLOCS],
- /// Number of active composite allocations
- count: usize,
-}
-
-impl CompositeBlockTracker {
- const fn new() -> Self {
- Self {
- blocks: [None; MAX_COMPOSITE_ALLOCS],
- count: 0,
- }
- }
-
- /// Insert a new composite block
- ///
- /// Returns false if the tracker is full
- fn insert(&mut self, base_addr: usize, parts: &[(usize, u32)], part_count: usize) -> bool {
- if self.count >= MAX_COMPOSITE_ALLOCS {
- return false;
- }
-
- let mut info = CompositeBlockInfo {
- base_addr,
- part_count: part_count as u8,
- parts: [(0, 0); MAX_PARTS_PER_ALLOC],
- };
-
- info.parts[..part_count].copy_from_slice(&parts[..part_count]);
-
- self.blocks[self.count] = Some(info);
- self.count += 1;
- true
- }
-
- /// Find a composite block by its base address
- fn find(&self, base_addr: usize) -> Option {
- for i in 0..self.count {
- if let Some(info) = self.blocks[i] {
- if info.base_addr == base_addr {
- return Some(info);
- }
- }
- }
- None
- }
-
- /// Remove a composite block by its base address
- ///
- /// Returns true if the block was found and removed
- fn remove(&mut self, base_addr: usize) -> bool {
- for i in 0..self.count {
- if let Some(info) = self.blocks[i] {
- if info.base_addr == base_addr {
- // Move the last element to this position to keep array compact
- if i < self.count - 1 {
- self.blocks[i] = self.blocks[self.count - 1];
- }
- self.blocks[self.count - 1] = None;
- self.count -= 1;
- return true;
- }
- }
- }
- false
- }
-}
-
-pub struct CompositePageAllocator {
- /// Underlying buddy allocator for standard allocations
- buddy: BuddyPageAllocator,
- /// Tracker for composite allocations (multiple buddy blocks combined)
- composite_tracker: CompositeBlockTracker,
-}
-
-impl CompositePageAllocator {
- /// Create a new page allocator with contiguous block support
- pub const fn new() -> Self {
- Self {
- buddy: BuddyPageAllocator::::new(),
- composite_tracker: CompositeBlockTracker::new(),
- }
- }
-
- /// Set the address translator so that the underlying buddy allocator can
- /// reason about physical address ranges (e.g. low-memory regions).
- pub fn set_addr_translator(&mut self, translator: &'static dyn crate::AddrTranslator) {
- self.buddy.set_addr_translator(translator);
- }
-
- /// Allocate low-memory pages (physical address < 4GiB).
- /// This is a thin wrapper over the buddy allocator's lowmem allocation.
- pub fn alloc_pages_lowmem(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- self.buddy.alloc_pages_lowmem(num_pages, alignment)
- }
-
- /// Try to find and allocate contiguous small blocks from buddy free lists.
- ///
- /// This method searches buddy free lists for contiguous blocks that can satisfy
- /// the allocation request. It uses the sorted nature of free lists to efficiently
- /// check for contiguity.
- ///
- /// # Algorithm
- /// 1. Iterate through free lists from largest to smallest blocks
- /// 2. For each block, check if it's contiguous with already collected blocks
- /// 3. Collect contiguous blocks until we have enough pages
- /// 4. If successful, allocate all collected blocks
- ///
- /// # Returns
- /// Base address of the first block if contiguous blocks found, otherwise None
- fn try_combine_contiguous_blocks(
- &mut self,
- num_pages: usize,
- alignment: usize,
- ) -> Option {
- let mut remaining_pages = num_pages;
- let mut contiguous_blocks: [(usize, u32); MAX_PARTS_PER_ALLOC] =
- [(0, 0); MAX_PARTS_PER_ALLOC];
- let mut block_count = 0;
- let mut min_addr = usize::MAX;
- let mut max_addr = 0;
-
- // Iterate from largest to smallest blocks
- for order in (0..=DEFAULT_MAX_ORDER).rev() {
- let block_pages = 1usize << order;
-
- if remaining_pages == 0 || block_count >= MAX_PARTS_PER_ALLOC {
- break;
- }
-
- // Get free blocks of this order from all zones
- for zone_id in 0..self.buddy.get_zone_count() {
- if let Some(blocks) = self.buddy.get_free_blocks_by_order(zone_id, order as u32) {
- // Iterate through sorted free blocks
- for block in blocks {
- if block_count >= MAX_PARTS_PER_ALLOC {
- break;
- }
-
- let block_start = block.addr;
- let block_end = block_start + block_pages * PAGE_SIZE;
-
- // Check alignment requirement
- if !crate::is_aligned(block_start, alignment) {
- continue;
- }
-
- // Check contiguity with existing blocks
- if block_count == 0 {
- // First block - just record it
- contiguous_blocks[block_count] = (block_start, order as u32);
- min_addr = block_start;
- max_addr = block_end;
- block_count += 1;
- remaining_pages -= block_pages.min(remaining_pages);
- } else if block_end == min_addr {
- // Block is to the left, update min_addr
- contiguous_blocks[block_count] = (block_start, order as u32);
- min_addr = block_start;
- block_count += 1;
- remaining_pages -= block_pages.min(remaining_pages);
- } else if block_start == max_addr {
- // Block is to the right, update max_addr
- contiguous_blocks[block_count] = (block_start, order as u32);
- max_addr = block_end;
- block_count += 1;
- remaining_pages -= block_pages.min(remaining_pages);
- }
-
- if remaining_pages == 0 {
- break;
- }
- }
- }
-
- if remaining_pages == 0 {
- break;
- }
- }
- }
-
- // If we found enough contiguous pages, allocate them
- if remaining_pages == 0 {
- let mut parts = [(0usize, 0u32); MAX_PARTS_PER_ALLOC];
-
- // Allocate all contiguous blocks
- for i in 0..block_count {
- let (addr, order) = contiguous_blocks[i];
- let block_pages = 1usize << order;
-
- debug!(
- "Block {}: addr={:#x}, order={}, pages={}, size={} MB",
- i,
- addr,
- order,
- block_pages,
- (block_pages * PAGE_SIZE) / (1024 * 1024)
- );
-
- // Allocate this specific block
- if let Err(_e) = self.buddy.alloc_pages_at(addr, block_pages, alignment) {
- // Allocation failed, rollback
- warn!("Contiguous block allocation failed at {i}, rolling back");
- #[allow(clippy::needless_range_loop)]
- for j in 0..i {
- let (dealloc_addr, dealloc_order) = parts[j];
- let dealloc_pages = 1usize << dealloc_order;
- self.buddy.dealloc_pages(dealloc_addr, dealloc_pages);
- }
- return None;
- }
-
- parts[i] = (addr, order);
- }
-
- // Assertion: allocated pages must be >= requested pages
- let actual_pages: usize = parts[..block_count]
- .iter()
- .map(|(_, order)| 1usize << *order as usize)
- .sum();
- debug_assert!(
- actual_pages >= num_pages,
- "Allocated pages {actual_pages} < requested pages {num_pages}"
- );
-
- // Save metadata to tracker for proper deallocation
- if !self
- .composite_tracker
- .insert(min_addr, &parts[..block_count], block_count)
- {
- // Tracker is full, rollback and fail
- warn!("Composite tracker full, rolling back allocation");
- #[allow(clippy::needless_range_loop)]
- for j in 0..block_count {
- let (dealloc_addr, dealloc_order) = parts[j];
- let dealloc_pages = 1usize << dealloc_order;
- self.buddy.dealloc_pages(dealloc_addr, dealloc_pages);
- }
- return None;
- }
-
- debug!("Contiguous block allocation succeeded: base_addr={min_addr:#x}, pages={num_pages}, parts={block_count}, actual_pages={actual_pages}");
-
- return Some(min_addr);
- }
-
- None
- }
-
- /// Print detailed statistics when allocation fails.
- ///
- /// This function delegates to buddy allocator's detailed statistics reporter.
- #[cfg(feature = "tracking")]
- fn print_alloc_failure_stats(&self, num_pages: usize, alignment: usize) {
- self.buddy.print_alloc_failure_stats(num_pages, alignment);
- }
-
- #[cfg(not(feature = "tracking"))]
- fn print_alloc_failure_stats(&self, _num_pages: usize, _alignment: usize) {
- // No-op when tracking is disabled
- }
-}
-
-impl PageAllocator for CompositePageAllocator {
- const PAGE_SIZE: usize = PAGE_SIZE;
-
- fn alloc_pages(&mut self, num_pages: usize, alignment: usize) -> AllocResult {
- if num_pages == 0 {
- return Err(AllocError::InvalidParam);
- }
-
- let buddy_pages = if num_pages.is_power_of_two() {
- num_pages
- } else {
- num_pages.next_power_of_two()
- };
-
- // Try to allocate from buddy system first
- let base_addr = match self.buddy.alloc_pages(buddy_pages, alignment) {
- Ok(addr) => addr,
- Err(_) => {
- // Standard allocation failed, try contiguous block combination
- debug!(
- "Standard allocation failed, trying contiguous block combination for {num_pages} pages"
- );
- if let Some(addr) = self.try_combine_contiguous_blocks(num_pages, alignment) {
- return Ok(addr);
- }
- self.print_alloc_failure_stats(num_pages, alignment);
- return Err(AllocError::NoMemory);
- }
- };
-
- Ok(base_addr)
- }
-
- fn dealloc_pages(&mut self, pos: usize, num_pages: usize) {
- if num_pages == 0 {
- return;
- }
-
- // Check if this is a composite allocation
- if let Some(info) = self.composite_tracker.find(pos) {
- // Composite block: deallocate each part separately
- debug!(
- "Deallocating composite block: base_addr={:#x}, part_count={}",
- pos, info.part_count
- );
-
- for i in 0..info.part_count as usize {
- let (addr, order) = info.parts[i];
- let pages = 1usize << order;
- debug!(" Part {i}: addr={addr:#x}, order={order}, pages={pages}");
- self.buddy.dealloc_pages(addr, pages);
- }
-
- // Remove from tracker
- self.composite_tracker.remove(pos);
- } else {
- // Regular buddy block: deallocate directly
- if num_pages.is_power_of_two() {
- self.buddy.dealloc_pages(pos, num_pages);
- } else {
- self.buddy.dealloc_pages(pos, num_pages.next_power_of_two());
- }
- }
- }
-
- /// Allocate contiguous memory pages at a specific address.
- ///
- /// Delegates to buddy allocator.
- fn alloc_pages_at(
- &mut self,
- base: usize,
- num_pages: usize,
- alignment: usize,
- ) -> AllocResult {
- self.buddy.alloc_pages_at(base, num_pages, alignment)
- }
-
- /// Return total number of memory pages.
- fn total_pages(&self) -> usize {
- self.buddy.total_pages()
- }
-
- /// Return number of allocated memory pages.
- fn used_pages(&self) -> usize {
- self.buddy.used_pages()
- }
-
- /// Return number of available memory pages.
- fn available_pages(&self) -> usize {
- self.buddy.available_pages()
- }
-}
-
-impl CompositePageAllocator {
- /// Get buddy allocator statistics
- #[cfg(feature = "tracking")]
- pub fn get_buddy_stats(&self) -> crate::buddy::BuddyStats {
- self.buddy.get_stats()
- }
-}
-
-impl