Benchmarks refactoring

.gitignore

@@ -19,3 +19,6 @@ academic/**/*.blg
 academic/**/*.log
 academic/**/*.out
 academic/**/*.run.xml
+
+# AI
+.kilo/plans/*

CMakeLists.txt

@@ -34,10 +34,15 @@ endif()
 # Build options
 # ---------------------------------------------------------------------------
 option(PIXIE_TESTS "Build unit tests" ON)
-option(PIXIE_BENCHMARKS "Build benchmarks (includes comparison benchmarks against third-party libraries)" OFF)
+option(PIXIE_BENCHMARKS "Build benchmarks" OFF)
+option(PIXIE_THIRD_PARTY_BACKENDS "Build optional third-party backend integrations" OFF)
 option(PIXIE_DIAGNOSTICS "Include diagnostic logs" OFF)
 option(PIXIE_DOCS "Build Doxygen documentation" OFF)
 
+if(PIXIE_THIRD_PARTY_BACKENDS)
+    add_compile_definitions(SDSL_SUPPORT)
+endif()
+
 if(PIXIE_DIAGNOSTICS)
     add_compile_definitions(PIXIE_DIAGNOSTICS)
     set(PIXIE_DIAGNOSTICS_LIBS spdlog::spdlog_header_only)
@@ -67,10 +72,16 @@ if(PIXIE_BENCHMARKS)
         GIT_REPOSITORY https://github.com/google/benchmark.git
         GIT_TAG v1.9.4
     )
-    set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests")
+    set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests" FORCE)
+    set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable Google Benchmark tests" FORCE)
+    set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Disable Google Benchmark install targets" FORCE)
+    set(BENCHMARK_INSTALL_DOCS OFF CACHE BOOL "Disable Google Benchmark install docs" FORCE)
     FetchContent_MakeAvailable(googlebenchmark)
+endif()
 
-    # Third-party libraries for comparison benchmarks
+if(PIXIE_THIRD_PARTY_BACKENDS)
+    set(PASTA_BIT_VECTOR_BUILD_TESTS OFF CACHE BOOL "Disable pasta::bit_vector tests" FORCE)
+    set(PASTA_BIT_VECTOR_BUILD_BENCHMARKS OFF CACHE BOOL "Disable pasta::bit_vector benchmarks" FORCE)
     FetchContent_Declare(
         pasta_bit_vector
         GIT_REPOSITORY https://github.com/pasta-toolbox/bit_vector.git
@@ -96,6 +107,8 @@ endif()
 
 if(PIXIE_TESTS)
     if(NOT TARGET gtest_main)
+        set(BUILD_GMOCK OFF CACHE BOOL "Disable GoogleMock" FORCE)
+        set(INSTALL_GTEST OFF CACHE BOOL "Disable GoogleTest install targets" FORCE)
         FetchContent_Declare(
             googletest
             GIT_REPOSITORY https://github.com/google/googletest.git
@@ -139,6 +152,10 @@ if(PIXIE_TESTS)
         gtest
         gtest_main
         ${PIXIE_DIAGNOSTICS_LIBS})
+    if(PIXIE_THIRD_PARTY_BACKENDS)
+        target_include_directories(test_rmm
+            PRIVATE ${sdsl_lite_SOURCE_DIR}/include)
+    endif()
 
     add_executable(louds_tree_tests
         src/tests/louds_tree_tests.cpp)
@@ -170,9 +187,11 @@ if(PIXIE_BENCHMARKS)
     target_link_libraries(benchmarks
         benchmark
         benchmark_main
-        pasta_bit_vector
         ${PIXIE_DIAGNOSTICS_LIBS})
-    target_compile_definitions(benchmarks PRIVATE PIXIE_THIRD_PARTY_BENCHMARKS)
+    if(PIXIE_THIRD_PARTY_BACKENDS)
+        target_link_libraries(benchmarks pasta_bit_vector)
+        target_compile_definitions(benchmarks PRIVATE PIXIE_THIRD_PARTY_BENCHMARKS)
+    endif()
 
     add_executable(bench_rmm
         src/benchmarks/bench_rmm.cpp)
@@ -182,15 +201,17 @@ if(PIXIE_BENCHMARKS)
         benchmark
         ${PIXIE_DIAGNOSTICS_LIBS})
 
-    add_executable(bench_rmm_sdsl
-        src/benchmarks/bench_rmm_sdsl.cpp)
-    target_include_directories(bench_rmm_sdsl
-        PUBLIC include
-        PRIVATE ${sdsl_lite_SOURCE_DIR}/include)
-    target_link_libraries(bench_rmm_sdsl
-        PRIVATE
-            benchmark
-            ${PIXIE_DIAGNOSTICS_LIBS})
+    if(PIXIE_THIRD_PARTY_BACKENDS)
+        add_executable(bench_rmm_sdsl
+            src/benchmarks/bench_rmm_sdsl.cpp)
+        target_include_directories(bench_rmm_sdsl
+            PUBLIC include
+            PRIVATE ${sdsl_lite_SOURCE_DIR}/include)
+        target_link_libraries(bench_rmm_sdsl
+            PRIVATE
+                benchmark
+                ${PIXIE_DIAGNOSTICS_LIBS})
+    endif()
 
     add_executable(louds_tree_benchmarks
         src/benchmarks/louds_tree_benchmarks.cpp)
@@ -234,6 +255,16 @@ if(PIXIE_DOCS)
 
     FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR)
 
+    set(PIXIE_DOC_INPUT
+        "${CMAKE_CURRENT_SOURCE_DIR}/include"
+        "${CMAKE_CURRENT_SOURCE_DIR}/README.md")
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/docs/benchmark_results.md")
+        list(APPEND PIXIE_DOC_INPUT
+             "${CMAKE_CURRENT_SOURCE_DIR}/src/docs/benchmark_results.md")
+    endif()
+    string(JOIN " \\\n                         " PIXIE_DOC_INPUT
+           ${PIXIE_DOC_INPUT})
+
     set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/src/docs/Doxyfile.in)
     set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/docs/Doxyfile)
     configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY)

CMakePresets.json

@@ -19,7 +19,9 @@
             "inherits": "base",
             "binaryDir": "${sourceDir}/build/debug",
             "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "Debug"
+                "CMAKE_BUILD_TYPE": "Debug",
+                "PIXIE_BENCHMARKS": "OFF",
+                "PIXIE_TESTS": "ON"
             }
         },
         {
@@ -28,7 +30,9 @@
             "inherits": "base",
             "binaryDir": "${sourceDir}/build/release",
             "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "Release"
+                "CMAKE_BUILD_TYPE": "Release",
+                "PIXIE_BENCHMARKS": "OFF",
+                "PIXIE_TESTS": "ON"
             }
         },
         {
@@ -38,9 +42,22 @@
             "binaryDir": "${sourceDir}/build/release",
             "cacheVariables": {
                 "CMAKE_BUILD_TYPE": "Release",
+                "PIXIE_TESTS": "OFF",
                 "PIXIE_BENCHMARKS": "ON"
             }
         },
+        {
+            "name": "benchmarks-third-party",
+            "displayName": "Benchmarks with third-party backends",
+            "inherits": "base",
+            "binaryDir": "${sourceDir}/build/release-third-party",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Release",
+                "PIXIE_BENCHMARKS": "ON",
+                "PIXIE_TESTS": "OFF",
+                "PIXIE_THIRD_PARTY_BACKENDS": "ON"
+            }
+        },
         {
             "name": "benchmarks-diagnostic",
             "displayName": "Benchmarks diagnostic build",
@@ -50,6 +67,7 @@
                 "BENCHMARK_ENABLE_LIBPFM": "ON",
                 "CMAKE_BUILD_TYPE": "RelWithDebInfo",
                 "PIXIE_DIAGNOSTICS": "ON",
+                "PIXIE_TESTS": "OFF",
                 "PIXIE_BENCHMARKS": "ON"
             }
         },
@@ -90,17 +108,63 @@
         {
             "name": "debug",
             "displayName": "Build Debug",
-            "configurePreset": "debug"
+            "configurePreset": "debug",
+            "targets": [
+                "unittests",
+                "benchmark_tests",
+                "test_rmm",
+                "louds_tree_tests",
+                "excess_positions_tests"
+            ]
         },
         {
             "name": "release",
             "displayName": "Build Release",
-            "configurePreset": "release"
+            "configurePreset": "release",
+            "targets": [
+                "unittests",
+                "benchmark_tests",
+                "test_rmm",
+                "louds_tree_tests",
+                "excess_positions_tests"
+            ]
+        },
+        {
+            "name": "benchmarks-all",
+            "displayName": "Build Benchmarks",
+            "configurePreset": "benchmarks-all",
+            "targets": [
+                "benchmarks",
+                "bench_rmm",
+                "louds_tree_benchmarks",
+                "alignment_comparison",
+                "excess_positions_benchmarks"
+            ]
+        },
+        {
+            "name": "benchmarks-third-party",
+            "displayName": "Build Benchmarks with third-party backends",
+            "configurePreset": "benchmarks-third-party",
+            "targets": [
+                "benchmarks",
+                "bench_rmm",
+                "bench_rmm_sdsl",
+                "louds_tree_benchmarks",
+                "alignment_comparison",
+                "excess_positions_benchmarks"
+            ]
         },
         {
             "name": "benchmarks-diagnostic",
             "displayName": "Benchmarks diagnostic",
-            "configurePreset": "benchmarks-diagnostic"
+            "configurePreset": "benchmarks-diagnostic",
+            "targets": [
+                "benchmarks",
+                "bench_rmm",
+                "louds_tree_benchmarks",
+                "alignment_comparison",
+                "excess_positions_benchmarks"
+            ]
         },
         {
             "name": "docs",
@@ -113,12 +177,26 @@
         {
             "name": "coverage",
             "displayName": "Build Coverage",
-            "configurePreset": "coverage"
+            "configurePreset": "coverage",
+            "targets": [
+                "unittests",
+                "benchmark_tests",
+                "test_rmm",
+                "louds_tree_tests",
+                "excess_positions_tests"
+            ]
         },
         {
             "name": "asan",
             "displayName": "Build AddressSanitizer",
-            "configurePreset": "asan"
+            "configurePreset": "asan",
+            "targets": [
+                "unittests",
+                "benchmark_tests",
+                "test_rmm",
+                "louds_tree_tests",
+                "excess_positions_tests"
+            ]
         }
     ]
 }

README.md

@@ -51,7 +51,7 @@ cmake -B build/release -DCMAKE_BUILD_TYPE=Release
 cmake --build build/release -j
 ```
 
-Tests are enabled by default (`PIXIE_TESTS=ON`). Benchmarks are opt-in; enable with `-DPIXIE_BENCHMARKS=ON` or configure with the `benchmarks-all` preset, you can use `benchmark-diagnostic` preset for performance diagnostics (Release with debug info + performance counters support). 
+Tests are enabled by default (`PIXIE_TESTS=ON`). Benchmarks are opt-in; enable with `-DPIXIE_BENCHMARKS=ON` or configure with the `benchmarks-all` preset. Use `benchmarks-third-party` for comparison backends such as sdsl-lite, and `benchmarks-diagnostic` for performance diagnostics (Release with debug info + performance counters support).
 
 ---
 
@@ -114,20 +114,57 @@ Benchmarks are random 50/50 0-1 bitvectors up to $2^{34}$ bits.
 ./build/release/benchmarks
 ```
 
+Write JSON and plot size-scaled benchmark curves with a log-scaled x-axis:
+
+```sh
+./build/release/benchmarks --benchmark_out=bitvector_bench.json --benchmark_out_format=json
+python3 scripts/plot_size_benchmarks.py bitvector_bench.json -o graphs/bitvector_size.png --size-key n
+```
+
+### Excess Positions
+
+```sh
+./build/release/excess_positions_benchmarks --benchmark_out=excess_positions.json --benchmark_out_format=json
+python3 scripts/excess_benchmark_table.py excess_positions.json -o src/docs/excess_positions_benchmark_results.md
+```
+
+Generated benchmark documentation can be written to `src/docs/benchmark_results.md`;
+the documentation pipeline does not run benchmarks.
+
 ### RmM Tree
 
 ```sh
 ./build/release/bench_rmm
 ```
 
-For comparison with range min-max tree implementation from [sdsl-lite](https://github.com/simongog/sdsl-lite) (Release build required; use the release preset or `-DCMAKE_BUILD_TYPE=Release`):
+For focused runs, `bench_rmm` accepts `--ops` with a comma-separated operation list. The benchmark harness only builds the query pools needed by the selected operations, so subset runs avoid much of the setup cost:
+
+```sh
+./build/release/bench_rmm --ops=rank1,select1 --benchmark_out=rmm_rank_select.json --benchmark_out_format=json
+```
+
+By default, RmM benchmarks step through sizes by powers of two. Use `--per_octave=<n>` for finer sampling between adjacent powers of two, or `--explicit_sizes=<csv>` for an exact size list.
+
+Google Benchmark filters are also used to limit RmM setup when `--ops` is not provided:
+
+```sh
+./build/release/bench_rmm --benchmark_filter='^rank1$' --benchmark_out=rmm_rank1.json --benchmark_out_format=json
+```
+
+For comparison with range min-max tree implementation from [sdsl-lite](https://github.com/simongog/sdsl-lite), use the third-party benchmark preset. This defines `SDSL_SUPPORT` and builds `bench_rmm_sdsl`:
 
 ```bash
+cmake --preset benchmarks-third-party
+cmake --build --preset benchmarks-third-party
 sudo cpupower frequency-set --governor performance
-./build/release/bench_rmm_sdsl --benchmark_out=rmm_bench_sdsl.json
+./build/release-third-party/bench_rmm_sdsl --benchmark_out=rmm_bench_sdsl.json
 ```
 
-For visualization, write the JSON output to a file using `--benchmark_out=<file>` (e.g. `./build/release/bench_rmm --benchmark_out=rmm_bench.json`) and plot it with `scripts/plot_rmm.py` (add `--sdsl-json rmm_bench_sdsl.json` for comparison).
+For visualization, write the JSON output to a file using `--benchmark_out=<file>` (e.g. `./build/release/bench_rmm --benchmark_out=rmm_bench.json`) and plot it with `scripts/plot_rmm.py` (add `--sdsl-json rmm_bench_sdsl.json` for per-operation sdsl-lite comparison plots). For size-scaled tree plots, use:
+
+```sh
+python3 scripts/plot_size_benchmarks.py rmm_bench.json -o graphs/rmm_size.png --size-key N
+```
 
 ---
 
@@ -152,8 +189,10 @@ int main() {
 
 ```cpp
 #include <pixie/rmm_tree.h>
-#include <string>
+#include <cstdint>
 #include <iostream>
+#include <string>
+#include <vector>
 
 using namespace pixie;
 
@@ -166,7 +205,15 @@ int main() {
     // └─ C
     //    └─ c1
     std::string bits = "11101001011000";
-    RmMTree t(bits);
+    std::vector<std::uint64_t> words((bits.size() + 63) / 64);
+    for (std::size_t i = 0; i < bits.size(); ++i) {
+        if (bits[i] == '1') {
+            words[i / 64] |= std::uint64_t{1} << (i % 64);
+        }
+    }
+
+    // RmMTree is non-owning: keep words alive and immutable while using t.
+    RmMTree t(words, bits.size());
 
     std::cout << "close(1): " << t.close(1) << "\n";     // expected 6 (A)
     std::cout << "open(3): " << t.open(3) << "\n";       // expected 2 (a1)